diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b078edf..60e24fb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#792]). - BREAKING: The app.kubernetes.io/managed-by label value changed from hbase.stackable.com_hbasecluster to hbase.stackable.tech_hbasecluster, aligning with all other operators ([#795]). +- The operator now watches all resources that it creates and early-exits the reconcile action when the + cluster is marked for deletion ([#797]). - BREAKING: The rest-server listener PVC template now carries the recommended labels without the version label, so that the labels stay stable across upgrades. Existing rest-server StatefulSets must be deleted once before the new operator can reconcile them ([#799]). @@ -37,6 +39,7 @@ [#787]: https://github.com/stackabletech/hbase-operator/pull/787 [#792]: https://github.com/stackabletech/hbase-operator/pull/792 [#795]: https://github.com/stackabletech/hbase-operator/pull/795 +[#797]: https://github.com/stackabletech/hbase-operator/pull/797 [#799]: https://github.com/stackabletech/hbase-operator/pull/799 ## [26.7.0] - 2026-07-21 diff --git a/deploy/helm/hbase-operator/templates/clusterrole-operator.yaml b/deploy/helm/hbase-operator/templates/clusterrole-operator.yaml index 6b0fc340..030741be 100644 --- a/deploy/helm/hbase-operator/templates/clusterrole-operator.yaml +++ b/deploy/helm/hbase-operator/templates/clusterrole-operator.yaml @@ -18,10 +18,13 @@ rules: # orphan cleanup (list + delete). # - configmaps: role group configuration and discovery configmap # - services: role group headless and metrics services + # - serviceaccounts created per HbaseCluster for workload pod identity. + # Applied via SSA, tracked for orphan cleanup and watched by the controller. - apiGroups: - "" resources: - configmaps + - serviceaccounts - services verbs: - create @@ -30,20 +33,8 @@ rules: - list - patch - watch - # ServiceAccount created per HbaseCluster for workload pod identity. - # Applied via SSA and tracked for orphan cleanup. - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - delete - - get - - list - - patch # RoleBinding created per HbaseCluster to bind the product ClusterRole to the workload - # ServiceAccount. Applied via SSA and tracked for orphan cleanup. + # ServiceAccount. Applied via SSA, tracked for orphan cleanup and watched by the controller. - apiGroups: - rbac.authorization.k8s.io resources: @@ -54,6 +45,7 @@ rules: - get - list - patch + - watch # Required to bind the product ClusterRole to the per-cluster ServiceAccount. - apiGroups: - rbac.authorization.k8s.io @@ -75,7 +67,8 @@ rules: - list - patch - watch - # PodDisruptionBudget created per role. Applied via SSA and tracked for orphan cleanup. + # PodDisruptionBudget created per role. Applied via SSA, tracked for orphan cleanup and + # watched by the controller. - apiGroups: - policy resources: @@ -86,6 +79,7 @@ rules: - get - list - patch + - watch # Required for maintaining the CRDs within the operator (including the conversion webhook info). # Also for the startup condition check before the controller can run. - apiGroups: diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 6b6751d2..0c9ff348 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -12,6 +12,7 @@ use stackable_operator::{ cli::OperatorEnvironmentOptions, cluster_resources::ClusterResourceApplyStrategy, kube::{ + Resource, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, @@ -76,6 +77,10 @@ pub async fn reconcile_hbase( ) -> Result { tracing::info!("Starting reconcile"); + if hbase.meta().deletion_timestamp.is_some() { + return Ok(Action::await_change()); + } + let hbase = hbase .0 .as_ref() @@ -126,3 +131,56 @@ pub fn error_policy( _ => Action::requeue(*Duration::from_secs(5)), } } + +#[cfg(test)] +mod tests { + use indoc::indoc; + use stackable_operator::{ + client::Client, + kube::{Client as KubeClient, Config}, + }; + + use super::*; + use crate::test_utils; + + /// The client points at a closed port, so any API call would fail the reconciliation: an `Ok` + /// proves that a cluster being deleted returns before the reconciler touches the Kubernetes + /// API, and because the spec is invalid, before the [`DeserializeGuard`] is unwrapped. + #[tokio::test] + async fn reconcile_exits_early_for_deleted_cluster() { + let hbase = serde_yaml::from_str(indoc! {r#" + --- + apiVersion: hbase.stackable.tech/v1alpha1 + kind: HbaseCluster + metadata: + name: hbase + namespace: default + deletionTimestamp: "2026-08-14T12:00:00Z" + spec: {} + "#}) + .expect("YAML parses; the invalid spec is captured inside the DeserializeGuard"); + + let ctx = Arc::new(Ctx { + client: Client::new( + KubeClient::try_from(Config::new( + "http://127.0.0.1:1".parse().expect("valid static URI"), + )) + .expect("client from static config"), + None, + "default".to_owned(), + test_utils::cluster_info(), + ), + operator_environment: OperatorEnvironmentOptions { + operator_namespace: "stackable-operators".to_owned(), + operator_service_name: "hbase-operator".to_owned(), + image_repository: "oci.stackable.tech/sdp".to_owned(), + }, + }); + + let action = reconcile_hbase(Arc::new(hbase), ctx) + .await + .expect("a deleted cluster reconciles without any API call"); + + assert_eq!(action, Action::await_change()); + } +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 155488cc..8e7eabb8 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -14,7 +14,9 @@ use stackable_operator::{ eos::EndOfSupportChecker, k8s_openapi::api::{ apps::v1::StatefulSet, - core::v1::{ConfigMap, Service}, + core::v1::{ConfigMap, Service, ServiceAccount}, + policy::v1::PodDisruptionBudget, + rbac::v1::RoleBinding, }, kube::{ CustomResourceExt, ResourceExt, @@ -126,10 +128,26 @@ async fn main() -> anyhow::Result<()> { ); let config_map_store = hbase_controller.store(); let hbase_controller = hbase_controller + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) .owns( watch_namespace.get_api::(&client), watcher::Config::default(), ) + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) .owns( watch_namespace.get_api::(&client), watcher::Config::default(), diff --git a/tests/templates/kuttl/cluster-operation/40-assert.yaml b/tests/templates/kuttl/cluster-operation/40-assert.yaml new file mode 100644 index 00000000..383419ec --- /dev/null +++ b/tests/templates/kuttl/cluster-operation/40-assert.yaml @@ -0,0 +1,174 @@ +--- +# The recreated StatefulSets must bring the cluster back to ready, and the recreated +# objects must carry an owner reference back to the HbaseCluster so that garbage +# collection still works for them. +# +# The budget is generous on purpose: recreating the role-group ConfigMaps makes the +# commons-operator restarter roll every pod once more on top of the StatefulSet +# recreation, and on a node without a cached image a single pull can take minutes. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: recreate-owned-resources +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available hbaseclusters.hbase.stackable.tech/test-hbase --timeout 601s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-master-default + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-regionserver-default + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-restserver-default + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-hbase-serviceaccount + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-hbase-rolebinding + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: test-hbase-master + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: test-hbase-regionserver + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: test-hbase-restserver + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-hbase + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hbase-master-default-headless + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hbase-master-default-metrics + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hbase-regionserver-default-headless + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hbase-regionserver-default-metrics + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hbase-restserver-default-headless + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hbase-restserver-default-metrics + ownerReferences: + - apiVersion: hbase.stackable.tech/v1alpha1 + controller: true + kind: HbaseCluster + name: test-hbase diff --git a/tests/templates/kuttl/cluster-operation/40-delete-owned-resources.yaml b/tests/templates/kuttl/cluster-operation/40-delete-owned-resources.yaml new file mode 100644 index 00000000..f23afd5b --- /dev/null +++ b/tests/templates/kuttl/cluster-operation/40-delete-owned-resources.yaml @@ -0,0 +1,75 @@ +--- +# Every resource the operator applies carries an ownerReference and a `.owns()` watch +# (main.rs); deleting it must trigger a reconcile of the HbaseCluster, which re-applies +# it. This step checks that chain — the ClusterRole's `watch` verbs plus the `.owns()` +# routing — and lives here because this test already cycles the pods. The `.watches()` +# registration (referenced-but-unowned ConfigMaps) can't be tested by deletion: the +# operator never recreates what it didn't apply. +# +# The resources are discovered by label rather than listed by name: everything the +# operator applies is guaranteed to carry the `instance` and `managed-by` labels +# (ClusterResources::add rejects resources without them), so a newly applied resource +# or kind is swept automatically — including one whose `.owns()` watch or ClusterRole +# `watch` verb was forgotten, which then fails the recreation check below. +# +# The labels alone over-match: derived objects inherit them (the listener-operator's +# per-pod Listeners, for example, carry this operator's `managed-by`), so each match +# is additionally gated on its controller ownerReference pointing at the HbaseCluster — +# the authoritative marker for "applied by this operator", and the same relationship +# the orphan cleanup in operator-rs filters on. Cheap bulk kinds that could never pass +# that gate are excluded up front: Pods and ControllerRevisions (StatefulSet-derived), +# Endpoints/EndpointSlices (Service-derived), PersistentVolumeClaims (bound claims +# wedge in Terminating under a running pod), and Events. +# +# Recreation is proven by UID change: mere existence could pass without any deletion. +# TestStep commands run exactly once (no kuttl retry loop), so the polling stays quiet. +# Deleting the role-group ConfigMaps briefly leaves their mounts without a source, but +# recreation is a single reconcile away and already-mounted volumes are unaffected. +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: delete-owned-resources +timeout: 300 +commands: + - script: | + set -eu + + delete_and_await_recreation() { + resource=$1 + old_uid=$(kubectl get -n "$NAMESPACE" "$resource" -o jsonpath='{.metadata.uid}') + kubectl delete -n "$NAMESPACE" "$resource" --wait=false + # Recreation is a single reconcile away, so this normally succeeds on the + # first iteration; 30s is a generous upper bound well below the step timeout. + for _ in $(seq 1 30); do + new_uid=$(kubectl get -n "$NAMESPACE" "$resource" -o jsonpath='{.metadata.uid}' 2>/dev/null || true) + if [ -n "$new_uid" ] && [ "$new_uid" != "$old_uid" ]; then + return 0 + fi + sleep 1 + done + echo "$resource was not recreated (old uid: $old_uid, current: '${new_uid:-}')" >&2 + return 1 + } + + selector="app.kubernetes.io/instance=test-hbase,app.kubernetes.io/managed-by=hbase.stackable.tech_hbasecluster" + excluded="^(pods|persistentvolumeclaims|endpoints|events)$|^endpointslices\.|^controllerrevisions\.|^events\." + + deleted=0 + for kind in $(kubectl api-resources --verbs=list --namespaced -o name | grep -Ev "$excluded" | sort); do + for resource in $(kubectl get -n "$NAMESPACE" "$kind" -l "$selector" -o name 2>/dev/null); do + owner=$(kubectl get -n "$NAMESPACE" "$resource" -o jsonpath='{.metadata.ownerReferences[?(@.controller==true)].kind}/{.metadata.ownerReferences[?(@.controller==true)].name}' 2>/dev/null || true) + if [ "$owner" != "HbaseCluster/test-hbase" ]; then + echo "skipping $resource: controller owner is '${owner:-none}', not the HbaseCluster" + continue + fi + delete_and_await_recreation "$resource" + deleted=$((deleted + 1)) + done + done + + # Guard against the sweep silently matching nothing (wrong selector, renamed + # labels): the fixture is known to produce well over this many owned resources. + if [ "$deleted" -lt 10 ]; then + echo "only $deleted labelled resources were swept - the label selector is broken" >&2 + exit 1 + fi diff --git a/tests/templates/kuttl/smoke/30-assert.yaml.j2 b/tests/templates/kuttl/smoke/30-assert.yaml.j2 index a29810e0..502b7ae1 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-assert.yaml.j2 @@ -384,7 +384,7 @@ spec: securityContext: fsGroup: 1000 serviceAccountName: test-hbase-serviceaccount - terminationGracePeriodSeconds: 1200 + terminationGracePeriodSeconds: 60 volumes: - configMap: name: test-hbase-master-default @@ -619,7 +619,7 @@ spec: securityContext: fsGroup: 1000 serviceAccountName: test-hbase-serviceaccount - terminationGracePeriodSeconds: 3600 + terminationGracePeriodSeconds: 60 volumes: - configMap: name: test-hbase-regionserver-default @@ -850,7 +850,7 @@ spec: securityContext: fsGroup: 1000 serviceAccountName: test-hbase-serviceaccount - terminationGracePeriodSeconds: 300 + terminationGracePeriodSeconds: 60 volumes: - configMap: name: test-hbase-restserver-default diff --git a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 index 7535a3e8..8c28c42f 100644 --- a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 @@ -20,6 +20,9 @@ spec: {% endif %} masters: config: + # Test-only: the production default (20m) becomes the pods' terminationGracePeriodSeconds, + # which the namespace controller uses to schedule its deletion retries. + gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} listenerClass: {{ test_scenario['values']['listener-class'] }} @@ -32,6 +35,7 @@ spec: replicas: 2 regionServers: config: + gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} listenerClass: {{ test_scenario['values']['listener-class'] }} @@ -44,6 +48,7 @@ spec: replicas: 2 restServers: config: + gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} listenerClass: {{ test_scenario['values']['listener-class'] }}