kube_runtime/
finalizer.rs

1//! Finalizer helper for [`Controller`](crate::Controller) reconcilers
2use crate::controller::Action;
3use futures::{TryFuture, TryFutureExt};
4use json_patch::{jsonptr::PointerBuf, AddOperation, PatchOperation, RemoveOperation, TestOperation};
5use kube_client::{
6    api::{Patch, PatchParams},
7    Api, Resource, ResourceExt,
8};
9
10use serde::{de::DeserializeOwned, Serialize};
11use std::{error::Error as StdError, fmt::Debug, str::FromStr, sync::Arc};
12use thiserror::Error;
13
14#[derive(Debug, Error)]
15pub enum Error<ReconcileErr>
16where
17    ReconcileErr: StdError + 'static,
18{
19    #[error("failed to apply object: {0}")]
20    ApplyFailed(#[source] ReconcileErr),
21    #[error("failed to clean up object: {0}")]
22    CleanupFailed(#[source] ReconcileErr),
23    #[error("failed to add finalizer: {0}")]
24    AddFinalizer(#[source] kube_client::Error),
25    #[error("failed to remove finalizer: {0}")]
26    RemoveFinalizer(#[source] kube_client::Error),
27    #[error("object has no name")]
28    UnnamedObject,
29    #[error("invalid finalizer")]
30    InvalidFinalizer,
31}
32
33struct FinalizerState {
34    finalizer_index: Option<usize>,
35    is_deleting: bool,
36}
37
38impl FinalizerState {
39    fn for_object<K: Resource>(obj: &K, finalizer_name: &str) -> Self {
40        Self {
41            finalizer_index: obj
42                .finalizers()
43                .iter()
44                .enumerate()
45                .find(|(_, fin)| *fin == finalizer_name)
46                .map(|(i, _)| i),
47            is_deleting: obj.meta().deletion_timestamp.is_some(),
48        }
49    }
50}
51
52/// Reconcile an object in a way that requires cleanup before an object can be deleted.
53///
54/// It does this by managing a [`ObjectMeta::finalizers`] entry,
55/// which prevents the object from being deleted before the cleanup is done.
56///
57/// In typical usage, if you use `finalizer` then it should be the only top-level "action"
58/// in your [`applier`](crate::applier)/[`Controller`](crate::Controller)'s `reconcile` function.
59///
60/// # Expected Flow
61///
62/// 1. User creates object
63/// 2. Reconciler sees object
64/// 3. `finalizer` adds `finalizer_name` to [`ObjectMeta::finalizers`]
65/// 4. Reconciler sees updated object
66/// 5. `finalizer` runs [`Event::Apply`]
67/// 6. User updates object
68/// 7. Reconciler sees updated object
69/// 8. `finalizer` runs [`Event::Apply`]
70/// 9. User deletes object
71/// 10. Reconciler sees deleting object
72/// 11. `finalizer` runs [`Event::Cleanup`]
73/// 12. `finalizer` removes `finalizer_name` from [`ObjectMeta::finalizers`]
74/// 13. Kubernetes sees that all [`ObjectMeta::finalizers`] are gone and finally deletes the object
75///
76/// # Guarantees
77///
78/// If [`Event::Apply`] is ever started then [`Event::Cleanup`] must succeed before the Kubernetes object deletion completes.
79///
80/// # Assumptions
81///
82/// `finalizer_name` must be unique among the controllers interacting with the object
83///
84/// [`Event::Apply`] and [`Event::Cleanup`] must both be idempotent, and tolerate being executed several times (even if previously cancelled).
85///
86/// [`Event::Cleanup`] must tolerate [`Event::Apply`] never having ran at all, or never having succeeded. Keep in mind that
87/// even infallible `.await`s are cancellation points.
88///
89/// # Caveats
90///
91/// Object deletes will get stuck while the controller is not running, or if `cleanup` fails for some reason.
92///
93/// `reconcile` should take the object that the [`Event`] contains, rather than trying to reuse `obj`, since it may have been updated.
94///
95/// # Errors
96///
97/// [`Event::Apply`] and [`Event::Cleanup`] are both fallible, their errors are passed through as [`Error::ApplyFailed`]
98/// and [`Error::CleanupFailed`], respectively.
99///
100/// In addition, adding and removing the finalizer itself may fail. In particular, this may be because of
101/// network errors, lacking permissions, or because another `finalizer` was updated in the meantime on the same object.
102///
103/// [`ObjectMeta::finalizers`]: kube_client::api::ObjectMeta#structfield.finalizers
104pub async fn finalizer<K, ReconcileFut>(
105    api: &Api<K>,
106    finalizer_name: &str,
107    obj: Arc<K>,
108    reconcile: impl FnOnce(Event<K>) -> ReconcileFut,
109) -> Result<Action, Error<ReconcileFut::Error>>
110where
111    K: Resource + Clone + DeserializeOwned + Serialize + Debug,
112    ReconcileFut: TryFuture<Ok = Action>,
113    ReconcileFut::Error: StdError + 'static,
114{
115    match FinalizerState::for_object(&*obj, finalizer_name) {
116        FinalizerState {
117            finalizer_index: Some(_),
118            is_deleting: false,
119        } => reconcile(Event::Apply(obj))
120            .into_future()
121            .await
122            .map_err(Error::ApplyFailed),
123        FinalizerState {
124            finalizer_index: Some(finalizer_i),
125            is_deleting: true,
126        } => {
127            // Cleanup reconciliation must succeed before it's safe to remove the finalizer
128            let name = obj.meta().name.clone().ok_or(Error::UnnamedObject)?;
129            let action = reconcile(Event::Cleanup(obj))
130                .into_future()
131                .await
132                // Short-circuit, so that we keep the finalizer if cleanup fails
133                .map_err(Error::CleanupFailed)?;
134            // Cleanup was successful, remove the finalizer so that deletion can continue
135            let finalizer_path = format!("/metadata/finalizers/{finalizer_i}");
136            api.patch::<K>(
137                &name,
138                &PatchParams::default(),
139                &Patch::Json(json_patch::Patch(vec![
140                    // All finalizers run concurrently and we use an integer index
141                    // `Test` ensures that we fail instead of deleting someone else's finalizer
142                    // (in which case a new `Cleanup` event will be sent)
143                    PatchOperation::Test(TestOperation {
144                        path: PointerBuf::from_str(finalizer_path.as_str())
145                            .map_err(|_err| Error::InvalidFinalizer)?,
146                        value: finalizer_name.into(),
147                    }),
148                    PatchOperation::Remove(RemoveOperation {
149                        path: PointerBuf::from_str(finalizer_path.as_str())
150                            .map_err(|_err| Error::InvalidFinalizer)?,
151                    }),
152                ])),
153            )
154            .await
155            .map_err(Error::RemoveFinalizer)?;
156            Ok(action)
157        }
158        FinalizerState {
159            finalizer_index: None,
160            is_deleting: false,
161        } => {
162            // Finalizer must be added before it's safe to run an `Apply` reconciliation
163            let patch = json_patch::Patch(if obj.finalizers().is_empty() {
164                vec![
165                    PatchOperation::Test(TestOperation {
166                        path: PointerBuf::from_str("/metadata/finalizers")
167                            .map_err(|_err| Error::InvalidFinalizer)?,
168                        value: serde_json::Value::Null,
169                    }),
170                    PatchOperation::Add(AddOperation {
171                        path: PointerBuf::from_str("/metadata/finalizers")
172                            .map_err(|_err| Error::InvalidFinalizer)?,
173                        value: vec![finalizer_name].into(),
174                    }),
175                ]
176            } else {
177                vec![
178                    // Kubernetes doesn't automatically deduplicate finalizers (see
179                    // https://github.com/kube-rs/kube/issues/964#issuecomment-1197311254),
180                    // so we need to fail and retry if anyone else has added the finalizer in the meantime
181                    PatchOperation::Test(TestOperation {
182                        path: PointerBuf::from_str("/metadata/finalizers")
183                            .map_err(|_err| Error::InvalidFinalizer)?,
184                        value: obj.finalizers().into(),
185                    }),
186                    PatchOperation::Add(AddOperation {
187                        path: PointerBuf::from_str("/metadata/finalizers/-")
188                            .map_err(|_err| Error::InvalidFinalizer)?,
189                        value: finalizer_name.into(),
190                    }),
191                ]
192            });
193            api.patch::<K>(
194                obj.meta().name.as_deref().ok_or(Error::UnnamedObject)?,
195                &PatchParams::default(),
196                &Patch::Json(patch),
197            )
198            .await
199            .map_err(Error::AddFinalizer)?;
200            // No point applying here, since the patch will cause a new reconciliation
201            Ok(Action::await_change())
202        }
203        FinalizerState {
204            finalizer_index: None,
205            is_deleting: true,
206        } => {
207            // Our work here is done
208            Ok(Action::await_change())
209        }
210    }
211}
212
213/// A representation of an action that should be taken by a reconciler.
214pub enum Event<K> {
215    /// The reconciler should ensure that the actual state matches the state desired in the object.
216    ///
217    /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
218    ///
219    /// - The controller is restarted
220    /// - The object is updated
221    /// - The reconciliation fails
222    /// - The grinch attacks
223    Apply(Arc<K>),
224    /// The object is being deleted, and the reconciler should remove all resources that it owns.
225    ///
226    /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
227    ///
228    /// - The controller is restarted while the deletion is in progress
229    /// - The reconciliation fails
230    /// - Another finalizer was removed in the meantime
231    /// - The grinch's heart grows a size or two
232    Cleanup(Arc<K>),
233}