kube_runtime/finalizer.rs
1//! Finalizer helper for [`Controller`](crate::Controller) reconcilers
2use crate::controller::Action;
3use futures::{TryFuture, TryFutureExt};
4use json_patch::{jsonptr::PointerBuf, AddOperation, PatchOperation, RemoveOperation, TestOperation};
5use kube_client::{
6 api::{Patch, PatchParams},
7 Api, Resource, ResourceExt,
8};
9
10use serde::{de::DeserializeOwned, Serialize};
11use std::{error::Error as StdError, fmt::Debug, str::FromStr, sync::Arc};
12use thiserror::Error;
13
14#[derive(Debug, Error)]
15pub enum Error<ReconcileErr>
16where
17 ReconcileErr: StdError + 'static,
18{
19 #[error("failed to apply object: {0}")]
20 ApplyFailed(#[source] ReconcileErr),
21 #[error("failed to clean up object: {0}")]
22 CleanupFailed(#[source] ReconcileErr),
23 #[error("failed to add finalizer: {0}")]
24 AddFinalizer(#[source] kube_client::Error),
25 #[error("failed to remove finalizer: {0}")]
26 RemoveFinalizer(#[source] kube_client::Error),
27 #[error("object has no name")]
28 UnnamedObject,
29 #[error("invalid finalizer")]
30 InvalidFinalizer,
31}
32
33struct FinalizerState {
34 finalizer_index: Option<usize>,
35 is_deleting: bool,
36}
37
38impl FinalizerState {
39 fn for_object<K: Resource>(obj: &K, finalizer_name: &str) -> Self {
40 Self {
41 finalizer_index: obj
42 .finalizers()
43 .iter()
44 .enumerate()
45 .find(|(_, fin)| *fin == finalizer_name)
46 .map(|(i, _)| i),
47 is_deleting: obj.meta().deletion_timestamp.is_some(),
48 }
49 }
50}
51
52/// Reconcile an object in a way that requires cleanup before an object can be deleted.
53///
54/// It does this by managing a [`ObjectMeta::finalizers`] entry,
55/// which prevents the object from being deleted before the cleanup is done.
56///
57/// In typical usage, if you use `finalizer` then it should be the only top-level "action"
58/// in your [`applier`](crate::applier)/[`Controller`](crate::Controller)'s `reconcile` function.
59///
60/// # Expected Flow
61///
62/// 1. User creates object
63/// 2. Reconciler sees object
64/// 3. `finalizer` adds `finalizer_name` to [`ObjectMeta::finalizers`]
65/// 4. Reconciler sees updated object
66/// 5. `finalizer` runs [`Event::Apply`]
67/// 6. User updates object
68/// 7. Reconciler sees updated object
69/// 8. `finalizer` runs [`Event::Apply`]
70/// 9. User deletes object
71/// 10. Reconciler sees deleting object
72/// 11. `finalizer` runs [`Event::Cleanup`]
73/// 12. `finalizer` removes `finalizer_name` from [`ObjectMeta::finalizers`]
74/// 13. Kubernetes sees that all [`ObjectMeta::finalizers`] are gone and finally deletes the object
75///
76/// # Guarantees
77///
78/// If [`Event::Apply`] is ever started then [`Event::Cleanup`] must succeed before the Kubernetes object deletion completes.
79///
80/// # Assumptions
81///
82/// `finalizer_name` must be unique among the controllers interacting with the object
83///
84/// [`Event::Apply`] and [`Event::Cleanup`] must both be idempotent, and tolerate being executed several times (even if previously cancelled).
85///
86/// [`Event::Cleanup`] must tolerate [`Event::Apply`] never having ran at all, or never having succeeded. Keep in mind that
87/// even infallible `.await`s are cancellation points.
88///
89/// # Caveats
90///
91/// Object deletes will get stuck while the controller is not running, or if `cleanup` fails for some reason.
92///
93/// `reconcile` should take the object that the [`Event`] contains, rather than trying to reuse `obj`, since it may have been updated.
94///
95/// # Errors
96///
97/// [`Event::Apply`] and [`Event::Cleanup`] are both fallible, their errors are passed through as [`Error::ApplyFailed`]
98/// and [`Error::CleanupFailed`], respectively.
99///
100/// In addition, adding and removing the finalizer itself may fail. In particular, this may be because of
101/// network errors, lacking permissions, or because another `finalizer` was updated in the meantime on the same object.
102///
103/// [`ObjectMeta::finalizers`]: kube_client::api::ObjectMeta#structfield.finalizers
104pub async fn finalizer<K, ReconcileFut>(
105 api: &Api<K>,
106 finalizer_name: &str,
107 obj: Arc<K>,
108 reconcile: impl FnOnce(Event<K>) -> ReconcileFut,
109) -> Result<Action, Error<ReconcileFut::Error>>
110where
111 K: Resource + Clone + DeserializeOwned + Serialize + Debug,
112 ReconcileFut: TryFuture<Ok = Action>,
113 ReconcileFut::Error: StdError + 'static,
114{
115 match FinalizerState::for_object(&*obj, finalizer_name) {
116 FinalizerState {
117 finalizer_index: Some(_),
118 is_deleting: false,
119 } => reconcile(Event::Apply(obj))
120 .into_future()
121 .await
122 .map_err(Error::ApplyFailed),
123 FinalizerState {
124 finalizer_index: Some(finalizer_i),
125 is_deleting: true,
126 } => {
127 // Cleanup reconciliation must succeed before it's safe to remove the finalizer
128 let name = obj.meta().name.clone().ok_or(Error::UnnamedObject)?;
129 let action = reconcile(Event::Cleanup(obj))
130 .into_future()
131 .await
132 // Short-circuit, so that we keep the finalizer if cleanup fails
133 .map_err(Error::CleanupFailed)?;
134 // Cleanup was successful, remove the finalizer so that deletion can continue
135 let finalizer_path = format!("/metadata/finalizers/{finalizer_i}");
136 api.patch::<K>(
137 &name,
138 &PatchParams::default(),
139 &Patch::Json(json_patch::Patch(vec![
140 // All finalizers run concurrently and we use an integer index
141 // `Test` ensures that we fail instead of deleting someone else's finalizer
142 // (in which case a new `Cleanup` event will be sent)
143 PatchOperation::Test(TestOperation {
144 path: PointerBuf::from_str(finalizer_path.as_str())
145 .map_err(|_err| Error::InvalidFinalizer)?,
146 value: finalizer_name.into(),
147 }),
148 PatchOperation::Remove(RemoveOperation {
149 path: PointerBuf::from_str(finalizer_path.as_str())
150 .map_err(|_err| Error::InvalidFinalizer)?,
151 }),
152 ])),
153 )
154 .await
155 .map_err(Error::RemoveFinalizer)?;
156 Ok(action)
157 }
158 FinalizerState {
159 finalizer_index: None,
160 is_deleting: false,
161 } => {
162 // Finalizer must be added before it's safe to run an `Apply` reconciliation
163 let patch = json_patch::Patch(if obj.finalizers().is_empty() {
164 vec![
165 PatchOperation::Test(TestOperation {
166 path: PointerBuf::from_str("/metadata/finalizers")
167 .map_err(|_err| Error::InvalidFinalizer)?,
168 value: serde_json::Value::Null,
169 }),
170 PatchOperation::Add(AddOperation {
171 path: PointerBuf::from_str("/metadata/finalizers")
172 .map_err(|_err| Error::InvalidFinalizer)?,
173 value: vec![finalizer_name].into(),
174 }),
175 ]
176 } else {
177 vec![
178 // Kubernetes doesn't automatically deduplicate finalizers (see
179 // https://github.com/kube-rs/kube/issues/964#issuecomment-1197311254),
180 // so we need to fail and retry if anyone else has added the finalizer in the meantime
181 PatchOperation::Test(TestOperation {
182 path: PointerBuf::from_str("/metadata/finalizers")
183 .map_err(|_err| Error::InvalidFinalizer)?,
184 value: obj.finalizers().into(),
185 }),
186 PatchOperation::Add(AddOperation {
187 path: PointerBuf::from_str("/metadata/finalizers/-")
188 .map_err(|_err| Error::InvalidFinalizer)?,
189 value: finalizer_name.into(),
190 }),
191 ]
192 });
193 api.patch::<K>(
194 obj.meta().name.as_deref().ok_or(Error::UnnamedObject)?,
195 &PatchParams::default(),
196 &Patch::Json(patch),
197 )
198 .await
199 .map_err(Error::AddFinalizer)?;
200 // No point applying here, since the patch will cause a new reconciliation
201 Ok(Action::await_change())
202 }
203 FinalizerState {
204 finalizer_index: None,
205 is_deleting: true,
206 } => {
207 // Our work here is done
208 Ok(Action::await_change())
209 }
210 }
211}
212
213/// A representation of an action that should be taken by a reconciler.
214pub enum Event<K> {
215 /// The reconciler should ensure that the actual state matches the state desired in the object.
216 ///
217 /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
218 ///
219 /// - The controller is restarted
220 /// - The object is updated
221 /// - The reconciliation fails
222 /// - The grinch attacks
223 Apply(Arc<K>),
224 /// The object is being deleted, and the reconciler should remove all resources that it owns.
225 ///
226 /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
227 ///
228 /// - The controller is restarted while the deletion is in progress
229 /// - The reconciliation fails
230 /// - Another finalizer was removed in the meantime
231 /// - The grinch's heart grows a size or two
232 Cleanup(Arc<K>),
233}