Merge pull request #4324 from vyaghras/remove_host_container_migration

Remove host container migration

Release.toml

@@ -1,4 +1,4 @@
-version = "1.32.0"
+version = "1.33.0"
 
 [migrations]
 "(0.3.1, 0.3.2)" = ["migrate_v0.3.2_admin-container-v0-5-0.lz4"]
@@ -395,3 +395,10 @@ version = "1.32.0"
     "migrate_v1.31.0_public-control-container-v0-7-20.lz4",
 ]
 "(1.31.0, 1.32.0)" = []
+"(1.32.0, 1.33.0)" = [
+    "migrate_v1.33.0_aws-remove-schnauzer-admin.lz4",
+    "migrate_v1.33.0_aws-remove-schnauzer-control.lz4",
+    "migrate_v1.33.0_public-remove-source-admin.lz4",
+    "migrate_v1.33.0_public-remove-source-control.lz4",
+    "migrate_v1.33.0_remove-metadata-and-weak-settings-migration.lz4",
+]

Twoliter.toml

@@ -1,5 +1,5 @@
 schema-version = 1
-release-version = "1.32.0"
+release-version = "1.33.0"
 
 [vendor.bottlerocket]
 registry = "public.ecr.aws/bottlerocket"

sources/Cargo.toml

@@ -81,6 +81,11 @@ members = [
     "settings-migrations/v1.31.0/public-admin-container-v0-11-16",
     "settings-migrations/v1.31.0/aws-control-container-v0-7-20",
     "settings-migrations/v1.31.0/public-control-container-v0-7-20",
+    "settings-migrations/v1.33.0/aws-remove-schnauzer-admin",
+    "settings-migrations/v1.33.0/aws-remove-schnauzer-control",
+    "settings-migrations/v1.33.0/public-remove-source-admin",
+    "settings-migrations/v1.33.0/public-remove-source-control",
+    "settings-migrations/v1.33.0/remove-metadata-and-weak-settings-migration",
 
     "settings-plugins/aws-dev",
     "settings-plugins/aws-ecs-1",

sources/api/apiclient/README.md

@@ -174,6 +174,10 @@ You can see all your pending settings like this:
 ```shell
 apiclient raw -u /tx
 ```
+You can also see pending metadata along with pending setting using version 2 of `/tx` like this:
+```shell
+apiclient raw -u /v2/tx
+```
 
 To *commit* the settings, and let the system apply them to any relevant configuration files or services, do this:
 ```shell

sources/api/apiclient/README.tpl

@@ -174,6 +174,10 @@ You can see all your pending settings like this:
 ```shell
 apiclient raw -u /tx
 ```
+You can also see pending metadata along with pending setting using version 2 of `/tx` like this:
+```shell
+apiclient raw -u /v2/tx
+```
 
 To *commit* the settings, and let the system apply them to any relevant configuration files or services, do this:
 ```shell

sources/api/datastore/Cargo.toml

@@ -16,6 +16,7 @@ serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 snafu.workspace = true
 walkdir.workspace = true
+serde_plain.workspace = true
 
 [build-dependencies]
 generate-readme.workspace = true

sources/api/datastore/src/constraints_check.rs

@@ -0,0 +1,52 @@
+//! The outcome of the constraint check determines whether the transaction can proceed to commit.
+//! A ‘rejected’ result means that one or more constraints have not been satisfied,
+//! preventing the transaction from being committed. On the other hand, an ‘approved’
+//! result confirms that all constraints are satisfied and provides the required
+//! settings and metadata for the commit.
+//! Constraint checks can alter the write.
+
+use std::collections::HashMap;
+
+use crate::{error, Key};
+
+type RejectReason = String;
+
+/// Represents a successful write operation after constraints have been approved.
+/// Contains the following fields:
+/// - `settings`: A collection of key-value pairs representing the settings to be committed.
+/// - `metadata`: A collection of metadata entries.
+#[derive(PartialEq)]
+pub struct ApprovedWrite {
+    pub settings: HashMap<Key, String>,
+    pub metadata: Vec<(Key, Key, String)>,
+}
+
+/// Represents the result of a constraint check.
+/// The result can either reject the operation or approve it with the required data.
+#[derive(PartialEq)]
+pub enum ConstraintCheckResult {
+    Reject(RejectReason),
+    Approve(ApprovedWrite),
+}
+
+impl TryFrom<ConstraintCheckResult> for ApprovedWrite {
+    type Error = error::Error;
+
+    fn try_from(constraint_check_result: ConstraintCheckResult) -> Result<Self, Self::Error> {
+        match constraint_check_result {
+            ConstraintCheckResult::Reject(err) => error::ConstraintCheckRejectSnafu { err }.fail(),
+            ConstraintCheckResult::Approve(approved_write) => Ok(approved_write),
+        }
+    }
+}
+
+impl From<Option<ApprovedWrite>> for ConstraintCheckResult {
+    fn from(approved_write: Option<ApprovedWrite>) -> Self {
+        match approved_write {
+            None => ConstraintCheckResult::Reject(
+                "The write for the given transaction is rejected".to_string(),
+            ),
+            Some(approved_write) => ConstraintCheckResult::Approve(approved_write),
+        }
+    }
+}

sources/api/datastore/src/error.rs

@@ -59,6 +59,20 @@ pub enum Error {
 
     #[snafu(display("Key name beyond maximum length {}: {}", name, max))]
     KeyTooLong { name: String, max: usize },
+
+    #[snafu(display("Unable to serialize data: {}", source))]
+    Serialize { source: serde_json::Error },
+
+    #[snafu(display("Unable to run the check constraint function: {}", source))]
+    CheckConstraintExecution {
+        source: Box<dyn std::error::Error + Send + Sync>,
+    },
+
+    #[snafu(display(
+        "Check constraint function rejected the transaction. Aborting commit : {}",
+        err
+    ))]
+    ConstraintCheckReject { err: String },
 }
 
-pub type Result<T> = std::result::Result<T, Error>;
+pub type Result<T, E = Error> = std::result::Result<T, E>;

sources/api/datastore/src/filesystem.rs

@@ -13,6 +13,8 @@ use std::io;
 use std::path::{self, Path, PathBuf};
 use walkdir::{DirEntry, WalkDir};
 
+use crate::constraints_check::{ApprovedWrite, ConstraintCheckResult};
+
 use super::key::{Key, KeyType};
 use super::{error, Committed, DataStore, Result};
 
@@ -413,14 +415,15 @@ impl DataStore for FilesystemDataStore {
     fn list_populated_metadata<S1, S2>(
         &self,
         prefix: S1,
+        committed: &Committed,
         metadata_key_name: &Option<S2>,
     ) -> Result<HashMap<Key, HashSet<Key>>>
     where
         S1: AsRef<str>,
         S2: AsRef<str>,
     {
         // Find metadata key paths on disk
-        let key_paths = find_populated_key_paths(self, KeyType::Meta, prefix, &Committed::Live)?;
+        let key_paths = find_populated_key_paths(self, KeyType::Meta, prefix, committed)?;
 
         // For each file on disk, check the user's conditions, and add it to our output
         let mut result = HashMap::new();
@@ -460,8 +463,13 @@ impl DataStore for FilesystemDataStore {
         self.delete_key_path(path, committed)
     }
 
-    fn get_metadata_raw(&self, metadata_key: &Key, data_key: &Key) -> Result<Option<String>> {
-        let path = self.metadata_path(metadata_key, data_key, &Committed::Live)?;
+    fn get_metadata_raw(
+        &self,
+        metadata_key: &Key,
+        data_key: &Key,
+        committed: &Committed,
+    ) -> Result<Option<String>> {
+        let path = self.metadata_path(metadata_key, data_key, committed)?;
         read_file_for_key(metadata_key, &path)
     }
 
@@ -470,8 +478,9 @@ impl DataStore for FilesystemDataStore {
         metadata_key: &Key,
         data_key: &Key,
         value: S,
+        committed: &Committed,
     ) -> Result<()> {
-        let path = self.metadata_path(metadata_key, data_key, &Committed::Live)?;
+        let path = self.metadata_path(metadata_key, data_key, committed)?;
         write_file_mkdir(path, value)
     }
 
@@ -482,27 +491,57 @@ impl DataStore for FilesystemDataStore {
 
     /// We commit by copying pending keys to live, then removing pending.  Something smarter (lock,
     /// atomic flip, etc.) will be required to make the server concurrent.
-    fn commit_transaction<S>(&mut self, transaction: S) -> Result<HashSet<Key>>
+    fn commit_transaction<S, C>(
+        &mut self,
+        transaction: S,
+        constraint_check: &C,
+    ) -> Result<HashSet<Key>>
     where
         S: Into<String> + AsRef<str>,
+        C: Fn(
+            &mut Self,
+            &Committed,
+        ) -> std::result::Result<
+            ConstraintCheckResult,
+            Box<dyn std::error::Error + Send + Sync + 'static>,
+        >,
     {
+        let mut pending_keys: HashSet<Key> = Default::default();
+
+        let transactions = self.list_transactions()?;
+        if !transactions.contains(transaction.as_ref()) {
+            return Ok(pending_keys);
+        }
+
         let pending = Committed::Pending {
             tx: transaction.into(),
         };
-        // Get data for changed keys
-        let pending_data = self.get_prefix("settings.", &pending)?;
 
-        // Nothing to do if no keys are present in pending
-        if pending_data.is_empty() {
-            return Ok(Default::default());
+        let constraints_check_result =
+            constraint_check(self, &pending).context(error::CheckConstraintExecutionSnafu)?;
+
+        let approved_write = ApprovedWrite::try_from(constraints_check_result)?;
+
+        trace!(
+            "commit_transaction: transaction_metadata: {:?}",
+            approved_write.metadata
+        );
+
+        // write the metadata.
+        for (metadata_key, data_key, value) in approved_write.metadata {
+            self.set_metadata(&metadata_key, &data_key, value, &Committed::Live)?;
         }
 
-        // Save Keys for return value
-        let pending_keys: HashSet<Key> = pending_data.keys().cloned().collect();
+        let pending_data = approved_write.settings;
 
-        // Apply changes to live
-        debug!("Writing pending keys to live");
-        self.set_keys(&pending_data, &Committed::Live)?;
+        if !pending_data.is_empty() {
+            // Save Keys for return value
+            pending_keys = pending_data.keys().cloned().collect();
+
+            // Apply changes to live
+            debug!("Writing pending keys to live");
+            self.set_keys(&pending_data, &Committed::Live)?;
+        }
 
         // Remove pending
         debug!("Removing old pending keys");