cdh: move storage and secret crates to cdh modules

the storage and secret crate is used exclusively by cdh, so we don't
need to we don't need to maintain those as individual crates, helping
with naming conflicts and reducing build complexity.

the storage crate has a dep on the secret crate, so we have to cover
them in a single change.

Signed-off-by: Magnus Kulke <[email protected]>

.github/workflows/cdh_basic.yml

@@ -67,13 +67,13 @@ jobs:
 
       - name: Run cargo test
         run: |
-          sudo -E PATH=$PATH -s cargo test --features kbs,aliyun,sev,bin -p kms -p confidential-data-hub -p secret -p image
+          sudo -E PATH=$PATH -s cargo test --features kbs,aliyun,sev,bin -p kms -p confidential-data-hub -p image
 
       - name: Run cargo fmt check
         run: |
-          sudo -E PATH=$PATH -s cargo fmt -p kms -p confidential-data-hub -p secret -p image -- --check
+          sudo -E PATH=$PATH -s cargo fmt -p kms -p confidential-data-hub -p image -- --check
 
       - name: Run rust lint check
         run: |
           # We are getting error in generated code due to derive_partial_eq_without_eq check, so ignore it for now
-          sudo -E PATH=$PATH -s cargo clippy -p kms -p confidential-data-hub -p secret -p image -- -D warnings -A clippy::derive-partial-eq-without-eq 
+          sudo -E PATH=$PATH -s cargo clippy -p kms -p confidential-data-hub -p image -- -D warnings -A clippy::derive-partial-eq-without-eq 

Cargo.toml

@@ -13,8 +13,6 @@ members = [
     "confidential-data-hub/hub",
     "confidential-data-hub/kms",
     "confidential-data-hub/image",
-    "confidential-data-hub/secret",
-    "confidential-data-hub/storage",
     "image-rs",
     "ocicrypt-rs",
 ]

README.md

@@ -26,7 +26,7 @@ CoCo Keyprovider. Used to encrypt the container images.
 
 ## Tools
 
-[secret-cli](confidential-data-hub/secret)
+[secret-cli](confidential-data-hub/hub/src/bin/secret_cli.rs)
 Utility for sealing and unsealing sealed secrets
 
 [CDH Client](confidential-data-hub/hub/src/bin)

confidential-data-hub/docs/SEALED_SECRET.md

@@ -10,8 +10,11 @@ in conjunction with an attestation.
 The Confidential Data Hub provides an API for unsealing secrets inside
 of a confidential guest.
 
-There is also a [secret cli](../secret/src/bin) tool that can be used to generate sealed
-secrets.
+You can also use the secret cli tool to generate a sealed secret:
+
+```bash
+cargo run -p confidential-data-hub --bin secret
+```
 
 ## Kubernetes Secrets
 
@@ -173,7 +176,6 @@ Start with a sealed secret such as
 	}
 }
 ```
-You can use the [secret cli](../secret/src/bin) tool to generate a sealed secret.
 
 Encode the payload in BASE64URL
 ```

confidential-data-hub/docs/SECURE_STORAGE.md

@@ -17,7 +17,7 @@ We reuse [direct block device assigned volume feature](https://github.com/kata-c
 
 [Aliyun OSS](https://www.alibabacloud.com/product/object-storage-service) is an object storage service provided by Alibaba Cloud (Aliyun).
 
-The [plugin](../storage/src/volume_type/aliyun) provides two different modes for secure mount.
+The [plugin](../hub/src/storage/volume_type/aliyun) provides two different modes for secure mount.
 
 Confidential Data Hub's `secure_mount()` [API](../hub/protos/api.proto) will help to instrument this.
 
@@ -65,7 +65,7 @@ For more details, please refer to [the guide](use-cases/secure-mount-with-aliyun
 
 ### Block Device
 
-The [plugin](../storage/src/volume_type/blockdevice) provides ways to encrypt a block device and mount it to a specific mount point. Currently only support LUKS in [cryptsetup](https://gitlab.com/cryptsetup/cryptsetup/) for block device encryption.
+The [plugin](../hub/src/storage/volume_type/blockdevice) provides ways to encrypt a block device and mount it to a specific mount point. Currently only support LUKS in [cryptsetup](https://gitlab.com/cryptsetup/cryptsetup/) for block device encryption.
 
 #### LUKS Encryption
 

confidential-data-hub/docs/use-cases/secure-mount-with-block-device.md

@@ -30,7 +30,7 @@ Follow the instructions in the [CDH README](../../README.md#confidential-data-hu
 
 2. Install `luks-encrypt-storage`
 
-Install [luks-encrypt-storage](../../storage/scripts/luks-encrypt-storage) into `/usr/local/bin`
+Install [luks-encrypt-storage](../../hub/src/storage/scripts/luks-encrypt-storage) into `/usr/local/bin`
 
 3. Run CDH
 ```shell
@@ -74,4 +74,4 @@ $ lsblk |grep "encrypted_disk"
 # Expected output:
 └─encrypted_disk_OEyEj_dif 253:1    0 968.6M  0 crypt
   └─encrypted_disk_OEyEj   253:2    0 968.6M  0 crypt /mnt/test-path
-```
+```

confidential-data-hub/hub/Cargo.toml

@@ -26,6 +26,11 @@ required-features = ["bin", "grpc"]
 name = "cdh-oneshot"
 required-features = ["bin"]
 
+[[bin]]
+name = "secret"
+path = "src/bin/secret_cli.rs"
+required-features = ["cli"]
+
 [dependencies]
 anyhow = { workspace = true, optional = true }
 async-trait.workspace = true
@@ -34,6 +39,7 @@ base64.workspace = true
 cfg-if = { workspace = true, optional = true }
 clap = { workspace = true, features = [ "derive" ], optional = true }
 config = { workspace = true, optional = true }
+crypto.path = "../../attestation-agent/deps/crypto"
 env_logger = { workspace = true, optional = true }
 image = { path = "../image", default-features = false }
 image-rs = { path = "../../image-rs", default-features = false, features = ["kata-cc-rustls-tls"] }
@@ -42,42 +48,50 @@ lazy_static.workspace = true
 log.workspace = true
 prost = { workspace = true, optional = true }
 protobuf = { workspace = true, optional = true }
-secret.path = "../secret"
-storage.path = "../storage"
+rand.workspace = true
 serde = { workspace = true, optional = true }
 serde_json.workspace = true
+strum = { workspace = true, features = ["derive"] }
+tempfile = { workspace = true, optional = true }
 thiserror.workspace = true
-tokio = { workspace = true, features = [ "rt-multi-thread", "macros", "sync" ] }
+tokio = { workspace = true, features = [ "rt-multi-thread", "macros", "io-util", "sync" ] }
 tonic = { workspace = true, optional = true }
 ttrpc = { workspace = true, features = ["async"], optional = true }
+zeroize.workspace = true
 
 [build-dependencies]
+anyhow.workspace = true
 tonic-build = { workspace = true, optional = true }
 ttrpc-codegen = { workspace = true, optional = true }
 
 [dev-dependencies]
 assert_cmd.workspace = true
+assert-json-diff.workspace = true
 nix.workspace = true
 rstest.workspace = true
 serial_test.workspace = true
 tempfile.workspace = true
+tokio = { workspace = true, features = ["rt", "macros" ] }
 
 [features]
-default = ["kbs", "bin", "ttrpc", "grpc"]
+default = ["aliyun", "kbs", "bin", "ttrpc", "grpc", "cli"]
 
 # support aliyun stacks (KMS, ..)
-aliyun = ["image/aliyun", "secret/aliyun"]
+aliyun = ["image/aliyun", "tempfile", "tokio/fs", "tokio/process", "tokio/time"]
 
 # support coco-KBS to provide confidential resources
-kbs = ["image/kbs", "kms/kbs", "secret/kbs"]
+kbs = ["image/kbs", "kms/kbs"]
 
 # support sev to provide confidential resources
-sev = ["image/sev", "kms/sev", "secret/sev"]
+sev = ["image/sev", "kms/sev"]
 
 # support eHSM stacks (KMS, ...)
-ehsm = ["image/ehsm", "secret/ehsm"]
+ehsm = ["image/ehsm"]
 
 # Binary RPC type
 bin = [ "anyhow", "attestation-agent", "cfg-if", "clap", "config", "env_logger", "serde" ]
 ttrpc = ["dep:ttrpc", "protobuf", "ttrpc-codegen", "tokio/signal"]
 grpc = ["prost", "tonic", "tonic-build", "tokio/signal"]
+
+# for secret_cli
+cli = ["clap/derive", "tokio/rt-multi-thread", "tokio/sync", "tokio/macros"]

confidential-data-hub/hub/src/api.rs

@@ -5,8 +5,8 @@
 
 use async_trait::async_trait;
 
+use crate::storage::volume_type::Storage;
 use crate::Result;
-use storage::volume_type::Storage;
 
 /// The APIs of the DataHub. See
 /// <https://github.com/confidential-containers/documentation/issues/131> for

confidential-data-hub/hub/src/bin/cdh-oneshot.rs

@@ -9,9 +9,8 @@
 
 use base64::{engine::general_purpose::STANDARD, Engine};
 use clap::{Args, Parser, Subcommand};
-use confidential_data_hub::{hub::Hub, CdhConfig, DataHub};
+use confidential_data_hub::{hub::Hub, storage::volume_type::Storage, CdhConfig, DataHub};
 use log::warn;
-use storage::volume_type::Storage;
 
 #[derive(Parser)]
 #[command(name = "cdh_oneshot")]

confidential-data-hub/hub/src/bin/grpc-cdh-tool.rs

@@ -16,7 +16,7 @@ use api::{
 };
 use base64::{engine::general_purpose::STANDARD, Engine};
 use clap::{Args, Parser, Subcommand};
-use storage::volume_type::Storage;
+use confidential_data_hub::storage::volume_type::Storage;
 
 mod api {
     tonic::include_proto!("api");

confidential-data-hub/hub/src/bin/grpc_server/mod.rs

@@ -5,10 +5,12 @@
 
 use anyhow::*;
 
-use confidential_data_hub::{hub::Hub, DataHub};
+use confidential_data_hub::{
+    storage::volume_type::Storage,
+    {hub::Hub, DataHub},
+};
 use log::{debug, error};
 use std::{error::Error as _, net::SocketAddr, sync::Arc};
-use storage::volume_type::Storage;
 use tonic::{transport::Server, Request, Response, Status};
 
 use crate::{

confidential-data-hub/secret/src/bin/secret_cli.rs → confidential-data-hub/hub/src/bin/secret_cli.rs

@@ -7,15 +7,17 @@ use std::{env, path::Path};
 
 use base64::{engine::general_purpose::STANDARD, Engine};
 use clap::{command, Args, Parser, Subcommand};
+use confidential_data_hub::secret::{
+    layout::{envelope::EnvelopeSecret, vault::VaultSecret},
+    Secret, SecretContent, VERSION,
+};
 use crypto::WrapType;
 #[cfg(feature = "aliyun")]
 use kms::plugins::aliyun::AliyunKmsClient;
 #[cfg(feature = "ehsm")]
 use kms::plugins::ehsm::EhsmKmsClient;
 use kms::{Encrypter, ProviderSettings};
 use rand::Rng;
-use secret::secret::layout::{envelope::EnvelopeSecret, vault::VaultSecret};
-use secret::secret::{Secret, SecretContent, VERSION};
 #[cfg(feature = "ehsm")]
 use serde_json::Value;
 use tokio::{fs, io::AsyncWriteExt};

confidential-data-hub/hub/src/bin/ttrpc-cdh-tool.rs

@@ -9,6 +9,7 @@
 
 use base64::{engine::general_purpose::STANDARD, Engine};
 use clap::{Args, Parser, Subcommand};
+use confidential_data_hub::storage::volume_type::Storage;
 use protos::{
     api::*,
     api_ttrpc::{
@@ -18,7 +19,6 @@ use protos::{
     keyprovider::*,
     keyprovider_ttrpc::KeyProviderServiceClient,
 };
-use storage::volume_type::Storage;
 use ttrpc::context;
 
 mod protos;

confidential-data-hub/hub/src/bin/ttrpc_server/mod.rs

@@ -7,9 +7,11 @@ use std::error::Error as _;
 
 use anyhow::Result;
 use async_trait::async_trait;
-use confidential_data_hub::{hub::Hub, CdhConfig, DataHub};
+use confidential_data_hub::{
+    storage::volume_type::Storage,
+    {hub::Hub, CdhConfig, DataHub},
+};
 use log::{debug, error};
-use storage::volume_type::Storage;
 use ttrpc::{asynchronous::TtrpcContext, Code, Error, Status};
 
 use crate::{

confidential-data-hub/hub/src/error.rs

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use crate::secret;
+use crate::storage;
 use thiserror::Error;
 
 pub type Result<T> = std::result::Result<T, Error>;

confidential-data-hub/hub/src/hub.rs

@@ -9,9 +9,10 @@ use async_trait::async_trait;
 use image_rs::{builder::ClientBuilder, config::ImageConfig, image::ImageClient};
 use kms::{Annotations, ProviderSettings};
 use log::{debug, info};
-use storage::volume_type::Storage;
 use tokio::sync::{Mutex, OnceCell};
 
+use crate::secret;
+use crate::storage::volume_type::Storage;
 use crate::{CdhConfig, DataHub, Error, Result};
 
 pub struct Hub {

confidential-data-hub/hub/src/lib.rs

@@ -15,3 +15,6 @@ pub mod auth;
 
 pub mod config;
 pub use config::*;
+
+pub mod secret;
+pub mod storage;