diff --git a/Cargo.lock b/Cargo.lock index 28fbc6f..30ea4ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,6 +16,7 @@ dependencies = [ "log", "notify", "notify-debouncer-mini", + "parse-size", "regex", "reqwest", "serde", @@ -841,6 +842,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "parse-size" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "944553dd59c802559559161f9816429058b869003836120e262e8caec061b7ae" + [[package]] name = "percent-encoding" version = "2.3.1" diff --git a/Cargo.toml b/Cargo.toml index a2386bf..948c440 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,4 +23,5 @@ cron = "0.12.0" dyn-clone = "1.0.16" serde_yaml = "0.9.29" notify = { version = "6.1.1", default-features = false, features = ["serde", "macos_kqueue"] } -notify-debouncer-mini = { version = "0.4.1", default-features = false, features = ["serde"] } \ No newline at end of file +notify-debouncer-mini = { version = "0.4.1", default-features = false, features = ["serde"] } +parse-size = "1.0.0" diff --git a/README.md b/README.md index f450a5c..a28af34 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ to open a new issue or contribute to an open issue! - [x] Add tests to rule parsing and rule affections - [ ] Add ping to registry container in instance creation - [x] Add policy to match tags by pattern -- [ ] Add policy to match tags by size +- [x] Add policy to match tags by size ## Credits diff --git a/docs/policies.md b/docs/policies.md index 3c82346..7fde943 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -99,6 +99,25 @@ Any valid regex (supported by this crate) can be used. tag.pattern: .+-(beta|alpha) ``` +### Size policy +> Affection type: `Target` +> +> Identifier: `size` +> +> Default: `None` + +The size policy matches all tags which exceed the provided blob size. For size parsing the [parse-size](https://crates.io/crates/parse-size) crate is used. +Any valid size (supported by this crate) can be used + +>[!NOTE] +> The library uses `MiB`, `GiB` etc. which are the binary representations instead of the usual decimal representations of the size. Therefore, `1 MiB` is `1_048_576` bytes +> instead of `1_000_000` bytes as one might expect + +```yaml +# Would match all tags whose total blob size exceed 256 MiB +size: 256 MiB +``` + ## Repository policies Repository policies are used to determine for which images a rule should be applied diff --git a/src/api/layer.rs b/src/api/layer.rs index c9bd994..2f19db1 100644 --- a/src/api/layer.rs +++ b/src/api/layer.rs @@ -5,5 +5,5 @@ pub struct Layer { #[serde(rename = "mediaType")] pub media_type: String, pub digest: String, - pub size: u32, + pub size: u64, } diff --git a/src/api/repository.rs b/src/api/repository.rs index eba5368..5781530 100644 --- a/src/api/repository.rs +++ b/src/api/repository.rs @@ -144,12 +144,12 @@ impl Repository { for tag in raw { match self.get_manifest(&tag).await? { ManifestResponse::Manifest(manifest) => { - let size: u32 = manifest.layers.iter().map(|l| l.size).sum(); + let size: u64 = manifest.layers.iter().map(|l| l.size).sum(); let config = manifest.get_config().await?; tags.push(Tag::new(tag, manifest.digest, config.created, size)); }, ManifestResponse::ManifestList(list) => { - let size: u32 = list.manifests.iter().map(|m| m.size).sum(); + let size: u64 = list.manifests.iter().map(|m| m.size).sum(); let layer = list.manifests.get(0).ok_or(ApiError::EmptyManifestList)?; let manifest = list.get_manifest(layer.digest.clone()).await?; let config = manifest.get_config().await?; diff --git a/src/api/tag.rs b/src/api/tag.rs index a2e717b..f14c605 100644 --- a/src/api/tag.rs +++ b/src/api/tag.rs @@ -5,11 +5,11 @@ pub struct Tag { pub name: String, pub digest: String, pub created: DateTime, - pub size: u32 + pub size: u64 } impl Tag { - pub fn new(name: String, digest: String, created: DateTime, size: u32) -> Self { + pub fn new(name: String, digest: String, created: DateTime, size: u64) -> Self { Self { name, digest, created, size } } } \ No newline at end of file diff --git a/src/instance.rs b/src/instance.rs index cd8a908..b845609 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -15,6 +15,7 @@ use crate::policies::age_max::{AGE_MAX_LABEL, AgeMaxPolicy}; use crate::policies::age_min::{AGE_MIN_LABEL, AgeMinPolicy}; use crate::policies::image_pattern::{IMAGE_PATTERN_LABEL, ImagePatternPolicy}; use crate::policies::revision::{REVISION_LABEL, RevisionPolicy}; +use crate::policies::size::{SIZE_LABEL, SizePolicy}; use crate::policies::tag_pattern::{TAG_PATTERN_LABEL, TagPatternPolicy}; use crate::rule::{parse_rule, parse_schedule, Rule}; @@ -161,6 +162,7 @@ impl Instance { default_rule.tag_policies.insert(AGE_MAX_LABEL, Box::::default()); default_rule.tag_policies.insert(AGE_MIN_LABEL, Box::::default()); default_rule.tag_policies.insert(REVISION_LABEL, Box::::default()); + default_rule.tag_policies.insert(SIZE_LABEL, Box::::default()); // parse default rules labels.iter() diff --git a/src/policies/mod.rs b/src/policies/mod.rs index e6f907c..775a972 100644 --- a/src/policies/mod.rs +++ b/src/policies/mod.rs @@ -11,6 +11,7 @@ pub mod age_min; pub mod image_pattern; pub mod revision; pub mod tag_pattern; +pub mod size; pub type PolicyMap = HashMap<&'static str, Box>>; @@ -55,4 +56,8 @@ pub fn parse_duration(duration_str: String) -> Option { Ok(duration_str) => Duration::from_std(duration_str.into()).ok(), Err(_) => None } +} + +pub fn parse_size(size_str: &str) -> Option { + parse_size::parse_size(size_str).ok() } \ No newline at end of file diff --git a/src/policies/size.rs b/src/policies/size.rs new file mode 100644 index 0000000..92380a2 --- /dev/null +++ b/src/policies/size.rs @@ -0,0 +1,122 @@ +use log::info; +use crate::api::tag::Tag; +use crate::policies::{AffectionType, parse_size, Policy}; + +pub const SIZE_LABEL: &str = "size"; + +/// Policy to match all tags which exceed a given blob size +/// # Example +/// ``` +/// let policy = SizePolicy::new(String::from("0.2 GiB")); +/// +/// // returns all tags which are bigger than 0.2 GiB +/// let affected = policy.affects(&tags); + +#[derive(Debug, Clone, Default)] +pub struct SizePolicy { + size: Option +} + +impl SizePolicy { + pub fn new(value: &str) -> Self { + if value.is_empty() { + Self { size: None } + } else { + let size = parse_size(value); + if size.is_none() { + info!("Received invalid size '{value}'") + } + Self { size } + } + } +} + +impl Policy for SizePolicy { + fn affects(&self, tags: Vec) -> Vec { + if let Some(size) = self.size { + tags.into_iter().filter(|tag| tag.size >= size).collect() + } else { + vec![] + } + } + + fn affection_type(&self) -> AffectionType { + AffectionType::Target + } + + fn id(&self) -> &'static str { + SIZE_LABEL + } + + fn enabled(&self) -> bool { + self.size.is_some() + } +} + +#[cfg(test)] +mod test { + use chrono::Duration; + use crate::api::tag::Tag; + use crate::policies::Policy; + use crate::policies::size::SizePolicy; + use crate::test::get_tags; + + fn get_current_tags() -> Vec { + get_tags(vec![ + ("first", Duration::hours(-5), 1_200_000), + ("second", Duration::minutes(-5), 1_000), + ("third", Duration::minutes(-30), 100_000_000), + ("fourth", Duration::minutes(-10), 100_000), + ("fifth", Duration::seconds(-15), 1_300_000), + ("sixth", Duration::minutes(-50), 1_100_000) + ]) + } + + #[test] + pub fn test_matching() { + let tags = get_current_tags(); + let policy = SizePolicy::new("1 MiB"); + assert!(policy.size.is_some()); + assert_eq!(policy.affects(tags.clone()), vec![tags[0].clone(), tags[2].clone(), tags[4].clone(), tags[5].clone()]) + } + + #[test] + pub fn test_empty() { + let tags = get_current_tags(); + let policy = SizePolicy::new(""); + assert!(policy.size.is_none()); + assert_eq!(policy.affects(tags), vec![]) + } + + #[test] + pub fn test_default() { + let tags = get_current_tags(); + let policy = SizePolicy::default(); + assert!(policy.size.is_none()); + assert_eq!(policy.affects(tags), vec![]) + } + + #[test] + pub fn test_invalid_size() { + let tags = get_current_tags(); + let policy = SizePolicy::new("120 asdf"); + assert!(policy.size.is_none()); + assert_eq!(policy.affects(tags), vec![]) + } + + #[test] + pub fn test_negative_size() { + let tags = get_current_tags(); + let policy = SizePolicy::new("-1 MiB"); + assert!(policy.size.is_none()); + assert_eq!(policy.affects(tags), vec![]) + } + + #[test] + pub fn test_without_unit() { + let tags = get_current_tags(); + let policy = SizePolicy::new("1_048_576"); + assert!(policy.size.is_some()); + assert_eq!(policy.affects(tags.clone()), vec![tags[0].clone(), tags[2].clone(), tags[4].clone(), tags[5].clone()]) + } +} \ No newline at end of file diff --git a/src/rule.rs b/src/rule.rs index 3000bd4..8a0f505 100644 --- a/src/rule.rs +++ b/src/rule.rs @@ -9,6 +9,7 @@ use crate::policies::age_min::{AGE_MIN_LABEL, AgeMinPolicy}; use crate::policies::age_max::{AGE_MAX_LABEL, AgeMaxPolicy}; use crate::policies::image_pattern::{IMAGE_PATTERN_LABEL, ImagePatternPolicy}; use crate::policies::revision::{REVISION_LABEL, RevisionPolicy}; +use crate::policies::size::{SIZE_LABEL, SizePolicy}; use crate::policies::tag_pattern::{TAG_PATTERN_LABEL, TagPatternPolicy}; #[derive(Debug)] @@ -97,6 +98,9 @@ pub fn parse_rule(name: String, policies: Vec<(String, &str)>) -> Option { REVISION_LABEL => { rule.tag_policies.insert(REVISION_LABEL, Box::new(RevisionPolicy::new(value.to_string()))); }, + SIZE_LABEL => { + rule.tag_policies.insert(SIZE_LABEL, Box::new(SizePolicy::new(value))); + } other => { warn!("Found unknown policy '{other}' for rule '{name}'. Ignoring policy") } @@ -134,6 +138,7 @@ mod test { use crate::policies::age_min::AGE_MIN_LABEL; use crate::policies::image_pattern::IMAGE_PATTERN_LABEL; use crate::policies::revision::REVISION_LABEL; + use crate::policies::size::SIZE_LABEL; use crate::policies::tag_pattern::TAG_PATTERN_LABEL; use crate::rule::{parse_rule, parse_schedule}; use crate::test::{get_repositories, get_tags, get_tags_by_name}; @@ -230,19 +235,21 @@ mod test { ("image.pattern", "test-.+"), ("tag.pattern", "test-.+"), ("test", "10s"), - ("revisions", "10") + ("revisions", "10"), + ("size", "100 MiB") ]); let rule = parse_rule(String::from("test-rule"), labels); assert!(rule.is_some()); let parsed = rule.unwrap(); assert_eq!(parsed.name, String::from("test-rule")); assert_eq!(parsed.schedule, String::from("* * * * 5 *")); - assert_eq!(parsed.tag_policies.len(), 4); + assert_eq!(parsed.tag_policies.len(), 5); assert_eq!(parsed.repository_policies.len(), 1); assert!(parsed.tag_policies.get(AGE_MAX_LABEL).is_some()); assert!(parsed.tag_policies.get(AGE_MIN_LABEL).is_some()); assert!(parsed.tag_policies.get(REVISION_LABEL).is_some()); assert!(parsed.tag_policies.get(TAG_PATTERN_LABEL).is_some()); + assert!(parsed.tag_policies.get(SIZE_LABEL).is_some()); assert!(parsed.repository_policies.get(IMAGE_PATTERN_LABEL).is_some()) } @@ -327,7 +334,9 @@ mod test { let parsed = rule.unwrap(); let repositories = get_repositories(vec!["test", "test-asdf", "not matching", "test-match"]); - assert_eq!(parsed.affected_repositories(repositories.clone()), vec![repositories[1].clone(), repositories[3].clone()]) + let mut affected = parsed.affected_repositories(repositories.clone()); + affected.sort_by(|x, y| x.name.cmp(&y.name)); + assert_eq!(affected, vec![repositories[1].clone(), repositories[3].clone()]) } #[test] @@ -401,17 +410,18 @@ mod test { ("schedule", "* * * * 5 *"), ("image.pattern", "test-.+"), ("tag.pattern", ".*th"), - ("test", "10s") + ("test", "10s"), + ("size", "1 MiB") ]); let rule = parse_rule(String::from("test-rule"), labels).unwrap(); let tags = get_tags(vec![ - ("first", Duration::hours(-5), 1_000_000), - ("second", Duration::minutes(-5), 1_000_000), - ("third", Duration::minutes(-30), 1_000_000), - ("fourth", Duration::minutes(-10), 1_000_000), + ("first", Duration::hours(-5), 1_000), + ("second", Duration::minutes(-5), 1_200_000), + ("third", Duration::minutes(-30), 1_400_000), + ("fourth", Duration::minutes(-10), 100_000_000), ("fifth", Duration::seconds(-15), 1_000_000), - ("sixth", Duration::minutes(-50), 1_000_000) + ("sixth", Duration::minutes(-50), 1_000) ]); let repositories = get_repositories(vec!["test-asdf", "test-", "test-test"]); @@ -421,6 +431,6 @@ mod test { let mut affected = rule.affected_tags(tags.clone()); affected.sort_by(|t1, t2| t1.created.cmp(&t2.created).reverse()); - assert_eq!(affected, vec![tags[3].clone(), tags[2].clone(), tags[5].clone(), tags[0].clone()]); + assert_eq!(affected, vec![tags[1].clone(), tags[3].clone(), tags[2].clone(), tags[5].clone(), tags[0].clone()]); } } \ No newline at end of file diff --git a/src/test/mod.rs b/src/test/mod.rs index 3e089c9..5b64324 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -18,7 +18,7 @@ pub fn get_repositories(names: Vec>) -> Vec { repositories } -pub fn get_tags(raw: Vec<(impl Into, Duration, u32)>) -> Vec { +pub fn get_tags(raw: Vec<(impl Into, Duration, u64)>) -> Vec { let mut tags = vec![]; let now = chrono::offset::Utc::now(); for (name, offset, size) in raw { @@ -27,6 +27,6 @@ pub fn get_tags(raw: Vec<(impl Into, Duration, u32)>) -> Vec { tags } -pub fn get_tags_by_name(raw: Vec>, duration: Duration, size: u32) -> Vec { - get_tags(raw.into_iter().map(|x| (x, duration.clone(), size)).collect()) +pub fn get_tags_by_name(raw: Vec>, duration: Duration, size: u64) -> Vec { + get_tags(raw.into_iter().map(|x| (x, duration, size)).collect()) } \ No newline at end of file