From f5acea634d64ee02b90171a48e9a8615741c5064 Mon Sep 17 00:00:00 2001 From: Jesper Jensen Date: Sun, 8 Feb 2026 11:05:45 +0100 Subject: Separate out update_image to add some tests --- src/updater.rs | 604 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 604 insertions(+) create mode 100644 src/updater.rs (limited to 'src/updater.rs') diff --git a/src/updater.rs b/src/updater.rs new file mode 100644 index 0000000..3f4fe51 --- /dev/null +++ b/src/updater.rs @@ -0,0 +1,604 @@ +use crate::version::{VersionPattern, CompareOutcome}; +use crate::docker::DockerRef; +use crate::db::Db; +use crate::registry::Registries; +use crate::metrics; +use chrono::{DateTime, Utc}; +use std::ops::Range; + +#[derive(Debug)] +pub struct FilePatch { + pub position: Range, + pub content: String, +} + +pub struct FileInput { + pub path: std::path::PathBuf, + pub content: String, + pub images: Vec>, +} + +pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, files: &[FileInput], outcomes: &mut Vec, String>>) { + for file in files { + let mut patches = vec![]; + let mut error: Option = None; + + let mut imgs: Vec = vec![]; + let mut image_ids: Vec = vec![]; + let mut last_checkeds: Vec> = vec![]; + + for image in &file.images { + let img = DockerRef::parse(&file.content, image); + let registry = img.registry.as_ref() + .map(|x| &file.content[x.clone()]) + .unwrap_or("registry.hub.docker.com"); + let image_name = &file.content[img.image.clone()]; + + let (image_id, last_checked) = match db.get_image(registry, image_name) { + Some(x) => x, + None => { + let epoch = DateTime::::UNIX_EPOCH; + let id = db.insert_image(registry, image_name, &epoch); + (id, epoch) + }, + }; + + imgs.push(img); + image_ids.push(image_id); + last_checkeds.push(last_checked); + } + + for i in 0..imgs.len() { + let img = &imgs[i]; + let registry = img.registry.as_ref() + .map(|x| &file.content[x.clone()]) + .unwrap_or("registry.hub.docker.com"); + let cache_max_age = reg.get_cache_ttl(registry); + let mut tag = img.tag.as_ref().map(|x| file.content[x.clone()].to_string()); + let image_name = &file.content[img.image.clone()]; + let image_id = image_ids[i]; + let last_checked = last_checkeds[i]; + + println!("Checking image {}", &file.content[file.images[i].clone()]); + metrics::get().images_checked.with_label_values(&[registry]).inc(); + + let cache_stale = *now - last_checked > cache_max_age; + + let cached_tags: Vec = db.get_tags_sorted(image_id); + + let tags_result: Result<(Vec, Option), String> = if cache_stale { + println!("Invalid in cache"); + match reg.get_tags(registry, image_name) { + Some(mut fetched_tags) => { + fetched_tags.sort(); + let mut existing = cached_tags.iter().peekable(); + for t in &fetched_tags { + while existing.peek().is_some_and(|e| *e < t) { + let gone = existing.next().unwrap(); + db.delete_tag(image_id, gone); + } + if existing.peek() == Some(&t) { + existing.next(); + } else { + db.insert_tag(image_id, t, now); + } + } + for gone in existing { + db.delete_tag(image_id, gone); + } + db.update_last_checked(image_id, now); + + Ok((fetched_tags, None)) + } + None => { + db.update_last_checked(image_id, now); + Err(format!("image not found: {}", image_name)) + } + } + } else { + println!("Valid in cache"); + let tag_for_digest = tag.as_deref().unwrap_or("latest"); + let digest = db.get_tag_digest(image_id, tag_for_digest); + + Ok((cached_tags, digest)) + }; + + let (tags, cached_digest) = match tags_result { + Ok(v) => v, + Err(msg) => { + error = Some(msg); + break; + } + }; + + if let Some(ref tag_str) = tag { + let mut current = VersionPattern::parse(&tag_str); + let mut new_tag = None; + + for candidate_str in &tags { + let candidate = VersionPattern::parse(&candidate_str); + match current.compare(&candidate) { + CompareOutcome::Higher => { + new_tag = Some(candidate_str.clone()); + current = candidate; + }, + CompareOutcome::Lower => {}, + CompareOutcome::Incompatible => {}, + CompareOutcome::Identical => {}, + } + } + + if let Some(new_tag) = new_tag { + tag = Some(new_tag.clone()); + patches.push(FilePatch { + position: img.tag.clone().unwrap(), + content: new_tag, + }); + metrics::get().images_updated.with_label_values(&[registry]).inc(); + } + } + + if let Some(ref digest) = img.digest { + let tag_for_digest = tag.as_deref().unwrap_or("latest"); + + let digest_result = match cached_digest { + Some(d) => Ok(d), + None => { + match reg.get_digest(registry, image_name, tag_for_digest) { + Some(fetched) => { + db.update_tag_digest(image_id, tag_for_digest, &fetched, now); + Ok(fetched) + } + None => Err(format!("digest not found for {}:{}", image_name, tag_for_digest)), + } + } + }; + + let digest_string = match digest_result { + Ok(d) => d, + Err(msg) => { + error = Some(msg); + break; + } + }; + + if file.content[digest.clone()] != digest_string { + patches.push(FilePatch { + position: img.digest.clone().unwrap(), + content: digest_string, + }); + metrics::get().images_updated.with_label_values(&[registry]).inc(); + } + } + } + + if let Some(msg) = error { + outcomes.push(Err(msg)); + } else { + patches.sort_by_key(|e| e.position.start); + outcomes.push(Ok(patches)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::StubDb; + use crate::registry::StubRegistry; + use chrono::TimeZone; + + #[test] + fn stale_cache_fetches_from_registry() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + let old = now - chrono::Duration::days(2); + let id = db.insert_image("registry.hub.docker.com", "nginx", &old); + db.insert_tag(id, "1.20", &old); + db.insert_tag(id, "1.21", &old); + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21".into(), + images: vec![7..17] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert_eq!(db.get_tags_sorted(id), vec!["1.21", "1.22"]); + let (_, last_checked) = db.get_image("registry.hub.docker.com", "nginx").unwrap(); + assert_eq!(last_checked, now); + } + + #[test] + fn fresh_cache_uses_cached_tags() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + let recent = now - chrono::Duration::minutes(30); + let id = db.insert_image("registry.hub.docker.com", "nginx", &recent); + db.insert_tag(id, "1.21", &recent); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21".into(), + images: vec![7..17] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert!(outcomes[0].as_ref().unwrap().is_empty()); + } + + #[test] + fn no_patch_when_at_highest() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21".into(), + images: vec![7..17] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert!(outcomes[0].as_ref().unwrap().is_empty()); + } + + #[test] + fn patch_to_higher_version() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21".into(), + images: vec![7..17] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert_eq!(outcomes[0].as_ref().unwrap()[0].content, "1.22"); + } + + #[test] + fn picks_highest_of_multiple() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + reg.add_tag("registry.hub.docker.com", "nginx", "1.25", "sha256:0000000000000000000000000000000000000000000000000000000000000003"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21".into(), + images: vec![7..17] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert_eq!(outcomes[0].as_ref().unwrap()[0].content, "1.25"); + } + + #[test] + fn no_tag_skips_comparison() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "latest", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx@sha256:0000000000000000000000000000000000000000000000000000000000000002".into(), + images: vec![7..84] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + let patches = outcomes[0].as_ref().unwrap(); + assert_eq!(patches.len(), 1); + assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + } + + #[test] + fn uses_cached_digest_when_fresh() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + let recent = now - chrono::Duration::minutes(30); + let id = db.insert_image("registry.hub.docker.com", "nginx", &recent); + db.insert_tag(id, "1.21", &recent); + db.update_tag_digest(id, "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000003", &recent); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), + images: vec![7..89] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + let patches = outcomes[0].as_ref().unwrap(); + assert_eq!(patches.len(), 1); + assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000003"); + } + + #[test] + fn fetches_digest_when_not_cached() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000004"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), + images: vec![7..89] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + let patches = outcomes[0].as_ref().unwrap(); + assert_eq!(patches.len(), 1); + assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000004"); + } + + #[test] + fn no_patch_when_digest_matches() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000002".into(), + images: vec![7..89] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert!(outcomes[0].as_ref().unwrap().is_empty()); + } + + #[test] + fn patch_when_digest_differs() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), + images: vec![7..89] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + let patches = outcomes[0].as_ref().unwrap(); + assert_eq!(patches.len(), 1); + assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + } + + #[test] + fn digest_uses_updated_tag() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), + images: vec![7..89] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + let patches = outcomes[0].as_ref().unwrap(); + assert_eq!(patches.len(), 2); + assert_eq!(patches[0].content, "1.22"); + assert_eq!(patches[1].content, "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + } + + #[test] + fn error_when_image_not_found() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21".into(), + images: vec![7..17] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert!(outcomes[0].is_err()); + assert!(outcomes[0].as_ref().unwrap_err().contains("image not found")); + let (_, last_checked) = db.get_image("registry.hub.docker.com", "nginx").unwrap(); + assert_eq!(last_checked, now); + } + + #[test] + fn error_when_digest_not_found() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.22@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), + images: vec![7..89] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert!(outcomes[0].is_err()); + } + + #[test] + fn multiple_images_produce_patches() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + reg.add_tag("registry.hub.docker.com", "redis", "6.0", "sha256:0000000000000000000000000000000000000000000000000000000000000003"); + reg.add_tag("registry.hub.docker.com", "redis", "6.2", "sha256:0000000000000000000000000000000000000000000000000000000000000004"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21\nimage: redis:6.0".into(), + images: vec![7..17, 25..34] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + let patches = outcomes[0].as_ref().unwrap(); + assert_eq!(patches.len(), 2); + } + + #[test] + fn patches_sorted_by_position() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + reg.add_tag("registry.hub.docker.com", "redis", "6.0", "sha256:0000000000000000000000000000000000000000000000000000000000000003"); + reg.add_tag("registry.hub.docker.com", "redis", "6.2", "sha256:0000000000000000000000000000000000000000000000000000000000000004"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21\nimage: redis:6.0".into(), + images: vec![7..17, 25..34] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + let patches = outcomes[0].as_ref().unwrap(); + assert!(patches[0].position.start < patches[1].position.start); + } + + #[test] + fn multiple_files_independent() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); + + let files = vec![ + FileInput { + path: "/a".into(), + content: "image: missing:1.0".into(), + images: vec![7..18] + }, + FileInput { + path: "/b".into(), + content: "image: nginx:1.21".into(), + images: vec![7..17] + }, + ]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert!(outcomes[0].is_err()); + assert!(outcomes[1].is_ok()); + assert_eq!(outcomes[1].as_ref().unwrap()[0].content, "1.22"); + } + + #[test] + fn stale_cache_ignores_cached_digest() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + let old = now - chrono::Duration::days(2); + let id = db.insert_image("registry.hub.docker.com", "nginx", &old); + db.insert_tag(id, "1.21", &old); + db.update_tag_digest(id, "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000003", &old); + + reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000005"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), + images: vec![7..89] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + let patches = outcomes[0].as_ref().unwrap(); + assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000005"); + } + + #[test] + fn error_in_first_image_skips_remaining() { + crate::metrics::init(); + let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let db = StubDb::default(); + let reg = StubRegistry::default(); + + reg.add_tag("registry.hub.docker.com", "redis", "6.0", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); + + let files = vec![FileInput { + path: "/test".into(), + content: "image: nginx:1.21\nimage: redis:6.0".into(), + images: vec![7..17, 25..34] + }]; + let mut outcomes = vec![]; + update_images(&now, &db, ®, &files, &mut outcomes); + + assert!(outcomes[0].is_err()); + assert!(db.get_image("registry.hub.docker.com", "redis").is_some()); + assert!(db.get_tags_sorted(2).is_empty()); + } +} -- cgit v1.2.3