use crate::version::{VersionPattern, CompareOutcome}; use crate::docker::DockerRef; use crate::db::Db; use crate::registry::Registries; use crate::metrics; use chrono::{DateTime, Utc}; use std::ops::Range; #[derive(Debug)] pub struct FilePatch { pub position: Range, pub content: String, } pub struct FileInput { pub path: std::path::PathBuf, pub content: String, pub images: Vec>, } pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, files: &[FileInput], outcomes: &mut Vec, String>>) { let mut file_indices: Vec = vec![]; let mut image_ranges: Vec> = vec![]; let mut imgs: Vec = vec![]; let mut image_ids: Vec = vec![]; let mut expires_ats: Vec> = vec![]; for (file_idx, file) in files.iter().enumerate() { for image in &file.images { let img = DockerRef::parse(&file.content, image); let registry = img.registry.as_ref() .map(|x| &file.content[x.clone()]) .unwrap_or("registry.hub.docker.com"); let image_name = &file.content[img.image.clone()]; let (image_id, expires_at) = match db.get_image(registry, image_name) { Some(x) => x, None => { let epoch = DateTime::::UNIX_EPOCH; let id = db.insert_image(registry, image_name, &epoch); (id, epoch) }, }; file_indices.push(file_idx); image_ranges.push(image.clone()); imgs.push(img); image_ids.push(image_id); expires_ats.push(expires_at); } } let mut file_patches: Vec> = Vec::with_capacity(files.len()); let mut file_errors: Vec> = Vec::with_capacity(files.len()); for _ in 0..files.len() { file_patches.push(vec![]); file_errors.push(None); } for i in 0..imgs.len() { let file_idx = file_indices[i]; let file = &files[file_idx]; if file_errors[file_idx].is_some() { continue; } let img = &imgs[i]; let registry = img.registry.as_ref() .map(|x| &file.content[x.clone()]) .unwrap_or("registry.hub.docker.com"); let cache_max_age = reg.get_cache_ttl(registry); let mut tag = img.tag.as_ref().map(|x| file.content[x.clone()].to_string()); let image_name = &file.content[img.image.clone()]; let image_id = image_ids[i]; let expires_at = expires_ats[i]; println!("Checking image {}", &file.content[image_ranges[i].clone()]); metrics::get().images_checked.with_label_values(&[registry]).inc(); let cache_stale = *now > expires_at; let cached_tags: Vec = db.get_tags_sorted(image_id); let tags_result: Result<(Vec, Option), String> = if cache_stale { println!("Invalid in cache"); match reg.get_tags(registry, image_name) { Some(mut fetched_tags) => { fetched_tags.sort(); let mut existing = cached_tags.iter().peekable(); for t in &fetched_tags { while existing.peek().is_some_and(|e| *e < t) { let gone = existing.next().unwrap(); db.delete_tag(image_id, gone); } if existing.peek() == Some(&t) { existing.next(); } else { db.insert_tag(image_id, t, now); } } for gone in existing { db.delete_tag(image_id, gone); } let new_expires_at = *now + cache_max_age; db.set_expires_at(image_id, &new_expires_at); // @CLEANUP: Propagate the update to other instances. We should really avoid // this situation in the first place for j in (i + 1)..image_ids.len() { if image_ids[j] == image_id { expires_ats[j] = new_expires_at; } } Ok((fetched_tags, None)) } None => { let new_expires_at = *now + cache_max_age; db.set_expires_at(image_id, &new_expires_at); // @CLEANUP: Propagate the update to other instances. We should really avoid // this situation in the first place for j in (i + 1)..image_ids.len() { if image_ids[j] == image_id { expires_ats[j] = new_expires_at; } } Err(format!("image not found: {}", image_name)) } } } else { println!("Valid in cache"); let tag_for_digest = tag.as_deref().unwrap_or("latest"); let digest = db.get_tag_digest(image_id, tag_for_digest); Ok((cached_tags, digest)) }; let (tags, cached_digest) = match tags_result { Ok(v) => v, Err(msg) => { file_errors[file_idx] = Some(msg); continue; } }; if let Some(ref tag_str) = tag { let mut current = VersionPattern::parse(&tag_str); let mut new_tag = None; for candidate_str in &tags { let candidate = VersionPattern::parse(&candidate_str); match current.compare(&candidate) { CompareOutcome::Higher => { new_tag = Some(candidate_str.clone()); current = candidate; }, CompareOutcome::Lower => {}, CompareOutcome::Incompatible => {}, CompareOutcome::Identical => {}, } } if let Some(new_tag) = new_tag { tag = Some(new_tag.clone()); file_patches[file_idx].push(FilePatch { position: img.tag.clone().unwrap(), content: new_tag, }); metrics::get().images_updated.with_label_values(&[registry]).inc(); } } if let Some(ref digest) = img.digest { let tag_for_digest = tag.as_deref().unwrap_or("latest"); let digest_result = match cached_digest { Some(d) => Ok(d), None => { match reg.get_digest(registry, image_name, tag_for_digest) { Some(fetched) => { db.update_tag_digest(image_id, tag_for_digest, &fetched, now); Ok(fetched) } None => Err(format!("digest not found for {}:{}", image_name, tag_for_digest)), } } }; let digest_string = match digest_result { Ok(d) => d, Err(msg) => { file_errors[file_idx] = Some(msg); continue; } }; if file.content[digest.clone()] != digest_string { file_patches[file_idx].push(FilePatch { position: img.digest.clone().unwrap(), content: digest_string, }); metrics::get().images_updated.with_label_values(&[registry]).inc(); } } } for file_idx in 0..files.len() { if let Some(msg) = file_errors[file_idx].take() { outcomes.push(Err(msg)); } else { let mut patches = std::mem::take(&mut file_patches[file_idx]); patches.sort_by_key(|e| e.position.start); outcomes.push(Ok(patches)); } } } #[cfg(test)] mod tests { use super::*; use crate::db::StubDb; use crate::registry::StubRegistry; use chrono::TimeZone; #[test] fn stale_cache_fetches_from_registry() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); let past_expiry = now - chrono::Duration::days(1); let id = db.insert_image("registry.hub.docker.com", "nginx", &past_expiry); db.insert_tag(id, "1.20", &past_expiry); db.insert_tag(id, "1.21", &past_expiry); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert_eq!(db.get_tags_sorted(id), vec!["1.21", "1.22"]); let (_, expires_at) = db.get_image("registry.hub.docker.com", "nginx").unwrap(); assert!(expires_at > now); } #[test] fn fresh_cache_uses_cached_tags() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); let future_expiry = now + chrono::Duration::minutes(30); let id = db.insert_image("registry.hub.docker.com", "nginx", &future_expiry); db.insert_tag(id, "1.21", &now); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert!(outcomes[0].as_ref().unwrap().is_empty()); } #[test] fn no_patch_when_at_highest() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert!(outcomes[0].as_ref().unwrap().is_empty()); } #[test] fn patch_to_higher_version() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert_eq!(outcomes[0].as_ref().unwrap()[0].content, "1.22"); } #[test] fn picks_highest_of_multiple() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); reg.add_tag("registry.hub.docker.com", "nginx", "1.25", "sha256:0000000000000000000000000000000000000000000000000000000000000003"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert_eq!(outcomes[0].as_ref().unwrap()[0].content, "1.25"); } #[test] fn no_tag_skips_comparison() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "latest", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx@sha256:0000000000000000000000000000000000000000000000000000000000000002".into(), images: vec![7..84] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); let patches = outcomes[0].as_ref().unwrap(); assert_eq!(patches.len(), 1); assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000001"); } #[test] fn uses_cached_digest_when_fresh() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); let future_expiry = now + chrono::Duration::minutes(30); let id = db.insert_image("registry.hub.docker.com", "nginx", &future_expiry); db.insert_tag(id, "1.21", &now); db.update_tag_digest(id, "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000003", &now); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), images: vec![7..89] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); let patches = outcomes[0].as_ref().unwrap(); assert_eq!(patches.len(), 1); assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000003"); } #[test] fn fetches_digest_when_not_cached() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000004"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), images: vec![7..89] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); let patches = outcomes[0].as_ref().unwrap(); assert_eq!(patches.len(), 1); assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000004"); } #[test] fn no_patch_when_digest_matches() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000002".into(), images: vec![7..89] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert!(outcomes[0].as_ref().unwrap().is_empty()); } #[test] fn patch_when_digest_differs() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), images: vec![7..89] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); let patches = outcomes[0].as_ref().unwrap(); assert_eq!(patches.len(), 1); assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000002"); } #[test] fn digest_uses_updated_tag() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), images: vec![7..89] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); let patches = outcomes[0].as_ref().unwrap(); assert_eq!(patches.len(), 2); assert_eq!(patches[0].content, "1.22"); assert_eq!(patches[1].content, "sha256:0000000000000000000000000000000000000000000000000000000000000002"); } #[test] fn error_when_image_not_found() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert!(outcomes[0].is_err()); assert!(outcomes[0].as_ref().unwrap_err().contains("image not found")); let (_, expires_at) = db.get_image("registry.hub.docker.com", "nginx").unwrap(); assert!(expires_at > now); } #[test] fn error_when_digest_not_found() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.22@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), images: vec![7..89] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert!(outcomes[0].is_err()); } #[test] fn multiple_images_produce_patches() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); reg.add_tag("registry.hub.docker.com", "redis", "6.0", "sha256:0000000000000000000000000000000000000000000000000000000000000003"); reg.add_tag("registry.hub.docker.com", "redis", "6.2", "sha256:0000000000000000000000000000000000000000000000000000000000000004"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21\nimage: redis:6.0".into(), images: vec![7..17, 25..34] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); let patches = outcomes[0].as_ref().unwrap(); assert_eq!(patches.len(), 2); } #[test] fn patches_sorted_by_position() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); reg.add_tag("registry.hub.docker.com", "redis", "6.0", "sha256:0000000000000000000000000000000000000000000000000000000000000003"); reg.add_tag("registry.hub.docker.com", "redis", "6.2", "sha256:0000000000000000000000000000000000000000000000000000000000000004"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21\nimage: redis:6.0".into(), images: vec![7..17, 25..34] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); let patches = outcomes[0].as_ref().unwrap(); assert!(patches[0].position.start < patches[1].position.start); } #[test] fn multiple_files_independent() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); let files = vec![ FileInput { path: "/a".into(), content: "image: missing:1.0".into(), images: vec![7..18] }, FileInput { path: "/b".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }, ]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert!(outcomes[0].is_err()); assert!(outcomes[1].is_ok()); assert_eq!(outcomes[1].as_ref().unwrap()[0].content, "1.22"); } #[test] fn stale_cache_ignores_cached_digest() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); let past_expiry = now - chrono::Duration::days(1); let id = db.insert_image("registry.hub.docker.com", "nginx", &past_expiry); db.insert_tag(id, "1.21", &past_expiry); db.update_tag_digest(id, "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000003", &past_expiry); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000005"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21@sha256:0000000000000000000000000000000000000000000000000000000000000001".into(), images: vec![7..89] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); let patches = outcomes[0].as_ref().unwrap(); assert_eq!(patches[0].content, "sha256:0000000000000000000000000000000000000000000000000000000000000005"); } #[test] fn error_in_first_image_skips_remaining() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "redis", "6.0", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); let files = vec![FileInput { path: "/test".into(), content: "image: nginx:1.21\nimage: redis:6.0".into(), images: vec![7..17, 25..34] }]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert!(outcomes[0].is_err()); assert!(db.get_image("registry.hub.docker.com", "redis").is_some()); assert!(db.get_tags_sorted(2).is_empty()); } #[test] fn same_image_multiple_files_fetches_twice() { crate::metrics::init(); let now = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let db = StubDb::default(); let reg = StubRegistry::default(); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); let files = vec![ FileInput { path: "/a".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }, FileInput { path: "/b".into(), content: "image: nginx:1.21".into(), images: vec![7..17] }, ]; let mut outcomes = vec![]; update_images(&now, &db, ®, &files, &mut outcomes); assert_eq!(outcomes[0].as_ref().unwrap()[0].content, "1.22"); assert_eq!(outcomes[1].as_ref().unwrap()[0].content, "1.22"); assert_eq!(reg.tags_calls.borrow()[&("registry.hub.docker.com".to_string(), "nginx".to_string())], 1); } }