From 0fd24af76e57a946129f7f9943cb61771e00c1b3 Mon Sep 17 00:00:00 2001 From: Jesper Jensen Date: Sun, 8 Feb 2026 16:28:37 +0100 Subject: Replace fetch time with expiry time This should make it easier (possible) to query for expired images --- src/db.rs | 48 +++++++++++++++++++++++++++--------------------- src/main.rs | 8 +++----- src/updater.rs | 58 ++++++++++++++++++++++++++++++---------------------------- 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/src/db.rs b/src/db.rs index ae6073f..c914769 100644 --- a/src/db.rs +++ b/src/db.rs @@ -4,11 +4,11 @@ use rusqlite::OptionalExtension; pub trait Db { fn get_image(&self, registry: &str, image: &str) -> Option<(i64, DateTime)>; - fn insert_image(&self, registry: &str, image: &str, last_checked: &DateTime) -> i64; + fn insert_image(&self, registry: &str, image: &str, expires_at: &DateTime) -> i64; fn get_tags_sorted(&self, image_id: i64) -> Vec; fn delete_tag(&self, image_id: i64, tag: &str); fn insert_tag(&self, image_id: i64, tag: &str, fetched_at: &DateTime); - fn update_last_checked(&self, image_id: i64, last_checked: &DateTime); + fn set_expires_at(&self, image_id: i64, expires_at: &DateTime); fn get_tag_digest(&self, image_id: i64, tag: &str) -> Option; fn update_tag_digest(&self, image_id: i64, tag: &str, digest: &str, fetched_at: &DateTime); } @@ -97,6 +97,12 @@ impl SqliteDb { conn.execute("INSERT INTO migrations(id) VALUES (?1)", (6, )).unwrap(); } + if newest_migration < 7 { + conn.execute("ALTER TABLE images RENAME COLUMN last_checked TO expires_at", ()).unwrap(); + conn.execute("UPDATE images SET expires_at = datetime(expires_at, '+1440 minutes')", ()).unwrap(); + conn.execute("INSERT INTO migrations(id) VALUES (?1)", (7, )).unwrap(); + } + return SqliteDb { conn }; } } @@ -105,7 +111,7 @@ impl Db for SqliteDb { fn get_image(&self, registry: &str, image: &str) -> Option<(i64, DateTime)> { let _timer = crate::metrics::get().db_query_duration.start_timer(); return self.conn.query_row(" - SELECT id, last_checked FROM images + SELECT id, expires_at FROM images WHERE registry = ?1 AND image = ?2 ", (registry, image), |row| Ok(( row.get::<_, i64>(0)?, @@ -113,11 +119,11 @@ impl Db for SqliteDb { ))).optional().unwrap(); } - fn insert_image(&self, registry: &str, image: &str, last_checked: &DateTime) -> i64 { + fn insert_image(&self, registry: &str, image: &str, expires_at: &DateTime) -> i64 { let _timer = crate::metrics::get().db_query_duration.start_timer(); self.conn.execute(" - INSERT INTO images(registry, image, last_checked) VALUES (?1, ?2, ?3) - ", (registry, image, last_checked)).unwrap(); + INSERT INTO images(registry, image, expires_at) VALUES (?1, ?2, ?3) + ", (registry, image, expires_at)).unwrap(); return self.conn.last_insert_rowid(); } @@ -144,9 +150,9 @@ impl Db for SqliteDb { (image_id, tag, fetched_at)).unwrap(); } - fn update_last_checked(&self, image_id: i64, last_checked: &DateTime) { + fn set_expires_at(&self, image_id: i64, expires_at: &DateTime) { let _timer = crate::metrics::get().db_query_duration.start_timer(); - self.conn.execute("UPDATE images SET last_checked = ?1 WHERE id = ?2", (last_checked, image_id)).unwrap(); + self.conn.execute("UPDATE images SET expires_at = ?1 WHERE id = ?2", (expires_at, image_id)).unwrap(); } fn get_tag_digest(&self, image_id: i64, tag: &str) -> Option { @@ -184,18 +190,18 @@ impl Default for StubDb { #[cfg(test)] impl Db for StubDb { fn get_image(&self, registry: &str, image: &str) -> Option<(i64, DateTime)> { - for (r, i, id, last_checked) in self.images.borrow().iter() { + for (r, i, id, expires_at) in self.images.borrow().iter() { if r == registry && i == image { - return Some((*id, *last_checked)); + return Some((*id, *expires_at)); } } return None; } - fn insert_image(&self, registry: &str, image: &str, last_checked: &DateTime) -> i64 { + fn insert_image(&self, registry: &str, image: &str, expires_at: &DateTime) -> i64 { let id = *self.next_id.borrow(); *self.next_id.borrow_mut() += 1; - self.images.borrow_mut().push((registry.to_string(), image.to_string(), id, last_checked.clone())); + self.images.borrow_mut().push((registry.to_string(), image.to_string(), id, expires_at.clone())); return id; } @@ -218,10 +224,10 @@ impl Db for StubDb { self.tags.borrow_mut().push((image_id, tag.to_string(), None, fetched_at.clone())); } - fn update_last_checked(&self, image_id: i64, last_checked: &DateTime) { - for (_, _, id, lc) in self.images.borrow_mut().iter_mut() { + fn set_expires_at(&self, image_id: i64, expires_at: &DateTime) { + for (_, _, id, ea) in self.images.borrow_mut().iter_mut() { if *id == image_id { - *lc = last_checked.clone(); + *ea = expires_at.clone(); return; } } @@ -295,11 +301,11 @@ mod tests { assert_eq!(db.get_tags_sorted(id), vec!["2.0"]); } - fn test_update_last_checked(db: &dyn Db) { + fn test_set_expires_at(db: &dyn Db) { let t1 = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); let t2 = Utc.with_ymd_and_hms(2000, 1, 1, 1, 0, 0).unwrap(); let id = db.insert_image("docker.io", "nginx", &t1); - db.update_last_checked(id, &t2); + db.set_expires_at(id, &t2); assert_eq!(db.get_image("docker.io", "nginx").unwrap().1, t2); } @@ -391,15 +397,15 @@ mod tests { } #[test] - fn conformance_stub_update_last_checked() { - test_update_last_checked(&StubDb::default()); + fn conformance_stub_set_expires_at() { + test_set_expires_at(&StubDb::default()); } #[test] - fn conformance_sqlite_update_last_checked() { + fn conformance_sqlite_set_expires_at() { crate::metrics::init(); let dir = tempfile::tempdir().unwrap(); - test_update_last_checked(&SqliteDb::new(&dir.path().join("db.sqlite"))); + test_set_expires_at(&SqliteDb::new(&dir.path().join("db.sqlite"))); } #[test] diff --git a/src/main.rs b/src/main.rs index 57f0d76..6eb2cb7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -331,11 +331,9 @@ fn run_tool(db: &dyn Db, reg: &dyn Registries, repos: &Vec, mut infi .unwrap_or("registry.hub.docker.com"); let image_name = &file.content[img.image.clone()]; - if let Some((_, last_checked)) = db.get_image(registry, image_name) { - let cache_ttl = reg.get_cache_ttl(registry); - let expiry = last_checked + cache_ttl; - if expiry < min_expiry { - min_expiry = expiry; + if let Some((_, expires_at)) = db.get_image(registry, image_name) { + if expires_at < min_expiry { + min_expiry = expires_at; } } } diff --git a/src/updater.rs b/src/updater.rs index 7b63bf7..e407a2f 100644 --- a/src/updater.rs +++ b/src/updater.rs @@ -23,7 +23,7 @@ pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, fil let mut image_ranges: Vec> = vec![]; let mut imgs: Vec = vec![]; let mut image_ids: Vec = vec![]; - let mut last_checkeds: Vec> = vec![]; + let mut expires_ats: Vec> = vec![]; for (file_idx, file) in files.iter().enumerate() { for image in &file.images { @@ -33,7 +33,7 @@ pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, fil .unwrap_or("registry.hub.docker.com"); let image_name = &file.content[img.image.clone()]; - let (image_id, last_checked) = match db.get_image(registry, image_name) { + let (image_id, expires_at) = match db.get_image(registry, image_name) { Some(x) => x, None => { let epoch = DateTime::::UNIX_EPOCH; @@ -46,7 +46,7 @@ pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, fil image_ranges.push(image.clone()); imgs.push(img); image_ids.push(image_id); - last_checkeds.push(last_checked); + expires_ats.push(expires_at); } } @@ -73,12 +73,12 @@ pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, fil let mut tag = img.tag.as_ref().map(|x| file.content[x.clone()].to_string()); let image_name = &file.content[img.image.clone()]; let image_id = image_ids[i]; - let last_checked = last_checkeds[i]; + let expires_at = expires_ats[i]; println!("Checking image {}", &file.content[image_ranges[i].clone()]); metrics::get().images_checked.with_label_values(&[registry]).inc(); - let cache_stale = *now - last_checked > cache_max_age; + let cache_stale = *now > expires_at; let cached_tags: Vec = db.get_tags_sorted(image_id); @@ -102,24 +102,26 @@ pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, fil for gone in existing { db.delete_tag(image_id, gone); } - db.update_last_checked(image_id, now); + let new_expires_at = *now + cache_max_age; + db.set_expires_at(image_id, &new_expires_at); // @CLEANUP: Propagate the update to other instances. We should really avoid // this situation in the first place for j in (i + 1)..image_ids.len() { if image_ids[j] == image_id { - last_checkeds[j] = *now; + expires_ats[j] = new_expires_at; } } Ok((fetched_tags, None)) } None => { - db.update_last_checked(image_id, now); + let new_expires_at = *now + cache_max_age; + db.set_expires_at(image_id, &new_expires_at); // @CLEANUP: Propagate the update to other instances. We should really avoid // this situation in the first place for j in (i + 1)..image_ids.len() { if image_ids[j] == image_id { - last_checkeds[j] = *now; + expires_ats[j] = new_expires_at; } } Err(format!("image not found: {}", image_name)) @@ -227,10 +229,10 @@ mod tests { let db = StubDb::default(); let reg = StubRegistry::default(); - let old = now - chrono::Duration::days(2); - let id = db.insert_image("registry.hub.docker.com", "nginx", &old); - db.insert_tag(id, "1.20", &old); - db.insert_tag(id, "1.21", &old); + let past_expiry = now - chrono::Duration::days(1); + let id = db.insert_image("registry.hub.docker.com", "nginx", &past_expiry); + db.insert_tag(id, "1.20", &past_expiry); + db.insert_tag(id, "1.21", &past_expiry); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000001"); reg.add_tag("registry.hub.docker.com", "nginx", "1.22", "sha256:0000000000000000000000000000000000000000000000000000000000000002"); @@ -243,8 +245,8 @@ mod tests { update_images(&now, &db, ®, &files, &mut outcomes); assert_eq!(db.get_tags_sorted(id), vec!["1.21", "1.22"]); - let (_, last_checked) = db.get_image("registry.hub.docker.com", "nginx").unwrap(); - assert_eq!(last_checked, now); + let (_, expires_at) = db.get_image("registry.hub.docker.com", "nginx").unwrap(); + assert!(expires_at > now); } #[test] @@ -254,9 +256,9 @@ mod tests { let db = StubDb::default(); let reg = StubRegistry::default(); - let recent = now - chrono::Duration::minutes(30); - let id = db.insert_image("registry.hub.docker.com", "nginx", &recent); - db.insert_tag(id, "1.21", &recent); + let future_expiry = now + chrono::Duration::minutes(30); + let id = db.insert_image("registry.hub.docker.com", "nginx", &future_expiry); + db.insert_tag(id, "1.21", &now); let files = vec![FileInput { path: "/test".into(), @@ -361,10 +363,10 @@ mod tests { let db = StubDb::default(); let reg = StubRegistry::default(); - let recent = now - chrono::Duration::minutes(30); - let id = db.insert_image("registry.hub.docker.com", "nginx", &recent); - db.insert_tag(id, "1.21", &recent); - db.update_tag_digest(id, "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000003", &recent); + let future_expiry = now + chrono::Duration::minutes(30); + let id = db.insert_image("registry.hub.docker.com", "nginx", &future_expiry); + db.insert_tag(id, "1.21", &now); + db.update_tag_digest(id, "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000003", &now); let files = vec![FileInput { path: "/test".into(), @@ -484,8 +486,8 @@ mod tests { assert!(outcomes[0].is_err()); assert!(outcomes[0].as_ref().unwrap_err().contains("image not found")); - let (_, last_checked) = db.get_image("registry.hub.docker.com", "nginx").unwrap(); - assert_eq!(last_checked, now); + let (_, expires_at) = db.get_image("registry.hub.docker.com", "nginx").unwrap(); + assert!(expires_at > now); } #[test] @@ -593,10 +595,10 @@ mod tests { let db = StubDb::default(); let reg = StubRegistry::default(); - let old = now - chrono::Duration::days(2); - let id = db.insert_image("registry.hub.docker.com", "nginx", &old); - db.insert_tag(id, "1.21", &old); - db.update_tag_digest(id, "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000003", &old); + let past_expiry = now - chrono::Duration::days(1); + let id = db.insert_image("registry.hub.docker.com", "nginx", &past_expiry); + db.insert_tag(id, "1.21", &past_expiry); + db.update_tag_digest(id, "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000003", &past_expiry); reg.add_tag("registry.hub.docker.com", "nginx", "1.21", "sha256:0000000000000000000000000000000000000000000000000000000000000005"); -- cgit v1.2.3