From c95756dda52d8817b6955080be473073267ec17f Mon Sep 17 00:00:00 2001 From: Jesper Jensen Date: Sun, 8 Feb 2026 12:55:18 +0100 Subject: Linearize images with update_images --- src/updater.rs | 248 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 133 insertions(+), 115 deletions(-) (limited to 'src') diff --git a/src/updater.rs b/src/updater.rs index 3f4fe51..a21f378 100644 --- a/src/updater.rs +++ b/src/updater.rs @@ -19,14 +19,13 @@ pub struct FileInput { } pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, files: &[FileInput], outcomes: &mut Vec, String>>) { - for file in files { - let mut patches = vec![]; - let mut error: Option = None; - - let mut imgs: Vec = vec![]; - let mut image_ids: Vec = vec![]; - let mut last_checkeds: Vec> = vec![]; + let mut file_indices: Vec = vec![]; + let mut image_ranges: Vec> = vec![]; + let mut imgs: Vec = vec![]; + let mut image_ids: Vec = vec![]; + let mut last_checkeds: Vec> = vec![]; + for (file_idx, file) in files.iter().enumerate() { for image in &file.images { let img = DockerRef::parse(&file.content, image); let registry = img.registry.as_ref() @@ -43,138 +42,157 @@ pub fn update_images(now: &DateTime, db: &dyn Db, reg: &dyn Registries, fil }, }; + file_indices.push(file_idx); + image_ranges.push(image.clone()); imgs.push(img); image_ids.push(image_id); last_checkeds.push(last_checked); } + } - for i in 0..imgs.len() { - let img = &imgs[i]; - let registry = img.registry.as_ref() - .map(|x| &file.content[x.clone()]) - .unwrap_or("registry.hub.docker.com"); - let cache_max_age = reg.get_cache_ttl(registry); - let mut tag = img.tag.as_ref().map(|x| file.content[x.clone()].to_string()); - let image_name = &file.content[img.image.clone()]; - let image_id = image_ids[i]; - let last_checked = last_checkeds[i]; - - println!("Checking image {}", &file.content[file.images[i].clone()]); - metrics::get().images_checked.with_label_values(&[registry]).inc(); - - let cache_stale = *now - last_checked > cache_max_age; - - let cached_tags: Vec = db.get_tags_sorted(image_id); - - let tags_result: Result<(Vec, Option), String> = if cache_stale { - println!("Invalid in cache"); - match reg.get_tags(registry, image_name) { - Some(mut fetched_tags) => { - fetched_tags.sort(); - let mut existing = cached_tags.iter().peekable(); - for t in &fetched_tags { - while existing.peek().is_some_and(|e| *e < t) { - let gone = existing.next().unwrap(); - db.delete_tag(image_id, gone); - } - if existing.peek() == Some(&t) { - existing.next(); - } else { - db.insert_tag(image_id, t, now); - } - } - for gone in existing { + let mut file_patches: Vec> = Vec::with_capacity(files.len()); + let mut file_errors: Vec> = Vec::with_capacity(files.len()); + for _ in 0..files.len() { + file_patches.push(vec![]); + file_errors.push(None); + } + + for i in 0..imgs.len() { + let file_idx = file_indices[i]; + let file = &files[file_idx]; + + if file_errors[file_idx].is_some() { + continue; + } + + let img = &imgs[i]; + let registry = img.registry.as_ref() + .map(|x| &file.content[x.clone()]) + .unwrap_or("registry.hub.docker.com"); + let cache_max_age = reg.get_cache_ttl(registry); + let mut tag = img.tag.as_ref().map(|x| file.content[x.clone()].to_string()); + let image_name = &file.content[img.image.clone()]; + let image_id = image_ids[i]; + let last_checked = last_checkeds[i]; + + println!("Checking image {}", &file.content[image_ranges[i].clone()]); + metrics::get().images_checked.with_label_values(&[registry]).inc(); + + let cache_stale = *now - last_checked > cache_max_age; + + let cached_tags: Vec = db.get_tags_sorted(image_id); + + let tags_result: Result<(Vec, Option), String> = if cache_stale { + println!("Invalid in cache"); + match reg.get_tags(registry, image_name) { + Some(mut fetched_tags) => { + fetched_tags.sort(); + let mut existing = cached_tags.iter().peekable(); + for t in &fetched_tags { + while existing.peek().is_some_and(|e| *e < t) { + let gone = existing.next().unwrap(); db.delete_tag(image_id, gone); } - db.update_last_checked(image_id, now); - - Ok((fetched_tags, None)) + if existing.peek() == Some(&t) { + existing.next(); + } else { + db.insert_tag(image_id, t, now); + } } - None => { - db.update_last_checked(image_id, now); - Err(format!("image not found: {}", image_name)) + for gone in existing { + db.delete_tag(image_id, gone); } - } - } else { - println!("Valid in cache"); - let tag_for_digest = tag.as_deref().unwrap_or("latest"); - let digest = db.get_tag_digest(image_id, tag_for_digest); + db.update_last_checked(image_id, now); - Ok((cached_tags, digest)) - }; - - let (tags, cached_digest) = match tags_result { - Ok(v) => v, - Err(msg) => { - error = Some(msg); - break; + Ok((fetched_tags, None)) } - }; - - if let Some(ref tag_str) = tag { - let mut current = VersionPattern::parse(&tag_str); - let mut new_tag = None; - - for candidate_str in &tags { - let candidate = VersionPattern::parse(&candidate_str); - match current.compare(&candidate) { - CompareOutcome::Higher => { - new_tag = Some(candidate_str.clone()); - current = candidate; - }, - CompareOutcome::Lower => {}, - CompareOutcome::Incompatible => {}, - CompareOutcome::Identical => {}, - } + None => { + db.update_last_checked(image_id, now); + Err(format!("image not found: {}", image_name)) } - - if let Some(new_tag) = new_tag { - tag = Some(new_tag.clone()); - patches.push(FilePatch { - position: img.tag.clone().unwrap(), - content: new_tag, - }); - metrics::get().images_updated.with_label_values(&[registry]).inc(); + } + } else { + println!("Valid in cache"); + let tag_for_digest = tag.as_deref().unwrap_or("latest"); + let digest = db.get_tag_digest(image_id, tag_for_digest); + + Ok((cached_tags, digest)) + }; + + let (tags, cached_digest) = match tags_result { + Ok(v) => v, + Err(msg) => { + file_errors[file_idx] = Some(msg); + continue; + } + }; + + if let Some(ref tag_str) = tag { + let mut current = VersionPattern::parse(&tag_str); + let mut new_tag = None; + + for candidate_str in &tags { + let candidate = VersionPattern::parse(&candidate_str); + match current.compare(&candidate) { + CompareOutcome::Higher => { + new_tag = Some(candidate_str.clone()); + current = candidate; + }, + CompareOutcome::Lower => {}, + CompareOutcome::Incompatible => {}, + CompareOutcome::Identical => {}, } } - if let Some(ref digest) = img.digest { - let tag_for_digest = tag.as_deref().unwrap_or("latest"); - - let digest_result = match cached_digest { - Some(d) => Ok(d), - None => { - match reg.get_digest(registry, image_name, tag_for_digest) { - Some(fetched) => { - db.update_tag_digest(image_id, tag_for_digest, &fetched, now); - Ok(fetched) - } - None => Err(format!("digest not found for {}:{}", image_name, tag_for_digest)), + if let Some(new_tag) = new_tag { + tag = Some(new_tag.clone()); + file_patches[file_idx].push(FilePatch { + position: img.tag.clone().unwrap(), + content: new_tag, + }); + metrics::get().images_updated.with_label_values(&[registry]).inc(); + } + } + + if let Some(ref digest) = img.digest { + let tag_for_digest = tag.as_deref().unwrap_or("latest"); + + let digest_result = match cached_digest { + Some(d) => Ok(d), + None => { + match reg.get_digest(registry, image_name, tag_for_digest) { + Some(fetched) => { + db.update_tag_digest(image_id, tag_for_digest, &fetched, now); + Ok(fetched) } + None => Err(format!("digest not found for {}:{}", image_name, tag_for_digest)), } - }; + } + }; - let digest_string = match digest_result { - Ok(d) => d, - Err(msg) => { - error = Some(msg); - break; - } - }; - - if file.content[digest.clone()] != digest_string { - patches.push(FilePatch { - position: img.digest.clone().unwrap(), - content: digest_string, - }); - metrics::get().images_updated.with_label_values(&[registry]).inc(); + let digest_string = match digest_result { + Ok(d) => d, + Err(msg) => { + file_errors[file_idx] = Some(msg); + continue; } + }; + + if file.content[digest.clone()] != digest_string { + file_patches[file_idx].push(FilePatch { + position: img.digest.clone().unwrap(), + content: digest_string, + }); + metrics::get().images_updated.with_label_values(&[registry]).inc(); } } + } - if let Some(msg) = error { + for file_idx in 0..files.len() { + if let Some(msg) = file_errors[file_idx].take() { outcomes.push(Err(msg)); } else { + let mut patches = std::mem::take(&mut file_patches[file_idx]); patches.sort_by_key(|e| e.position.start); outcomes.push(Ok(patches)); } -- cgit v1.2.3