mod parser; mod version; mod docker; mod manifest; mod dockerfile; mod metrics; mod db; mod registry; use crate::version::{VersionPattern, CompareOutcome}; use crate::docker::DockerRef; use crate::manifest::ManifestFile; use crate::dockerfile::DockerfileFile; use crate::db::{Db, SqliteDb}; use crate::registry::{Registry, HttpRegistry, AuthInfo, basic_auth}; use rand::distr::{Alphanumeric, SampleString}; use std::ops::Range; use std::io::Read; use std::io::Seek; use std::io::Write; fn help(cmd: &str) { println!( "{} [options] [--] Search FILE for docker images and suggest updates Options: --auth Authenticate against REGISTRY (repeatable) --config Read config from PATH --scratch Store temporary files in DIR --metrics-port Serve Prometheus metrics on PORT (default: disabled)", cmd ); } struct Repository { url: String, key: Option, host: Option, dest: std::path::PathBuf, } #[derive(Debug)] struct FilePatch { position: Range, content: String, } fn update_images(db: &dyn Db, reg: &dyn Registry, file: &str, img: DockerRef, edits: &mut Vec) -> Result<(), String> { let registry = img.registry.map(|x| &file[x]).unwrap_or("registry.hub.docker.com"); let mut tag = img.tag.as_ref().map(|x| file[x.clone()].to_string()); let image = &file[img.image.clone()]; metrics::get().images_checked.with_label_values(&[registry]).inc(); let now = chrono::offset::Utc::now(); let cache_max_age = chrono::Duration::hours(24); let (image_id, last_checked) = match db.get_image(registry, image) { Some(x) => x, None => { let id = db.insert_image(registry, image, now); (id, now) }, }; let cache_stale = now - last_checked > cache_max_age; let cached_tags: Vec = db.get_tags_sorted(image_id); let (tags, cached_digest): (Vec, Option) = if cache_stale { println!("Invalid in cache"); let mut fetched_tags = reg.get_tags(registry, image) .ok_or_else(|| format!("image not found: {}", image))?; fetched_tags.sort(); let mut existing = cached_tags.iter().peekable(); for t in &fetched_tags { while existing.peek().is_some_and(|e| *e < t) { let gone = existing.next().unwrap(); db.delete_tag(image_id, gone); } if existing.peek() == Some(&t) { existing.next(); } else { db.insert_tag(image_id, t, now); } } for gone in existing { db.delete_tag(image_id, gone); } db.update_last_checked(image_id, now); (fetched_tags, None) } else { println!("Valid in cache"); let tag_for_digest = tag.as_deref().unwrap_or("latest"); let digest = db.get_tag_digest(image_id, tag_for_digest); (cached_tags, digest) }; if let Some(ref tag_str) = tag { let mut current = VersionPattern::parse(&tag_str); let mut new_tag = None; for candidate_str in &tags { let candidate = VersionPattern::parse(&candidate_str); match current.compare(&candidate) { CompareOutcome::Higher => { new_tag = Some(candidate_str.clone()); current = candidate; }, CompareOutcome::Lower => {}, CompareOutcome::Incompatible => {}, CompareOutcome::Identical => {}, } } if let Some(new_tag) = new_tag { tag = Some(new_tag.clone()); edits.push(FilePatch { position: img.tag.unwrap(), content: new_tag, }); metrics::get().images_updated.with_label_values(&[registry]).inc(); } } if let Some(ref digest) = img.digest { let tag_for_digest = tag.as_deref().unwrap_or("latest"); let digest_string = match cached_digest { Some(d) => d, None => { let fetched = reg.get_digest(registry, image, tag_for_digest) .ok_or_else(|| format!("digest not found for {}:{}", image, tag_for_digest))?; db.update_tag_digest(image_id, tag_for_digest, &fetched, now); fetched } }; if file[digest.clone()] != digest_string { edits.push(FilePatch { position: img.digest.unwrap(), content: digest_string, }); metrics::get().images_updated.with_label_values(&[registry]).inc(); } } return Ok(()); } enum Output { Overlay(std::fs::File, std::path::PathBuf, std::path::PathBuf), Stdout(std::io::Stdout), } impl Output { fn overlay_file(infile_path: &std::path::Path) -> Self { loop { let mut filename = std::ffi::OsString::new(); filename.push(infile_path.file_name().unwrap()); filename.push(std::ffi::OsStr::new(".edit")); filename.push(Alphanumeric.sample_string(&mut rand::rng(), 16)); let outfile_path = Some(infile_path.parent().unwrap().join(filename)); if let Ok(output_file) = std::fs::File::create_new(outfile_path.as_ref().unwrap()) { return Output::Overlay(output_file, outfile_path.unwrap().into(), infile_path.into()); } } } fn commit(&mut self) { match self { Output::Overlay(_, outfile_path, infile_path) => { std::fs::remove_file(&infile_path).unwrap(); std::fs::rename(&outfile_path, &infile_path).unwrap(); } Output::Stdout(_) => {} } } } impl std::ops::Deref for Output { type Target = dyn Write; fn deref(&self) -> &Self::Target { match self { Output::Overlay(file, _, _) => file, Output::Stdout(stdout) => stdout, } } } impl std::ops::DerefMut for Output { fn deref_mut(&mut self) -> &mut Self::Target { match self { Output::Overlay(file, _, _) => file, Output::Stdout(stdout) => stdout, } } } impl std::io::Write for Output { fn write(&mut self, buf: &[u8]) -> std::io::Result { return (**self).write(buf); } fn flush(&mut self) -> std::io::Result<()> { return (**self).flush(); } fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result { return (**self).write_vectored(bufs); } fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> { return (**self).write_all(buf); } fn write_fmt(&mut self, args: std::fmt::Arguments<'_>) -> std::io::Result<()> { return (**self).write_fmt(args); } } fn is_dockerfile(path: &std::path::Path) -> bool { if let Some(name) = path.file_name() { if let Some(name) = name.to_str() { return name == "Dockerfile" || name == "dockerfile"; } } return false; } fn is_yaml(path: &std::path::Path) -> bool { if let Some(ext) = path.extension() { if let Some(ext) = ext.to_str() { return ext == "yaml"; } } return false; } fn run_tool(db: &dyn Db, reg: &dyn Registry, repos: &Vec, mut infile_paths: Vec, overwrite: bool) { for repo in repos { if repo.dest.exists() { let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("fetch") .arg("origin"); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("reset") .arg("--hard") .arg("@{u}"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("clean") .arg("--force") .arg("-d") .arg("-x"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } } else { let mut gitcmd = std::process::Command::new("git"); println!("Cloning {}", repo.url); gitcmd .arg("clone") .arg(&repo.url) .arg(&repo.dest); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } } infile_paths.push(repo.dest.clone()); } let mut inpaths = vec![]; for infile_path in infile_paths { if infile_path.is_dir() { let mut unsearched = vec![infile_path.clone()]; while let Some(next) = unsearched.pop() { for child in next.read_dir().unwrap() { let child = child.unwrap(); let path = child.path(); let ft = child.file_type().unwrap(); if ft.is_dir() { unsearched.push(path); continue; } if is_yaml(&path) || is_dockerfile(&path) { inpaths.push(path); } } } } else { inpaths.push(infile_path); }; } for infile_path in inpaths { let mut file = std::fs::File::open(&infile_path).unwrap(); let mut file_content = String::new(); file.read_to_string(&mut file_content).unwrap(); let images: Vec> = if is_dockerfile(&infile_path) { DockerfileFile::parse(&file_content).image_refs } else { ManifestFile::parse(&file_content).image_tags }; let mut failed = None; let mut edits = vec![]; for ref image in images { println!("Checking image {}", &file_content[image.clone()]); let image_ref = DockerRef::parse(&file_content, image); if let Err(msg) = update_images(db, reg, &file_content, image_ref, &mut edits) { failed = Some(msg); break; } } if let Some(msg) = failed { println!("{}: {} ", infile_path.display(), msg); continue; } let mut out = if overwrite { Output::overlay_file(&infile_path) } else { Output::Stdout(std::io::stdout()) }; let mut current_position = 0; file.seek(std::io::SeekFrom::Start(0)).unwrap(); let mut file_block = file.take(0); for edit in edits { if edit.position.start > current_position { file_block.set_limit((edit.position.start - current_position) as u64); std::io::copy(&mut file_block, &mut out).unwrap(); } out.write_all(edit.content.as_bytes()).unwrap(); // We should have been able to just seek to the position.end here, but that doesn't work // for whatever reason. What we can do is calculate the amount to skip ahead, and then do // that. let skip = (edit.position.end - edit.position.start) as i64; file_block.set_limit(skip as u64); file_block.seek(std::io::SeekFrom::Current(skip)).unwrap(); current_position = edit.position.end; } file_block.set_limit(u64::MAX); std::io::copy(&mut file_block, &mut out).unwrap(); out.commit(); } for repo in repos { let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-c") .arg("user.name=vbump") .arg("-c") .arg("user.email=jesper@jnsn.dev") .arg("-C") .arg(&repo.dest) .arg("commit") .arg("--all") .arg("--message") .arg("Update versions"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { metrics::get().git_operations.with_label_values(&["commit", "failure"]).inc(); println!("Commit failed, presumably there were no changes"); continue; } metrics::get().git_operations.with_label_values(&["commit", "success"]).inc(); let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("push") .arg("origin") .arg("+HEAD:version-bump"); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { metrics::get().git_operations.with_label_values(&["push", "failure"]).inc(); panic!("Git exited with failure"); } metrics::get().git_operations.with_label_values(&["push", "success"]).inc(); } } fn main() { let argv: Vec = std::env::args().collect(); let mut it = argv.iter(); let cmd = &it.next().unwrap(); let mut auths = vec![]; let mut positional: Vec<&str> = vec!(); let mut overwrite = false; let mut config_path = None; let mut scratch_path = None; let mut continuous = false; let mut metrics_port: Option = None; loop { match it.next().map(|x| x.as_str()) { None => break, Some("--auth") => { if let Some(registry) = it.next() && let Some(username) = it.next() && let Some(password) = it.next() { auths.push(AuthInfo { host: registry.clone(), auth: basic_auth(username, password), }); } else { println!("Error: --auth requires three parameters"); help(cmd); std::process::exit(1); } }, Some("--config") => { if let Some(path) = it.next() { config_path = Some(std::path::PathBuf::from(path)); } else { println!("Error: --config requires an parameters"); help(cmd); std::process::exit(1); } }, Some("--scratch") => { if let Some(path) = it.next() { scratch_path = Some(std::path::PathBuf::from(path)); } else { println!("Error: --scratch requires an parameters"); help(cmd); std::process::exit(1); } }, Some("--continuous") => { continuous = true; }, Some("--metrics-port") => { if let Some(port_str) = it.next() { match port_str.parse::() { Ok(port) => metrics_port = Some(port), Err(_) => { println!("Error: --metrics-port requires a valid port number"); help(cmd); std::process::exit(1); } } } else { println!("Error: --metrics-port requires a parameter"); help(cmd); std::process::exit(1); } }, Some("-i") | Some("--inplace") => { overwrite = true; }, Some("-h") | Some("--help") => { help(cmd); std::process::exit(0); }, Some(arg) => positional.push(arg), }; } let mut infile_paths = vec![]; if positional.len() < 1 { panic!("BAD ARGUMENTS"); } let sqlite_db = SqliteDb::new(&std::path::PathBuf::from(positional[0])); if positional.len() > 1 { infile_paths.push(std::path::PathBuf::from(positional[1])); } let mut repos = vec![]; let workdir = std::env::current_dir().unwrap(); if let Some(config_path) = config_path { if !config_path.is_dir() { println!("config is not a directory"); std::process::exit(1); } let mut unsearched = vec![config_path]; while let Some(next) = unsearched.pop() { for child in next.read_dir().unwrap() { let child = child.unwrap(); let path = child.path(); let ft = child.file_type().unwrap(); if ft.is_dir() { unsearched.push(path); continue; } if let Some(name) = path.file_name() { match name.to_str() { Some(".dockerconfigjson") => { let file = std::fs::File::open(&path).unwrap(); let mut reader = json_event_parser::ReaderJsonParser::new(file); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "auths" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } let mut possible_key = reader.parse_next(); while let Ok(json_event_parser::JsonEvent::ObjectKey(k)) = possible_key { let key = k.into_owned(); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "auth" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { auths.push(AuthInfo { host: key, auth: v.into_owned(), }); }, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } possible_key = reader.parse_next(); } match possible_key { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } } Some("repository") => { let file = std::fs::File::open(&path).unwrap(); let mut reader = json_event_parser::ReaderJsonParser::new(file); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "repositories" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } let mut possible_key = reader.parse_next(); while let Ok(json_event_parser::JsonEvent::ObjectKey(k)) = possible_key { let k = k.into_owned(); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "key" => {}, _ => panic!("Invalid config json"), } let key_name = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "host" => {}, _ => panic!("Invalid config json"), } let host_name = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "dest" => {}, _ => panic!("Invalid config json"), } let dest = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } repos.push(Repository { url: k, key: Some(workdir.join(path.parent().unwrap().join(std::path::PathBuf::from(key_name)))), host: Some(workdir.join(path.parent().unwrap().join(std::path::PathBuf::from(host_name)))), dest: scratch_path.as_ref().unwrap().join(dest), }); possible_key = reader.parse_next(); } match possible_key { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } } Some(_) | None => {} } } } } } let registry = HttpRegistry::new(auths); metrics::init(); metrics::get().state.set(metrics::STATE_IDLE); if let Some(port) = metrics_port { metrics::serve(port); } loop { metrics::get().state.set(metrics::STATE_ACTIVE); let run_start = std::time::Instant::now(); run_tool(&sqlite_db, ®istry, &repos, infile_paths.clone(), overwrite); let run_duration = run_start.elapsed().as_secs_f64(); metrics::get().run_duration.set(run_duration); metrics::get().state.set(metrics::STATE_IDLE); if !continuous { break; } println!("Waiting for next run"); std::thread::sleep(std::time::Duration::from_hours(24)); } }