mod parser; mod version; mod docker; mod manifest; mod dockerfile; mod metrics; mod time; mod db; mod registry; mod updater; mod refresher; use crate::docker::DockerRef; use crate::manifest::ManifestFile; use crate::dockerfile::DockerfileFile; use crate::db::{Db, SqliteDb}; use crate::time::Timestamp; use crate::registry::{Registries, HttpRegistry, Config, Credentials}; use crate::updater::{FileInput, FilePatch, update_images}; use rand::distr::{Alphanumeric, SampleString}; use std::io::Write; use std::time::Duration; struct Repository { url: String, key: Option, host: Option, dest: std::path::PathBuf, } fn create_temp_path(path: &std::path::Path) -> std::path::PathBuf { loop { let mut filename = std::ffi::OsString::new(); filename.push(path.file_name().unwrap()); filename.push(std::ffi::OsStr::new(".edit")); filename.push(Alphanumeric.sample_string(&mut rand::rng(), 16)); let temp_path = path.parent().unwrap().join(filename); if !temp_path.exists() { return temp_path; } } } fn is_dockerfile(path: &std::path::Path) -> bool { if let Some(name) = path.file_name() { if let Some(name) = name.to_str() { return name == "Dockerfile" || name == "dockerfile"; } } return false; } fn is_yaml(path: &std::path::Path) -> bool { if let Some(ext) = path.extension() { if let Some(ext) = ext.to_str() { return ext == "yaml"; } } return false; } fn run_tool(db: &dyn Db, reg: &dyn Registries, repos: &Vec, mut infile_paths: Vec) -> Timestamp { for repo in repos { if repo.dest.exists() { let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("fetch") .arg("origin"); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("reset") .arg("--hard") .arg("@{u}"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("clean") .arg("--force") .arg("-d") .arg("-x"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } } else { let mut gitcmd = std::process::Command::new("git"); println!("Cloning {}", repo.url); gitcmd .arg("clone") .arg(&repo.url) .arg(&repo.dest); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } } infile_paths.push(repo.dest.clone()); } let mut inpaths = vec![]; for infile_path in infile_paths { if infile_path.is_dir() { let mut unsearched = vec![infile_path.clone()]; while let Some(next) = unsearched.pop() { for child in next.read_dir().unwrap() { let child = child.unwrap(); let path = child.path(); let ft = child.file_type().unwrap(); if ft.is_dir() { unsearched.push(path); continue; } if is_yaml(&path) || is_dockerfile(&path) { inpaths.push(path); } } } } else { inpaths.push(infile_path); }; } let mut file_inputs: Vec = vec![]; for path in inpaths { let content = std::fs::read_to_string(&path).unwrap(); let images = if is_dockerfile(&path) { DockerfileFile::parse(&content).image_refs } else { ManifestFile::parse(&content).image_tags }; file_inputs.push(FileInput { path, content, images }); } let now = Timestamp::now(); let mut outcomes: Vec, String>> = Vec::with_capacity(file_inputs.len()); update_images(now, db, reg, &file_inputs, &mut outcomes); for i in 0..file_inputs.len() { let file = &file_inputs[i]; match &outcomes[i] { Ok(patches) => { if patches.is_empty() { continue; } let temp_path = create_temp_path(&file.path); let mut out = std::fs::File::create(&temp_path).unwrap(); let mut current_position = 0; for patch in patches { if patch.position.start > current_position { out.write_all(file.content[current_position..patch.position.start].as_bytes()).unwrap(); } out.write_all(patch.content.as_bytes()).unwrap(); current_position = patch.position.end; } out.write_all(file.content[current_position..].as_bytes()).unwrap(); std::fs::remove_file(&file.path).unwrap(); std::fs::rename(&temp_path, &file.path).unwrap(); } Err(msg) => { println!("{}: {}", file.path.display(), msg); } } } for repo in repos { let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-c") .arg("user.name=vbump") .arg("-c") .arg("user.email=jesper@jnsn.dev") .arg("-C") .arg(&repo.dest) .arg("commit") .arg("--all") .arg("--message") .arg("Update versions"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { metrics::get().git_operations.with_label_values(&["commit", "failure"]).inc(); println!("Commit failed, presumably there were no changes"); continue; } metrics::get().git_operations.with_label_values(&["commit", "success"]).inc(); let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("push") .arg("origin") .arg("+HEAD:version-bump"); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { metrics::get().git_operations.with_label_values(&["push", "failure"]).inc(); panic!("Git exited with failure"); } metrics::get().git_operations.with_label_values(&["push", "success"]).inc(); } let now = Timestamp::now(); let mut min_expiry = now + Duration::from_secs(86400); for file in &file_inputs { for image_range in &file.images { let img = DockerRef::parse(&file.content, image_range); let registry = img.registry.as_ref() .map(|x| &file.content[x.clone()]) .unwrap_or("registry.hub.docker.com"); let image_name = &file.content[img.image.clone()]; if let Some(img) = db.get_image(registry, image_name) { if img.expires_at < min_expiry { min_expiry = img.expires_at; } } } } if min_expiry < now { min_expiry = now; } return min_expiry; } fn main() { let config_path = std::path::PathBuf::from("/config/"); let scratch_path = std::path::PathBuf::from("/workdir"); let db_path = std::path::PathBuf::from("/data/db.sqlite"); let sqlite_db = SqliteDb::new(&db_path); let infile_paths: Vec = vec![]; let mut repos = vec![]; let mut auths = vec![]; let mut docker_auths: Vec<(String, Credentials)> = vec![]; let mut registry_ttls: Vec<(String, Duration)> = vec![]; let workdir = std::env::current_dir().unwrap(); if config_path.is_dir() { let mut unsearched = vec![config_path]; while let Some(next) = unsearched.pop() { for child in next.read_dir().unwrap() { let child = child.unwrap(); let path = child.path(); let ft = child.file_type().unwrap(); if ft.is_dir() { unsearched.push(path); continue; } if let Some(name) = path.file_name() { match name.to_str() { Some(".dockerconfigjson") => { let file = std::fs::File::open(&path).unwrap(); let mut reader = json_event_parser::ReaderJsonParser::new(file); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "auths" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } let mut possible_key = reader.parse_next(); while let Ok(json_event_parser::JsonEvent::ObjectKey(k)) = possible_key { let key = k.into_owned(); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "auth" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { docker_auths.push((key, Credentials::Basic(v.into_owned()))); }, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } possible_key = reader.parse_next(); } match possible_key { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } } Some("repository") => { let file = std::fs::File::open(&path).unwrap(); let mut reader = json_event_parser::ReaderJsonParser::new(file); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "repositories" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } let mut possible_key = reader.parse_next(); while let Ok(json_event_parser::JsonEvent::ObjectKey(k)) = possible_key { let k = k.into_owned(); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "key" => {}, _ => panic!("Invalid config json"), } let key_name = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "host" => {}, _ => panic!("Invalid config json"), } let host_name = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "dest" => {}, _ => panic!("Invalid config json"), } let dest = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } repos.push(Repository { url: k, key: Some(workdir.join(path.parent().unwrap().join(std::path::PathBuf::from(key_name)))), host: Some(workdir.join(path.parent().unwrap().join(std::path::PathBuf::from(host_name)))), dest: (&scratch_path).join(dest), }); possible_key = reader.parse_next(); } match possible_key { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } } Some("registry") => { let file = std::fs::File::open(&path).unwrap(); let mut reader = json_event_parser::ReaderJsonParser::new(file); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid registry config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "registries" => {}, _ => panic!("Invalid registry config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid registry config json"), } let mut possible_key = reader.parse_next(); while let Ok(json_event_parser::JsonEvent::ObjectKey(k)) = possible_key { let host = k.into_owned(); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid registry config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "cache_ttl_minutes" => {}, _ => panic!("Invalid registry config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::Number(n)) => { let minutes: u64 = n.parse().unwrap(); registry_ttls.push((host, Duration::from_secs(minutes * 60))); }, _ => panic!("Invalid registry config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid registry config json"), } possible_key = reader.parse_next(); } match possible_key { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid registry config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid registry config json"), } } Some(_) | None => {} } } } } } // Merge docker_auths and registry_ttls into final configs for (host, creds) in docker_auths { let mut ttl = Duration::from_secs(86400); for (h, t) in ®istry_ttls { if h == &host { ttl = *t; break; } } auths.push(Config { host, credentials: creds, cache_ttl: ttl }); } // Add registry_ttls entries that don't have auth (anonymous) for (host, ttl) in registry_ttls { let mut found = false; for c in &auths { if c.host == host { found = true; break; } } if !found { auths.push(Config { host, credentials: Credentials::Anonymous, cache_ttl: ttl, }); } } let registry = HttpRegistry::new(auths); metrics::init(); metrics::get().state.set(metrics::STATE_IDLE); metrics::serve(9090); loop { metrics::get().state.set(metrics::STATE_ACTIVE); let run_start = std::time::Instant::now(); let next_wakeup = run_tool(&sqlite_db, ®istry, &repos, infile_paths.clone()); let run_duration = run_start.elapsed().as_secs_f64(); metrics::get().run_duration.set(run_duration); metrics::get().next_wakeup.set(next_wakeup.duration_since(Timestamp::ZERO).unwrap().as_secs() as f64); metrics::get().state.set(metrics::STATE_IDLE); let sleep_duration = next_wakeup.duration_since(Timestamp::now()).unwrap_or(Duration::ZERO); println!("Waiting {} seconds for next run", sleep_duration.as_secs()); std::thread::sleep(sleep_duration); } }