mod parser; mod version; mod docker; mod manifest; mod dockerfile; mod metrics; use crate::parser::*; use crate::version::{VersionPattern, CompareOutcome}; use crate::docker::DockerRef; use crate::manifest::ManifestFile; use crate::dockerfile::DockerfileFile; use base64::prelude::*; use rand::distr::{Alphanumeric, SampleString}; use rusqlite::Connection; use rusqlite::OptionalExtension; use std::collections::HashMap; use std::ops::Range; use std::io::Read; use std::io::Seek; use std::io::Write; fn help(cmd: &str) { println!( "{} [options] [--] Search FILE for docker images and suggest updates Options: --auth Authenticate against REGISTRY (repeatable) --config Read config from PATH --scratch Store temporary files in DIR --metrics-port Serve Prometheus metrics on PORT (default: disabled)", cmd ); } #[derive(Debug)] enum AuthMethod { Basic, Bearer { realm: String, service: String, scope: String }, } impl AuthMethod { fn from_header(header: &ureq::http::HeaderValue) -> Option { let header_str = header.to_str().unwrap(); let parse = parse_authenticate_header(header_str).unwrap(); if &header_str[parse.scheme.clone()] == "Basic" { return Some(AuthMethod::Basic); } if &header_str[parse.scheme.clone()] == "Bearer" { return Some(AuthMethod::Bearer { realm: header_str[parse.realm.unwrap()].to_string(), scope: header_str[parse.scope.unwrap()].to_string(), service: header_str[parse.service.unwrap()].to_string(), }); } return None; } } fn basic_auth(user: &str, pass: &str) -> String { return BASE64_STANDARD.encode(format!("{}:{}", user, pass)); } struct AuthInfo { host: String, auth: String, } enum AuthStage { Idle, Authorized(String), } struct AuthState { info: AuthInfo, stage: AuthStage, } struct Auth { states: HashMap } impl Auth { fn new(infos: Vec) -> Self { let mut states = HashMap::new(); for info in infos { states.insert(info.host.clone(), AuthState { info: info, stage: AuthStage::Idle, }); } return Auth { states } } } impl AuthState{ fn add_to_request(&self, req: ureq::RequestBuilder) -> ureq::RequestBuilder { if let AuthStage::Authorized(x) = &self.stage { return req.header("Authorization", x); } return req } fn authenticate(&mut self, response: &ureq::http::Response) -> Result<(), String> { match self.stage { AuthStage::Authorized(_) => // The token must have expired self.stage = AuthStage::Idle, AuthStage::Idle => {}, } let auth_header = response.headers().get("www-authenticate").unwrap(); match AuthMethod::from_header(auth_header) { Some(AuthMethod::Basic) => { let basic_auth = format!("Basic {}", self.info.auth); self.stage = AuthStage::Authorized(basic_auth); return Ok(()); }, Some(AuthMethod::Bearer{realm, scope, service}) => { let basic_auth = format!("Basic {}", self.info.auth); let url = format!("{}?service={}&scope={}", realm, service, scope); let body = ureq::get(url) .config().http_status_as_error(false).build() .header("Authorization", basic_auth) .call(); let body = body.map_err(|x| format!("Server {} authentication request failed: {}", self.info.host, x.to_string()))? .body_mut().read_to_string().expect(format!("Server {} responded with something non-string like", self.info.host).as_str()); let body = body.parse::() .unwrap(); let token = body["token"].get::().unwrap(); self.stage = AuthStage::Authorized(format!("Bearer {}", token)); return Ok(()); }, None => return Err(format!("Server {} provided us with a challenge, but we didn't understand it", self.info.host)), } } } #[derive(Debug)] enum RegistryError { NotFound, Other(String), } fn perform_registry_request(registry: &str, url: &str, accept: &'static str, auth: &mut Auth) -> Result, RegistryError> { let mut state = auth.states.get_mut(registry); let url = format!("https://{}{}", registry, &url); let mut authentication_retry = false; loop { let mut request = ureq::get(&url) .header("Accept", accept) .config().http_status_as_error(false).build(); if let Some(ref state) = state { request = state.add_to_request(request); } let start_time = std::time::Instant::now(); let response = request.call().unwrap(); let duration = start_time.elapsed().as_secs_f64(); metrics::get().http_request_duration.with_label_values(&[registry]).observe(duration); if response.status() == 401 { if !authentication_retry { if let Some(ref mut state) = state { match state.authenticate(&response) { Ok(()) => { metrics::get().auth_attempts.with_label_values(&[registry, "success"]).inc(); authentication_retry = true; continue; } Err(e) => { metrics::get().auth_attempts.with_label_values(&[registry, "failure"]).inc(); return Err(RegistryError::Other(e)); } } } else { metrics::get().http_request.with_label_values(&[registry, "error"]).inc(); return Err(RegistryError::Other(format!("Server {} returned 401 but we have no credentials", registry))); } } else { metrics::get().auth_attempts.with_label_values(&[registry, "failure"]).inc(); metrics::get().http_request.with_label_values(&[registry, "error"]).inc(); return Err(RegistryError::Other(format!("Authentication failed"))); } } if response.status() == 429 { metrics::get().rate_limits.with_label_values(&[registry]).inc(); metrics::get().state.set(metrics::STATE_THROTTLED); println!("Too many requests"); std::thread::sleep(std::time::Duration::from_secs(8)); metrics::get().state.set(metrics::STATE_ACTIVE); continue; } if response.status() == 404 { metrics::get().http_request.with_label_values(&[registry, "not_found"]).inc(); return Err(RegistryError::NotFound); } if response.status() != 200 { metrics::get().http_request.with_label_values(&[registry, "error"]).inc(); return Err(RegistryError::Other(format!("Unexpected status code: {}", response.status()))); } metrics::get().http_request.with_label_values(&[registry, "success"]).inc(); return Ok(response); } } struct Repository { url: String, key: Option, host: Option, dest: std::path::PathBuf, } #[derive(Debug)] struct FilePatch { position: Range, content: String, } fn update_images(db: &Connection, file: &str, auth: &mut Auth, img: DockerRef, edits: &mut Vec) -> Result<(), String> { let registry = img.registry.map(|x| &file[x]).unwrap_or("registry.hub.docker.com"); let mut tag = img.tag.as_ref().map(|x| file[x.clone()].to_string()); let image = &file[img.image.clone()]; metrics::get().images_checked.with_label_values(&[registry]).inc(); let now = chrono::offset::Utc::now(); let cache_max_age = chrono::Duration::hours(24); let query_result = { let _timer = metrics::get().db_query_duration.start_timer(); db.query_one(" SELECT id, last_checked FROM images WHERE registry = ?1 AND image = ?2 ", (registry, image), |row| Ok(( row.get::<_, i64>(0)?, row.get::<_, chrono::DateTime>(1)?, ))).optional() }; let (image_id, last_checked) = match query_result { Ok(Some(x)) => x, Ok(None) => { { let _timer = metrics::get().db_query_duration.start_timer(); db.execute(" INSERT INTO images(registry, image, last_checked) VALUES (?1, ?2, ?3) ", (registry, &image, now)).unwrap(); } (db.last_insert_rowid(), now) }, Err(_x) => return Err("Database Failure".to_string()), }; let cache_stale = now - last_checked > cache_max_age; let cached_tags: Vec = { let _timer = metrics::get().db_query_duration.start_timer(); let mut stmt = db.prepare("SELECT tag FROM tags WHERE image_id = ?1 ORDER BY tag").unwrap(); let mut rows = stmt.query((image_id,)).unwrap(); let mut tags = vec![]; while let Some(row) = rows.next().unwrap() { tags.push(row.get(0).unwrap()); } tags }; let (tags, cached_digest): (Vec, Option) = if cache_stale { let mut url = format!("/v2/{}/tags/list", &image); let mut fetched_tags = vec![]; loop { let mut response = match perform_registry_request(registry, &url, "application/vnd.oci.image.index.v1+json", auth) { Ok(r) => r, Err(RegistryError::NotFound) => { println!("Warning: image not found, skipping: {}", image); return Ok(()); }, Err(RegistryError::Other(msg)) => return Err(msg), }; let content_type = response.headers()["Content-Type"].to_str().unwrap(); if !content_type.starts_with("application/json") { return Err(format!("Unexpected Content-Type: {}", content_type)); } let body = response.body_mut().read_to_string().unwrap(); let body = body.parse::().unwrap(); for it in body["tags"].get::>().unwrap().iter() { fetched_tags.push(it.get::().unwrap().clone()); } if let Some(link_header) = response.headers().get("link") { let link_str = link_header.to_str().unwrap(); let link = extract_next_page(&link_str).unwrap(); url = link_str[link.next_uri.unwrap().clone()].to_string(); } else { break; } } fetched_tags.sort(); let mut existing = cached_tags.iter().peekable(); for t in &fetched_tags { while existing.peek().is_some_and(|e| *e < t) { let gone = existing.next().unwrap(); let _timer = metrics::get().db_query_duration.start_timer(); db.execute("DELETE FROM tags WHERE image_id = ?1 AND tag = ?2", (image_id, gone)).unwrap(); } if existing.peek() == Some(&t) { existing.next(); } else { let _timer = metrics::get().db_query_duration.start_timer(); db.execute("INSERT INTO tags(image_id, tag, fetched_at) VALUES (?1, ?2, ?3)", (image_id, t, now)).unwrap(); } } for gone in existing { let _timer = metrics::get().db_query_duration.start_timer(); db.execute("DELETE FROM tags WHERE image_id = ?1 AND tag = ?2", (image_id, gone)).unwrap(); } { let _timer = metrics::get().db_query_duration.start_timer(); db.execute("UPDATE images SET last_checked = ?1 WHERE id = ?2", (now, image_id)).unwrap(); } (fetched_tags, None) } else { let tag_for_digest = tag.as_deref().unwrap_or("latest"); let digest = { let _timer = metrics::get().db_query_duration.start_timer(); db.query_one(" SELECT digest FROM tags WHERE image_id = ?1 AND tag = ?2 AND digest IS NOT NULL ", (image_id, tag_for_digest), |row| row.get::<_, String>(0)).optional().unwrap() }; (cached_tags, digest) }; if let Some(ref tag_str) = tag { let mut current = VersionPattern::parse(&tag_str); let mut new_tag = None; for candidate_str in &tags { let candidate = VersionPattern::parse(&candidate_str); match current.compare(&candidate) { CompareOutcome::Higher => { new_tag = Some(candidate_str.clone()); current = candidate; }, CompareOutcome::Lower => {}, CompareOutcome::Incompatible => {}, CompareOutcome::Identical => {}, } } if let Some(new_tag) = new_tag { tag = Some(new_tag.clone()); edits.push(FilePatch { position: img.tag.unwrap(), content: new_tag, }); metrics::get().images_updated.with_label_values(&[registry]).inc(); } } if let Some(ref digest) = img.digest { let tag_for_digest = tag.as_deref().unwrap_or("latest"); let digest_string = match cached_digest { Some(d) => d, None => { let url = format!("/v2/{}/manifests/{}", &file[img.image.clone()], tag_for_digest); let response = match perform_registry_request(registry, &url, "application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json", auth) { Ok(r) => r, Err(RegistryError::NotFound) => { println!("Warning: image not found, skipping: {}", image); return Ok(()); }, Err(RegistryError::Other(msg)) => return Err(msg), }; let fetched = response.headers()["docker-content-digest"].to_str().unwrap().to_string(); { let _timer = metrics::get().db_query_duration.start_timer(); db.execute("UPDATE tags SET digest = ?1, fetched_at = ?2 WHERE image_id = ?3 AND tag = ?4", (&fetched, now, image_id, tag_for_digest)).unwrap(); } fetched } }; if file[digest.clone()] != digest_string { edits.push(FilePatch { position: img.digest.unwrap(), content: digest_string, }); metrics::get().images_updated.with_label_values(&[registry]).inc(); } } return Ok(()); } enum Output { Overlay(std::fs::File, std::path::PathBuf, std::path::PathBuf), Stdout(std::io::Stdout), } impl Output { fn overlay_file(infile_path: &std::path::Path) -> Self { loop { let mut filename = std::ffi::OsString::new(); filename.push(infile_path.file_name().unwrap()); filename.push(std::ffi::OsStr::new(".edit")); filename.push(Alphanumeric.sample_string(&mut rand::rng(), 16)); let outfile_path = Some(infile_path.parent().unwrap().join(filename)); if let Ok(output_file) = std::fs::File::create_new(outfile_path.as_ref().unwrap()) { return Output::Overlay(output_file, outfile_path.unwrap().into(), infile_path.into()); } } } fn commit(&mut self) { match self { Output::Overlay(_, outfile_path, infile_path) => { std::fs::remove_file(&infile_path).unwrap(); std::fs::rename(&outfile_path, &infile_path).unwrap(); } Output::Stdout(_) => {} } } } impl std::ops::Deref for Output { type Target = dyn Write; fn deref(&self) -> &Self::Target { match self { Output::Overlay(file, _, _) => file, Output::Stdout(stdout) => stdout, } } } impl std::ops::DerefMut for Output { fn deref_mut(&mut self) -> &mut Self::Target { match self { Output::Overlay(file, _, _) => file, Output::Stdout(stdout) => stdout, } } } impl std::io::Write for Output { fn write(&mut self, buf: &[u8]) -> std::io::Result { return (**self).write(buf); } fn flush(&mut self) -> std::io::Result<()> { return (**self).flush(); } fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result { return (**self).write_vectored(bufs); } fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> { return (**self).write_all(buf); } fn write_fmt(&mut self, args: std::fmt::Arguments<'_>) -> std::io::Result<()> { return (**self).write_fmt(args); } } fn is_dockerfile(path: &std::path::Path) -> bool { if let Some(name) = path.file_name() { if let Some(name) = name.to_str() { return name == "Dockerfile" || name == "dockerfile"; } } return false; } fn is_yaml(path: &std::path::Path) -> bool { if let Some(ext) = path.extension() { if let Some(ext) = ext.to_str() { return ext == "yaml"; } } return false; } fn run_tool(db: &Connection, auth: &mut Auth, repos: &Vec, mut infile_paths: Vec, overwrite: bool) { for repo in repos { if repo.dest.exists() { let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("fetch") .arg("origin"); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("reset") .arg("--hard") .arg("@{u}"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("clean") .arg("--force") .arg("-d") .arg("-x"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } } else { let mut gitcmd = std::process::Command::new("git"); println!("Cloning {}", repo.url); gitcmd .arg("clone") .arg(&repo.url) .arg(&repo.dest); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { panic!("Git exited with failure"); } } infile_paths.push(repo.dest.clone()); } let mut inpaths = vec![]; for infile_path in infile_paths { if infile_path.is_dir() { let mut unsearched = vec![infile_path.clone()]; while let Some(next) = unsearched.pop() { for child in next.read_dir().unwrap() { let child = child.unwrap(); let path = child.path(); let ft = child.file_type().unwrap(); if ft.is_dir() { unsearched.push(path); continue; } if is_yaml(&path) || is_dockerfile(&path) { inpaths.push(path); } } } } else { inpaths.push(infile_path); }; } for infile_path in inpaths { let mut file = std::fs::File::open(&infile_path).unwrap(); let mut file_content = String::new(); file.read_to_string(&mut file_content).unwrap(); let images: Vec> = if is_dockerfile(&infile_path) { DockerfileFile::parse(&file_content).image_refs } else { ManifestFile::parse(&file_content).image_tags }; let mut failed = None; let mut edits = vec![]; for ref image in images { println!("Checking image {}", &file_content[image.clone()]); let image_ref = DockerRef::parse(&file_content, image); if let Err(msg) = update_images(db, &file_content, auth, image_ref, &mut edits) { failed = Some(msg); break; } } if let Some(msg) = failed { println!("{}: {} ", infile_path.display(), msg); continue; } let mut out = if overwrite { Output::overlay_file(&infile_path) } else { Output::Stdout(std::io::stdout()) }; let mut current_position = 0; file.seek(std::io::SeekFrom::Start(0)).unwrap(); let mut file_block = file.take(0); for edit in edits { if edit.position.start > current_position { file_block.set_limit((edit.position.start - current_position) as u64); std::io::copy(&mut file_block, &mut out).unwrap(); } out.write_all(edit.content.as_bytes()).unwrap(); // We should have been able to just seek to the position.end here, but that doesn't work // for whatever reason. What we can do is calculate the amount to skip ahead, and then do // that. let skip = (edit.position.end - edit.position.start) as i64; file_block.set_limit(skip as u64); file_block.seek(std::io::SeekFrom::Current(skip)).unwrap(); current_position = edit.position.end; } file_block.set_limit(u64::MAX); std::io::copy(&mut file_block, &mut out).unwrap(); out.commit(); } for repo in repos { let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-c") .arg("user.name=vbump") .arg("-c") .arg("user.email=jesper@jnsn.dev") .arg("-C") .arg(&repo.dest) .arg("commit") .arg("--all") .arg("--message") .arg("Update versions"); let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { metrics::get().git_operations.with_label_values(&["commit", "failure"]).inc(); println!("Commit failed, presumably there were no changes"); continue; } metrics::get().git_operations.with_label_values(&["commit", "success"]).inc(); let mut gitcmd = std::process::Command::new("git"); gitcmd .arg("-C") .arg(&repo.dest) .arg("push") .arg("origin") .arg("+HEAD:version-bump"); if let Some(key) = &repo.key && let Some(host) = &repo.host { gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -o UserKnownHostsFile=\"{}\" -i \"{}\"", host.to_str().unwrap(), key.to_str().unwrap())); } let exit = gitcmd.status() .expect("Git command failed"); if !exit.success() { metrics::get().git_operations.with_label_values(&["push", "failure"]).inc(); panic!("Git exited with failure"); } metrics::get().git_operations.with_label_values(&["push", "success"]).inc(); } } fn main() { let argv: Vec = std::env::args().collect(); let mut it = argv.iter(); let cmd = &it.next().unwrap(); let mut auths = vec![]; let mut positional: Vec<&str> = vec!(); let mut overwrite = false; let mut config_path = None; let mut scratch_path = None; let mut continuous = false; let mut metrics_port: Option = None; loop { match it.next().map(|x| x.as_str()) { None => break, Some("--auth") => { if let Some(registry) = it.next() && let Some(username) = it.next() && let Some(password) = it.next() { auths.push(AuthInfo { host: registry.clone(), auth: basic_auth(username, password), }); } else { println!("Error: --auth requires three parameters"); help(cmd); std::process::exit(1); } }, Some("--config") => { if let Some(path) = it.next() { config_path = Some(std::path::PathBuf::from(path)); } else { println!("Error: --config requires an parameters"); help(cmd); std::process::exit(1); } }, Some("--scratch") => { if let Some(path) = it.next() { scratch_path = Some(std::path::PathBuf::from(path)); } else { println!("Error: --scratch requires an parameters"); help(cmd); std::process::exit(1); } }, Some("--continuous") => { continuous = true; }, Some("--metrics-port") => { if let Some(port_str) = it.next() { match port_str.parse::() { Ok(port) => metrics_port = Some(port), Err(_) => { println!("Error: --metrics-port requires a valid port number"); help(cmd); std::process::exit(1); } } } else { println!("Error: --metrics-port requires a parameter"); help(cmd); std::process::exit(1); } }, Some("-i") | Some("--inplace") => { overwrite = true; }, Some("-h") | Some("--help") => { help(cmd); std::process::exit(0); }, Some(arg) => positional.push(arg), }; } let mut infile_paths = vec![]; if positional.len() < 1 { panic!("BAD ARGUMENTS"); } let db = Connection::open(positional[0]).unwrap(); if positional.len() > 1 { infile_paths.push(std::path::PathBuf::from(positional[1])); } { db.execute(" CREATE TABLE IF NOT EXISTS migrations ( id INTEGER PRIMARY KEY NOT NULL ) ", ()).unwrap(); let newest_migration: u32 = db.query_row(" SELECT MAX(id) FROM migrations ", [], |row| row.get::<_, Option>(0)).unwrap().unwrap_or(0); if newest_migration < 1 { db.execute("INSERT INTO migrations(id) VALUES (?1)", (1, )).unwrap(); } if newest_migration < 2 { db.execute(" CREATE TABLE known_images ( id INTEGER PRIMARY KEY NOT NULL, registry TEXT NOT NULL, image TEXT NOT NULL, tag TEXT NOT NULL ) ", ()).unwrap(); db.execute("INSERT INTO migrations(id) VALUES (?1)", (2, )).unwrap(); } if newest_migration < 3 { db.execute(" ALTER TABLE known_images ADD COLUMN discovered DATETIME NOT NULL ", ()).unwrap(); db.execute("INSERT INTO migrations(id) VALUES (?1)", (3, )).unwrap(); } if newest_migration < 4 { // db.execute(" // CREATE UNIQUE INDEX known_images__registry_image_tag // ON known_images(registry, image, tag) // ", ()).unwrap(); db.execute("INSERT INTO migrations(id) VALUES (?1)", (4, )).unwrap(); } if newest_migration < 5 { db.execute(" CREATE TABLE images ( id INTEGER PRIMARY KEY NOT NULL, registry TEXT NOT NULL, image TEXT NOT NULL, last_checked DATETIME NOT NULL ) ", ()).unwrap(); db.execute(" CREATE UNIQUE INDEX images__registry_image ON images(registry, image) ", ()).unwrap(); db.execute("INSERT INTO migrations(id) VALUES (?1)", (5, )).unwrap(); } if newest_migration < 6 { db.execute(" CREATE TABLE tags ( id INTEGER PRIMARY KEY, image_id INTEGER NOT NULL REFERENCES images(id), tag TEXT NOT NULL, digest TEXT, fetched_at DATETIME NOT NULL ) ", ()).unwrap(); db.execute(" CREATE UNIQUE INDEX tags__image_id_tag ON tags(image_id, tag) ", ()).unwrap(); db.execute("INSERT INTO migrations(id) VALUES (?1)", (6, )).unwrap(); } } let mut repos = vec![]; let workdir = std::env::current_dir().unwrap(); if let Some(config_path) = config_path { if !config_path.is_dir() { println!("config is not a directory"); std::process::exit(1); } let mut unsearched = vec![config_path]; while let Some(next) = unsearched.pop() { for child in next.read_dir().unwrap() { let child = child.unwrap(); let path = child.path(); let ft = child.file_type().unwrap(); if ft.is_dir() { unsearched.push(path); continue; } if let Some(name) = path.file_name() { match name.to_str() { Some(".dockerconfigjson") => { let file = std::fs::File::open(&path).unwrap(); let mut reader = json_event_parser::ReaderJsonParser::new(file); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "auths" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } let mut possible_key = reader.parse_next(); while let Ok(json_event_parser::JsonEvent::ObjectKey(k)) = possible_key { let key = k.into_owned(); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "auth" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { auths.push(AuthInfo { host: key, auth: v.into_owned(), }); }, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } possible_key = reader.parse_next(); } match possible_key { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } } Some("repository") => { let file = std::fs::File::open(&path).unwrap(); let mut reader = json_event_parser::ReaderJsonParser::new(file); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "repositories" => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } let mut possible_key = reader.parse_next(); while let Ok(json_event_parser::JsonEvent::ObjectKey(k)) = possible_key { let k = k.into_owned(); match reader.parse_next() { Ok(json_event_parser::JsonEvent::StartObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "key" => {}, _ => panic!("Invalid config json"), } let key_name = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "host" => {}, _ => panic!("Invalid config json"), } let host_name = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::ObjectKey(k)) if &k == "dest" => {}, _ => panic!("Invalid config json"), } let dest = match reader.parse_next() { Ok(json_event_parser::JsonEvent::String(v)) => { v.into_owned() }, _ => panic!("Invalid config json"), }; match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } repos.push(Repository { url: k, key: Some(workdir.join(path.parent().unwrap().join(std::path::PathBuf::from(key_name)))), host: Some(workdir.join(path.parent().unwrap().join(std::path::PathBuf::from(host_name)))), dest: scratch_path.as_ref().unwrap().join(dest), }); possible_key = reader.parse_next(); } match possible_key { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } match reader.parse_next() { Ok(json_event_parser::JsonEvent::EndObject) => {}, _ => panic!("Invalid config json"), } } Some(_) | None => {} } } } } } let mut auth = Auth::new(auths); metrics::init(); metrics::get().state.set(metrics::STATE_IDLE); if let Some(port) = metrics_port { metrics::serve(port); } loop { metrics::get().state.set(metrics::STATE_ACTIVE); let run_start = std::time::Instant::now(); run_tool(&db, &mut auth, &repos, infile_paths.clone(), overwrite); let run_duration = run_start.elapsed().as_secs_f64(); metrics::get().run_duration.set(run_duration); metrics::get().state.set(metrics::STATE_IDLE); if !continuous { break; } println!("Waiting for next run"); std::thread::sleep(std::time::Duration::from_hours(24)); } }