diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/main.rs | 468 |
1 files changed, 273 insertions, 195 deletions
diff --git a/src/main.rs b/src/main.rs index a73421c..3fed758 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,8 @@ use crate::parser::*; use base64::prelude::*; use rand::distr::{Alphanumeric, SampleString}; +use rusqlite::Connection; +use rusqlite::OptionalExtension; use yaml_rust2::parser::Parser; use yaml_rust2::Event; use std::collections::HashMap; @@ -147,7 +149,9 @@ fn perform_registry_request(registry: &str, url: &str, accept: &'static str, aut let url = format!("https://{}{}", registry, &url); - for _ in 0..2 { + let mut authentication_retry = false; + + loop { let mut request = ureq::get(&url) .header("Accept", accept) .config().http_status_as_error(false).build(); @@ -158,14 +162,17 @@ fn perform_registry_request(registry: &str, url: &str, accept: &'static str, aut let response = request.call().unwrap(); - if response.status() == 401 { + if response.status() == 401 && !authentication_retry { if let Some(ref mut state) = state { state.authenticate(&response)?; + authentication_retry = true; continue; } else { return Err(format!("Server {} returned 401 but we have no credentials", registry)); } - } else if response.status() != 200 { + } + + if response.status() != 200 { return Err(format!("Unexpected status code: {}", response.status())) } @@ -373,7 +380,7 @@ impl VersionPattern { } } -fn update_images(file: &str, auth: &mut Auth, img: DockerRef, edits: &mut Vec<FilePatch>) -> Result<(), String> { +fn update_images(db: &Connection, file: &str, auth: &mut Auth, img: DockerRef, edits: &mut Vec<FilePatch>) -> Result<(), String> { let registry = img.registry.map(|x| &file[x]).unwrap_or("registry.hub.docker.com"); let mut tag = img.tag.as_ref().map(|x| file[x.clone()].to_string()); @@ -396,6 +403,11 @@ fn update_images(file: &str, auth: &mut Auth, img: DockerRef, edits: &mut Vec<Fi for it in body["tags"].get::<Vec<tinyjson::JsonValue>>().unwrap().iter() { let candidate_str = it.get::<String>().unwrap(); + + db.execute(" + INSERT INTO known_images(registry, image, tag, discovered) VALUES (?1, ?2, ?3, ?4) + ", (registry, &file[img.image.clone()], candidate_str, chrono::offset::Utc::now())).unwrap(); + let candidate = VersionPattern::parse(candidate_str); match current.compare(&candidate) { CompareOutcome::Higher => { @@ -516,6 +528,200 @@ impl std::io::Write for Output { } } +fn run_tool(db: &Connection, auth: &mut Auth, repos: &Vec<Repository>, mut infile_paths: Vec<std::path::PathBuf>, overwrite: bool) { + for repo in repos { + if repo.dest.exists() { + let mut gitcmd = std::process::Command::new("git"); + + gitcmd + .arg("-C") + .arg(&repo.dest) + .arg("fetch") + .arg("origin"); + + if let Some(key) = &repo.key { + gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap())); + } + let exit = gitcmd.status() + .expect("Git command failed"); + if !exit.success() { + panic!("Git exited with failure"); + } + + let mut gitcmd = std::process::Command::new("git"); + + gitcmd + .arg("-C") + .arg(&repo.dest) + .arg("reset") + .arg("--hard") + .arg("@{u}"); + + let exit = gitcmd.status() + .expect("Git command failed"); + if !exit.success() { + panic!("Git exited with failure"); + } + + let mut gitcmd = std::process::Command::new("git"); + + gitcmd + .arg("-C") + .arg(&repo.dest) + .arg("clean") + .arg("--force") + .arg("-d") + .arg("-x"); + + let exit = gitcmd.status() + .expect("Git command failed"); + if !exit.success() { + panic!("Git exited with failure"); + } + } else { + let mut gitcmd = std::process::Command::new("git"); + + gitcmd + .arg("clone") + .arg(&repo.url) + .arg(&repo.dest); + + if let Some(key) = &repo.key { + gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap())); + } + + let exit = gitcmd.status() + .expect("Git command failed"); + if !exit.success() { + panic!("Git exited with failure"); + } + } + + infile_paths.push(repo.dest.clone()); + } + + let mut inpaths = vec!(); + for infile_path in infile_paths { + if infile_path.is_dir() { + let mut unsearched = vec![infile_path.clone()]; + + while let Some(next) = unsearched.pop() { + for child in next.read_dir().unwrap() { + let child = child.unwrap(); + let path = child.path(); + + let ft = child.file_type().unwrap(); + if ft.is_dir() { + unsearched.push(path); + continue; + } + + if let Some(ext) = path.extension() { + if ext == "yaml" { + inpaths.push(path); + } + } + } + } + } else { + inpaths.push(infile_path); + }; + } + + for infile_path in inpaths { + let mut file = std::fs::File::open(&infile_path).unwrap(); + let mut file_content = String::new(); + file.read_to_string(&mut file_content).unwrap(); + + let images = ManifestFile::parse(&file_content); + + let mut failed = None; + + let mut edits = vec![]; + for ref image in images.image_tags { + println!("Checking image {}", &file_content[image.clone()]); + let image_ref = DockerRef::parse(&file_content, image); + if let Err(msg) = update_images(db, &file_content, auth, image_ref, &mut edits) { + failed = Some(msg); + break; + } + } + + if let Some(msg) = failed { + println!("{}: {} ", infile_path.display(), msg); + continue; + } + + let mut out = if overwrite { + Output::overlay_file(&infile_path) + } else { + Output::Stdout(std::io::stdout()) + }; + + let mut current_position = 0; + file.seek(std::io::SeekFrom::Start(0)).unwrap(); + let mut file_block = file.take(0); + for edit in edits { + if edit.position.start > current_position { + file_block.set_limit((edit.position.start - current_position) as u64); + std::io::copy(&mut file_block, &mut out).unwrap(); + } + + out.write_all(edit.content.as_bytes()).unwrap(); + + // We should have been able to just seek to the position.end here, but that doesn't work + // for whatever reason. What we can do is calculate the amount to skip ahead, and then do + // that. + let skip = (edit.position.end - edit.position.start) as i64; + file_block.set_limit(skip as u64); + file_block.seek(std::io::SeekFrom::Current(skip)).unwrap(); + current_position = edit.position.end; + } + + file_block.set_limit(u64::MAX); + std::io::copy(&mut file_block, &mut out).unwrap(); + + out.commit(); + } + + for repo in repos { + let mut gitcmd = std::process::Command::new("git"); + + gitcmd + .arg("-C") + .arg(&repo.dest) + .arg("commit") + .arg("--all") + .arg("--message") + .arg("Update versions"); + + let exit = gitcmd.status() + .expect("Git command failed"); + if !exit.success() { + println!("Commit failed, presumably there were no changes"); + continue; + } + + let mut gitcmd = std::process::Command::new("git"); + gitcmd + .arg("-C") + .arg(&repo.dest) + .arg("push") + .arg("origin") + .arg("+HEAD:version-bump"); + + if let Some(key) = &repo.key { + gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap())); + } + + let exit = gitcmd.status() + .expect("Git command failed"); + if !exit.success() { + panic!("Git exited with failure"); + } + } +} + fn main() { let argv: Vec<String> = std::env::args().collect(); let mut it = argv.iter(); @@ -526,6 +732,7 @@ fn main() { let mut overwrite = false; let mut config_path = None; let mut scratch_path = None; + let mut continuous = false; loop { match it.next().map(|x| x.as_str()) { @@ -560,6 +767,9 @@ fn main() { std::process::exit(1); } }, + Some("--continuous") => { + continuous = true; + }, Some("-i") | Some("--inplace") => { overwrite = true; }, @@ -572,8 +782,60 @@ fn main() { } let mut infile_paths = vec![]; - if positional.len() == 1 { - infile_paths.push(std::path::PathBuf::from(positional[0])); + if positional.len() < 1 { + panic!("BAD ARGUMENTS"); + } + + let db = Connection::open(positional[0]).unwrap(); + if positional.len() > 1 { + infile_paths.push(std::path::PathBuf::from(positional[1])); + } + + { + db.execute(" + CREATE TABLE IF NOT EXISTS migrations ( + id INTEGER PRIMARY KEY NOT NULL + ) + ", ()).unwrap(); + + let newest_migration: u32 = db.query_row(" + SELECT MAX(id) FROM migrations + ", [], |row| row.get::<_, Option<u32>>(0)).unwrap().unwrap_or(0); + + if newest_migration < 1 { + db.execute("INSERT INTO migrations(id) VALUES (?1)", (1, )).unwrap(); + } + + if newest_migration < 2 { + db.execute(" + CREATE TABLE known_images ( + id INTEGER PRIMARY KEY NOT NULL, + registry TEXT NOT NULL, + image TEXT NOT NULL, + tag TEXT NOT NULL + ) + ", ()).unwrap(); + + db.execute("INSERT INTO migrations(id) VALUES (?1)", (2, )).unwrap(); + } + + if newest_migration < 3 { + db.execute(" + ALTER TABLE known_images ADD COLUMN + discovered DATETIME NOT NULL + ", ()).unwrap(); + + db.execute("INSERT INTO migrations(id) VALUES (?1)", (3, )).unwrap(); + } + + if newest_migration < 4 { + db.execute(" + CREATE UNIQUE INDEX known_images__registry_image_tag + ON known_images(registry, image, tag) + ", ()).unwrap(); + + db.execute("INSERT INTO migrations(id) VALUES (?1)", (4, )).unwrap(); + } } let mut repos = vec![]; @@ -742,197 +1004,13 @@ fn main() { } } - for repo in &repos { - if repo.dest.exists() { - let mut gitcmd = std::process::Command::new("git"); - - gitcmd - .arg("-C") - .arg(&repo.dest) - .arg("fetch") - .arg("origin"); - - if let Some(key) = &repo.key { - gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap())); - } - let exit = gitcmd.status() - .expect("Git command failed"); - if !exit.success() { - panic!("Git exited with failure"); - } - - let mut gitcmd = std::process::Command::new("git"); - - gitcmd - .arg("-C") - .arg(&repo.dest) - .arg("reset") - .arg("--hard") - .arg("@{u}"); - - let exit = gitcmd.status() - .expect("Git command failed"); - if !exit.success() { - panic!("Git exited with failure"); - } - - let mut gitcmd = std::process::Command::new("git"); - - gitcmd - .arg("-C") - .arg(&repo.dest) - .arg("clean") - .arg("--force") - .arg("-d") - .arg("-x"); - - let exit = gitcmd.status() - .expect("Git command failed"); - if !exit.success() { - panic!("Git exited with failure"); - } - } else { - let mut gitcmd = std::process::Command::new("git"); - - gitcmd - .arg("clone") - .arg(&repo.url) - .arg(&repo.dest); - - if let Some(key) = &repo.key { - gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap())); - } - - let exit = gitcmd.status() - .expect("Git command failed"); - if !exit.success() { - panic!("Git exited with failure"); - } - } - - infile_paths.push(repo.dest.clone()); - } - - let mut inpaths = vec!(); - for infile_path in infile_paths { - if infile_path.is_dir() { - let mut unsearched = vec![infile_path.clone()]; - - while let Some(next) = unsearched.pop() { - for child in next.read_dir().unwrap() { - let child = child.unwrap(); - let path = child.path(); - - let ft = child.file_type().unwrap(); - if ft.is_dir() { - unsearched.push(path); - continue; - } - - if let Some(ext) = path.extension() { - if ext == "yaml" { - inpaths.push(path); - } - } - } - } - } else { - inpaths.push(infile_path); - }; - } - let mut auth = Auth::new(auths); - for infile_path in inpaths { - let mut file = std::fs::File::open(&infile_path).unwrap(); - let mut file_content = String::new(); - file.read_to_string(&mut file_content).unwrap(); - - let images = ManifestFile::parse(&file_content); - - let mut failed = None; - - let mut edits = vec![]; - for ref image in images.image_tags { - println!("Checking image {}", &file_content[image.clone()]); - let image_ref = DockerRef::parse(&file_content, image); - if let Err(msg) = update_images(&file_content, &mut auth, image_ref, &mut edits) { - failed = Some(msg); - break; - } - } - - if let Some(msg) = failed { - println!("{}: {} ", infile_path.display(), msg); - continue; - } - - let mut out = if overwrite { - Output::overlay_file(&infile_path) - } else { - Output::Stdout(std::io::stdout()) - }; - - let mut current_position = 0; - file.seek(std::io::SeekFrom::Start(0)).unwrap(); - let mut file_block = file.take(0); - for edit in edits { - if edit.position.start > current_position { - file_block.set_limit((edit.position.start - current_position) as u64); - std::io::copy(&mut file_block, &mut out).unwrap(); - } - - out.write_all(edit.content.as_bytes()).unwrap(); - - // We should have been able to just seek to the position.end here, but that doesn't work - // for whatever reason. What we can do is calculate the amount to skip ahead, and then do - // that. - let skip = (edit.position.end - edit.position.start) as i64; - file_block.set_limit(skip as u64); - file_block.seek(std::io::SeekFrom::Current(skip)).unwrap(); - current_position = edit.position.end; - } - - file_block.set_limit(u64::MAX); - std::io::copy(&mut file_block, &mut out).unwrap(); - - out.commit(); - } - - for repo in &repos { - let mut gitcmd = std::process::Command::new("git"); - - gitcmd - .arg("-C") - .arg(&repo.dest) - .arg("commit") - .arg("--all") - .arg("--message") - .arg("Update versions"); - - let exit = gitcmd.status() - .expect("Git command failed"); - if !exit.success() { - println!("Commit failed, presumably there were no changes"); - continue; - } - - let mut gitcmd = std::process::Command::new("git"); - gitcmd - .arg("-C") - .arg(&repo.dest) - .arg("push") - .arg("origin") - .arg("+HEAD:version-bump"); - - if let Some(key) = &repo.key { - gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap())); - } + loop { + run_tool(&db, &mut auth, &repos, infile_paths.clone(), overwrite); + if !continuous { break; } - let exit = gitcmd.status() - .expect("Git command failed"); - if !exit.success() { - panic!("Git exited with failure"); - } + println!("Waiting for next run"); + std::thread::sleep(std::time::Duration::from_hours(24)); } } |