Make it a server

1 files changed, 273 insertions, 195 deletions

diff --git a/src/main.rs b/src/main.rs
index a73421c..3fed758 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -3,6 +3,8 @@ use crate::parser::*;
 
 use base64::prelude::*;
 use rand::distr::{Alphanumeric, SampleString};
+use rusqlite::Connection;
+use rusqlite::OptionalExtension;
 use yaml_rust2::parser::Parser;
 use yaml_rust2::Event;
 use std::collections::HashMap;
@@ -147,7 +149,9 @@ fn perform_registry_request(registry: &str, url: &str, accept: &'static str, aut
 
     let url = format!("https://{}{}", registry, &url);
 
-    for _ in 0..2 {
+    let mut authentication_retry = false;
+
+    loop {
         let mut request = ureq::get(&url)
             .header("Accept", accept)
             .config().http_status_as_error(false).build();
@@ -158,14 +162,17 @@ fn perform_registry_request(registry: &str, url: &str, accept: &'static str, aut
 
         let response = request.call().unwrap();
 
-        if response.status() == 401 {
+        if response.status() == 401 && !authentication_retry {
             if let Some(ref mut state) = state {
                 state.authenticate(&response)?;
+                authentication_retry = true;
                 continue;
             } else {
                 return Err(format!("Server {} returned 401 but we have no credentials", registry));
             }
-        } else if response.status() != 200 {
+        }
+
+        if response.status() != 200 {
             return Err(format!("Unexpected status code: {}", response.status()))
         }
 
@@ -373,7 +380,7 @@ impl VersionPattern {
     }
 }
 
-fn update_images(file: &str, auth: &mut Auth, img: DockerRef, edits: &mut Vec<FilePatch>) -> Result<(), String> {
+fn update_images(db: &Connection, file: &str, auth: &mut Auth, img: DockerRef, edits: &mut Vec<FilePatch>) -> Result<(), String> {
     let registry = img.registry.map(|x| &file[x]).unwrap_or("registry.hub.docker.com");
     let mut tag = img.tag.as_ref().map(|x| file[x.clone()].to_string());
 
@@ -396,6 +403,11 @@ fn update_images(file: &str, auth: &mut Auth, img: DockerRef, edits: &mut Vec<Fi
 
             for it in body["tags"].get::<Vec<tinyjson::JsonValue>>().unwrap().iter() {
                 let candidate_str = it.get::<String>().unwrap();
+
+                db.execute("
+                    INSERT INTO known_images(registry, image, tag, discovered) VALUES (?1, ?2, ?3, ?4)
+                ", (registry, &file[img.image.clone()], candidate_str, chrono::offset::Utc::now())).unwrap();
+
                 let candidate = VersionPattern::parse(candidate_str);
                 match current.compare(&candidate) {
                     CompareOutcome::Higher => {
@@ -516,6 +528,200 @@ impl std::io::Write for Output {
     }
 }
 
+fn run_tool(db: &Connection, auth: &mut Auth, repos: &Vec<Repository>, mut infile_paths: Vec<std::path::PathBuf>, overwrite: bool) {
+    for repo in repos {
+        if repo.dest.exists() {
+            let mut gitcmd = std::process::Command::new("git");
+
+            gitcmd
+                .arg("-C")
+                .arg(&repo.dest)
+                .arg("fetch")
+                .arg("origin");
+
+            if let Some(key) = &repo.key {
+                gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap()));
+            }
+            let exit = gitcmd.status()
+                .expect("Git command failed");
+            if !exit.success() {
+                panic!("Git exited with failure");
+            }
+
+            let mut gitcmd = std::process::Command::new("git");
+
+            gitcmd
+                .arg("-C")
+                .arg(&repo.dest)
+                .arg("reset")
+                .arg("--hard")
+                .arg("@{u}");
+
+            let exit = gitcmd.status()
+                .expect("Git command failed");
+            if !exit.success() {
+                panic!("Git exited with failure");
+            }
+
+            let mut gitcmd = std::process::Command::new("git");
+
+            gitcmd
+                .arg("-C")
+                .arg(&repo.dest)
+                .arg("clean")
+                .arg("--force")
+                .arg("-d")
+                .arg("-x");
+
+            let exit = gitcmd.status()
+                .expect("Git command failed");
+            if !exit.success() {
+                panic!("Git exited with failure");
+            }
+        } else {
+            let mut gitcmd = std::process::Command::new("git");
+
+            gitcmd
+                .arg("clone")
+                .arg(&repo.url)
+                .arg(&repo.dest);
+
+            if let Some(key) = &repo.key {
+                gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap()));
+            }
+
+            let exit = gitcmd.status()
+                .expect("Git command failed");
+            if !exit.success() {
+                panic!("Git exited with failure");
+            }
+        }
+
+        infile_paths.push(repo.dest.clone());
+    }
+
+    let mut inpaths = vec!();
+    for infile_path in infile_paths {
+        if infile_path.is_dir() {
+            let mut unsearched = vec![infile_path.clone()];
+
+            while let Some(next) = unsearched.pop() {
+                for child in next.read_dir().unwrap() {
+                    let child = child.unwrap();
+                    let path = child.path();
+
+                    let ft = child.file_type().unwrap();
+                    if ft.is_dir() {
+                        unsearched.push(path);
+                        continue;
+                    }
+
+                    if let Some(ext) = path.extension() {
+                        if ext == "yaml" {
+                            inpaths.push(path);
+                        }
+                    }
+                }
+            }
+        } else {
+            inpaths.push(infile_path);
+        };
+    }
+
+    for infile_path in inpaths {
+        let mut file = std::fs::File::open(&infile_path).unwrap();
+        let mut file_content = String::new();
+        file.read_to_string(&mut file_content).unwrap();
+
+        let images = ManifestFile::parse(&file_content);
+
+        let mut failed = None;
+
+        let mut edits = vec![];
+        for ref image in images.image_tags {
+            println!("Checking image {}", &file_content[image.clone()]);
+            let image_ref = DockerRef::parse(&file_content, image);
+            if let Err(msg) = update_images(db, &file_content, auth, image_ref, &mut edits) {
+                failed = Some(msg);
+                break;
+            }
+        }
+
+        if let Some(msg) = failed {
+            println!("{}: {} ", infile_path.display(), msg);
+            continue;
+        }
+
+        let mut out = if overwrite {
+            Output::overlay_file(&infile_path)
+        } else {
+            Output::Stdout(std::io::stdout())
+        };
+
+        let mut current_position = 0;
+        file.seek(std::io::SeekFrom::Start(0)).unwrap();
+        let mut file_block = file.take(0);
+        for edit in edits {
+            if edit.position.start > current_position {
+                file_block.set_limit((edit.position.start - current_position) as u64);
+                std::io::copy(&mut file_block, &mut out).unwrap();
+            }
+
+            out.write_all(edit.content.as_bytes()).unwrap();
+
+            // We should have been able to just seek to the position.end here, but that doesn't work
+            // for whatever reason. What we can do is calculate the amount to skip ahead, and then do
+            // that.
+            let skip = (edit.position.end - edit.position.start) as i64;
+            file_block.set_limit(skip as u64);
+            file_block.seek(std::io::SeekFrom::Current(skip)).unwrap();
+            current_position = edit.position.end;
+        }
+
+        file_block.set_limit(u64::MAX);
+        std::io::copy(&mut file_block, &mut out).unwrap();
+
+        out.commit();
+    }
+
+    for repo in repos {
+        let mut gitcmd = std::process::Command::new("git");
+
+        gitcmd
+            .arg("-C")
+            .arg(&repo.dest)
+            .arg("commit")
+            .arg("--all")
+            .arg("--message")
+            .arg("Update versions");
+
+        let exit = gitcmd.status()
+            .expect("Git command failed");
+        if !exit.success() {
+            println!("Commit failed, presumably there were no changes");
+            continue;
+        }
+
+        let mut gitcmd = std::process::Command::new("git");
+        gitcmd
+            .arg("-C")
+            .arg(&repo.dest)
+            .arg("push")
+            .arg("origin")
+            .arg("+HEAD:version-bump");
+
+        if let Some(key) = &repo.key {
+            gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap()));
+        }
+
+        let exit = gitcmd.status()
+            .expect("Git command failed");
+        if !exit.success() {
+            panic!("Git exited with failure");
+        }
+    }
+}
+
 fn main() {
     let argv: Vec<String> = std::env::args().collect();
     let mut it = argv.iter();
@@ -526,6 +732,7 @@ fn main() {
     let mut overwrite = false;
     let mut config_path = None;
     let mut scratch_path = None;
+    let mut continuous = false;
 
     loop {
         match it.next().map(|x| x.as_str()) {
@@ -560,6 +767,9 @@ fn main() {
                     std::process::exit(1);
                 }
             },
+            Some("--continuous") => {
+                continuous = true;
+            },
             Some("-i") | Some("--inplace") => {
                 overwrite = true;
             },
@@ -572,8 +782,60 @@ fn main() {
     }
 
     let mut infile_paths = vec![];
-    if positional.len() == 1 {
-        infile_paths.push(std::path::PathBuf::from(positional[0]));
+    if positional.len() < 1 {
+        panic!("BAD ARGUMENTS");
+    }
+
+    let db = Connection::open(positional[0]).unwrap();
+    if positional.len() > 1 {
+        infile_paths.push(std::path::PathBuf::from(positional[1]));
+    }
+
+    {
+        db.execute("
+            CREATE TABLE IF NOT EXISTS migrations (
+                id INTEGER PRIMARY KEY NOT NULL
+            )
+        ", ()).unwrap();
+
+        let newest_migration: u32 = db.query_row("
+            SELECT MAX(id) FROM migrations
+        ", [], |row| row.get::<_, Option<u32>>(0)).unwrap().unwrap_or(0);
+
+        if newest_migration < 1 {
+            db.execute("INSERT INTO migrations(id) VALUES (?1)", (1, )).unwrap();
+        }
+
+        if newest_migration < 2 {
+            db.execute("
+                CREATE TABLE known_images (
+                    id INTEGER PRIMARY KEY NOT NULL,
+                    registry TEXT NOT NULL,
+                    image TEXT NOT NULL,
+                    tag TEXT NOT NULL
+                )
+            ", ()).unwrap();
+
+            db.execute("INSERT INTO migrations(id) VALUES (?1)", (2, )).unwrap();
+        }
+
+        if newest_migration < 3 {
+            db.execute("
+                ALTER TABLE known_images ADD COLUMN
+                    discovered DATETIME NOT NULL
+            ", ()).unwrap();
+
+            db.execute("INSERT INTO migrations(id) VALUES (?1)", (3, )).unwrap();
+        }
+
+        if newest_migration < 4 {
+            db.execute("
+                CREATE UNIQUE INDEX known_images__registry_image_tag
+                    ON known_images(registry, image, tag)
+            ", ()).unwrap();
+
+            db.execute("INSERT INTO migrations(id) VALUES (?1)", (4, )).unwrap();
+        }
     }
 
     let mut repos = vec![];
@@ -742,197 +1004,13 @@ fn main() {
         }
     }
 
-    for repo in &repos {
-        if repo.dest.exists() {
-            let mut gitcmd = std::process::Command::new("git");
-
-            gitcmd
-                .arg("-C")
-                .arg(&repo.dest)
-                .arg("fetch")
-                .arg("origin");
-
-            if let Some(key) = &repo.key {
-                gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap()));
-            }
-            let exit = gitcmd.status()
-                .expect("Git command failed");
-            if !exit.success() {
-                panic!("Git exited with failure");
-            }
-
-            let mut gitcmd = std::process::Command::new("git");
-
-            gitcmd
-                .arg("-C")
-                .arg(&repo.dest)
-                .arg("reset")
-                .arg("--hard")
-                .arg("@{u}");
-
-            let exit = gitcmd.status()
-                .expect("Git command failed");
-            if !exit.success() {
-                panic!("Git exited with failure");
-            }
-
-            let mut gitcmd = std::process::Command::new("git");
-
-            gitcmd
-                .arg("-C")
-                .arg(&repo.dest)
-                .arg("clean")
-                .arg("--force")
-                .arg("-d")
-                .arg("-x");
-
-            let exit = gitcmd.status()
-                .expect("Git command failed");
-            if !exit.success() {
-                panic!("Git exited with failure");
-            }
-        } else {
-            let mut gitcmd = std::process::Command::new("git");
-
-            gitcmd
-                .arg("clone")
-                .arg(&repo.url)
-                .arg(&repo.dest);
-
-            if let Some(key) = &repo.key {
-                gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap()));
-            }
-
-            let exit = gitcmd.status()
-                .expect("Git command failed");
-            if !exit.success() {
-                panic!("Git exited with failure");
-            }
-        }
-
-        infile_paths.push(repo.dest.clone());
-    }
-
-    let mut inpaths = vec!();
-    for infile_path in infile_paths {
-        if infile_path.is_dir() {
-            let mut unsearched = vec![infile_path.clone()];
-
-            while let Some(next) = unsearched.pop() {
-                for child in next.read_dir().unwrap() {
-                    let child = child.unwrap();
-                    let path = child.path();
-
-                    let ft = child.file_type().unwrap();
-                    if ft.is_dir() {
-                        unsearched.push(path);
-                        continue;
-                    }
-
-                    if let Some(ext) = path.extension() {
-                        if ext == "yaml" {
-                            inpaths.push(path);
-                        }
-                    }
-                }
-            }
-        } else {
-            inpaths.push(infile_path);
-        };
-    }
-
     let mut auth = Auth::new(auths);
 
-    for infile_path in inpaths {
-        let mut file = std::fs::File::open(&infile_path).unwrap();
-        let mut file_content = String::new();
-        file.read_to_string(&mut file_content).unwrap();
-
-        let images = ManifestFile::parse(&file_content);
-
-        let mut failed = None;
-
-        let mut edits = vec![];
-        for ref image in images.image_tags {
-            println!("Checking image {}", &file_content[image.clone()]);
-            let image_ref = DockerRef::parse(&file_content, image);
-            if let Err(msg) = update_images(&file_content, &mut auth, image_ref, &mut edits) {
-                failed = Some(msg);
-                break;
-            }
-        }
-
-        if let Some(msg) = failed {
-            println!("{}: {} ", infile_path.display(), msg);
-            continue;
-        }
-
-        let mut out = if overwrite {
-            Output::overlay_file(&infile_path)
-        } else {
-            Output::Stdout(std::io::stdout())
-        };
-
-        let mut current_position = 0;
-        file.seek(std::io::SeekFrom::Start(0)).unwrap();
-        let mut file_block = file.take(0);
-        for edit in edits {
-            if edit.position.start > current_position {
-                file_block.set_limit((edit.position.start - current_position) as u64);
-                std::io::copy(&mut file_block, &mut out).unwrap();
-            }
-
-            out.write_all(edit.content.as_bytes()).unwrap();
-
-            // We should have been able to just seek to the position.end here, but that doesn't work
-            // for whatever reason. What we can do is calculate the amount to skip ahead, and then do
-            // that.
-            let skip = (edit.position.end - edit.position.start) as i64;
-            file_block.set_limit(skip as u64);
-            file_block.seek(std::io::SeekFrom::Current(skip)).unwrap();
-            current_position = edit.position.end;
-        }
-
-        file_block.set_limit(u64::MAX);
-        std::io::copy(&mut file_block, &mut out).unwrap();
-
-        out.commit();
-    }
-
-    for repo in &repos {
-        let mut gitcmd = std::process::Command::new("git");
-
-        gitcmd
-            .arg("-C")
-            .arg(&repo.dest)
-            .arg("commit")
-            .arg("--all")
-            .arg("--message")
-            .arg("Update versions");
-
-        let exit = gitcmd.status()
-            .expect("Git command failed");
-        if !exit.success() {
-            println!("Commit failed, presumably there were no changes");
-            continue;
-        }
-
-        let mut gitcmd = std::process::Command::new("git");
-        gitcmd
-            .arg("-C")
-            .arg(&repo.dest)
-            .arg("push")
-            .arg("origin")
-            .arg("+HEAD:version-bump");
-
-        if let Some(key) = &repo.key {
-            gitcmd.env("GIT_SSH_COMMAND", format!("ssh -F none -o IdentitiesOnly=yes -i \"{}\"", key.to_str().unwrap()));
-        }
+    loop {
+        run_tool(&db, &mut auth, &repos, infile_paths.clone(), overwrite);
+        if !continuous { break; }
 
-        let exit = gitcmd.status()
-            .expect("Git command failed");
-        if !exit.success() {
-            panic!("Git exited with failure");
-        }
+        println!("Waiting for next run");
+        std::thread::sleep(std::time::Duration::from_hours(24));
     }
 }