summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/main.rs194
-rw-r--r--src/metrics.rs4
2 files changed, 132 insertions, 66 deletions
diff --git a/src/main.rs b/src/main.rs
index 3811e4a..97bc034 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -38,7 +38,7 @@ Options:
#[derive(Debug)]
enum AuthMethod {
Basic,
- Bearer{realm: String, service: String, scope: String},
+ Bearer { realm: String, service: String, scope: String },
}
impl AuthMethod {
@@ -51,7 +51,7 @@ impl AuthMethod {
}
if &header_str[parse.scheme.clone()] == "Bearer" {
- return Some(AuthMethod::Bearer{
+ return Some(AuthMethod::Bearer {
realm: header_str[parse.realm.unwrap()].to_string(),
scope: header_str[parse.scope.unwrap()].to_string(),
service: header_str[parse.service.unwrap()].to_string(),
@@ -147,7 +147,7 @@ impl AuthState{
return Ok(());
},
- None => Err(format!("Server {} provided us with a challenge, but we didn't understand it", self.info.host)),
+ None => return Err(format!("Server {} provided us with a challenge, but we didn't understand it", self.info.host)),
}
}
}
@@ -251,79 +251,103 @@ fn update_images(db: &Connection, file: &str, auth: &mut Auth, img: DockerRef, e
metrics::get().images_checked.with_label_values(&[registry]).inc();
+ let now = chrono::offset::Utc::now();
+ let cache_max_age = chrono::Duration::hours(24);
+
let query_result = {
let _timer = metrics::get().db_query_duration.start_timer();
db.query_one("
- SELECT last_checked FROM images
+ SELECT id, last_checked FROM images
WHERE registry = ?1 AND image = ?2
- ", (registry, image), |row| row.get::<_, chrono::DateTime<chrono::Utc>>(0)).optional()
+ ", (registry, image), |row| Ok((
+ row.get::<_, i64>(0)?,
+ row.get::<_, chrono::DateTime<chrono::Utc>>(1)?,
+ ))).optional()
};
- let _last_checked = match query_result {
+ let (image_id, last_checked) = match query_result {
Ok(Some(x)) => x,
Ok(None) => {
- let time = chrono::offset::Utc::now();
{
let _timer = metrics::get().db_query_duration.start_timer();
db.execute("
INSERT INTO images(registry, image, last_checked) VALUES (?1, ?2, ?3)
- ", (registry, &image, time)).unwrap();
+ ", (registry, &image, now)).unwrap();
}
- time
+ (db.last_insert_rowid(), now)
},
Err(_x) => return Err("Database Failure".to_string()),
};
+ let cache_stale = now - last_checked > cache_max_age;
if let Some(ref tag_str) = tag {
- let mut current = VersionPattern::parse(&tag_str);
+ if cache_stale {
+ let mut url = format!("/v2/{}/tags/list", &image);
+ loop {
+ let mut response = match perform_registry_request(registry, &url, "application/vnd.oci.image.index.v1+json", auth) {
+ Ok(r) => r,
+ Err(RegistryError::NotFound) => {
+ println!("Warning: image not found, skipping: {}", image);
+ return Ok(());
+ },
+ Err(RegistryError::Other(msg)) => return Err(msg),
+ };
- let mut new_tag = None;
+ let content_type = response.headers()["Content-Type"].to_str().unwrap();
+ if !content_type.starts_with("application/json") {
+ return Err(format!("Unexpected Content-Type: {}", content_type));
+ }
- let mut url = format!("/v2/{}/tags/list", &image);
- loop {
- let mut response = match perform_registry_request(registry, &url, "application/vnd.oci.image.index.v1+json", auth) {
- Ok(r) => r,
- Err(RegistryError::NotFound) => {
- println!("Warning: image not found, skipping: {}", image);
- return Ok(());
- },
- Err(RegistryError::Other(msg)) => return Err(msg),
- };
+ let body = response.body_mut().read_to_string().unwrap();
+ let body = body.parse::<tinyjson::JsonValue>().unwrap();
- let content_type = response.headers()["Content-Type"].to_str().unwrap();
- if !content_type.starts_with("application/json") {
- return Err(format!("Unexpected Content-Type: {}", content_type));
- }
+ for it in body["tags"].get::<Vec<tinyjson::JsonValue>>().unwrap().iter() {
+ let candidate_str = it.get::<String>().unwrap();
- let body = response.body_mut().read_to_string().unwrap();
- let body = body.parse::<tinyjson::JsonValue>().unwrap();
+ let _timer = metrics::get().db_query_duration.start_timer();
+ db.execute("
+ INSERT OR IGNORE INTO tags(image_id, tag, fetched_at) VALUES (?1, ?2, ?3)
+ ", (image_id, candidate_str, now)).unwrap();
+ }
- for it in body["tags"].get::<Vec<tinyjson::JsonValue>>().unwrap().iter() {
- let candidate_str = it.get::<String>().unwrap();
+ if let Some(link_header) = response.headers().get("link") {
+ let link_str = link_header.to_str().unwrap();
+ let link = extract_next_page(&link_str).unwrap();
+ url = link_str[link.next_uri.unwrap().clone()].to_string();
+ } else {
+ break;
+ }
+ }
- // db.execute("
- // INSERT INTO known_images(registry, image, tag, discovered) VALUES (?1, ?2, ?3, ?4)
- // ", (registry, &file[img.image.clone()], candidate_str, chrono::offset::Utc::now())).unwrap();
+ let _timer = metrics::get().db_query_duration.start_timer();
+ db.execute("UPDATE images SET last_checked = ?1 WHERE id = ?2", (now, image_id)).unwrap();
+ }
- let candidate = VersionPattern::parse(candidate_str);
- match current.compare(&candidate) {
- CompareOutcome::Higher => {
- new_tag = Some(candidate_str.clone());
- current = candidate;
- },
- CompareOutcome::Lower => {},
- CompareOutcome::Incompatible => {},
- CompareOutcome::Identical => {},
- }
+ let tags: Vec<String> = {
+ let _timer = metrics::get().db_query_duration.start_timer();
+ let mut stmt = db.prepare("SELECT tag FROM tags WHERE image_id = ?1").unwrap();
+ let mut rows = stmt.query((image_id,)).unwrap();
+ let mut tags = vec![];
+ while let Some(row) = rows.next().unwrap() {
+ tags.push(row.get(0).unwrap());
}
+ tags
+ };
- if let Some(link_header) = response.headers().get("link") {
- let link_str = link_header.to_str().unwrap();
- let link = extract_next_page(&link_str).unwrap();
- url = link_str[link.next_uri.unwrap().clone()].to_string();
- } else {
- break;
+ let mut current = VersionPattern::parse(&tag_str);
+ let mut new_tag = None;
+
+ for candidate_str in tags {
+ let candidate = VersionPattern::parse(&candidate_str);
+ match current.compare(&candidate) {
+ CompareOutcome::Higher => {
+ new_tag = Some(candidate_str);
+ current = candidate;
+ },
+ CompareOutcome::Lower => {},
+ CompareOutcome::Incompatible => {},
+ CompareOutcome::Identical => {},
}
}
@@ -338,21 +362,44 @@ fn update_images(db: &Connection, file: &str, auth: &mut Auth, img: DockerRef, e
}
if let Some(ref digest) = img.digest {
- // Find the digest for the selected tag
- let url = format!("/v2/{}/manifests/{}", &file[img.image.clone()], tag.as_deref().unwrap_or("latest"));
- let response = match perform_registry_request(registry, &url, "application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json", auth) {
- Ok(r) => r,
- Err(RegistryError::NotFound) => {
- println!("Warning: image not found, skipping: {}", image);
- return Ok(());
- },
- Err(RegistryError::Other(msg)) => return Err(msg),
- };
+ let tag_for_digest = tag.as_deref().unwrap_or("latest");
+
+ let mut digest_string = None;
+ if !cache_stale {
+ let _timer = metrics::get().db_query_duration.start_timer();
+ digest_string = db.query_one("
+ SELECT digest FROM tags WHERE image_id = ?1 AND tag = ?2 AND digest IS NOT NULL
+ ", (image_id, tag_for_digest), |row| row.get::<_, String>(0)).optional().unwrap();
+ }
- let digest_string = response.headers()["docker-content-digest"].to_str().unwrap().to_string();
+ if digest_string.is_none() {
+ let url = format!("/v2/{}/manifests/{}", &file[img.image.clone()], tag_for_digest);
+ let response = match perform_registry_request(registry, &url, "application/vnd.oci.image.manifest.v1+json,application/vnd.oci.image.index.v1+json", auth) {
+ Ok(r) => r,
+ Err(RegistryError::NotFound) => {
+ println!("Warning: image not found, skipping: {}", image);
+ return Ok(());
+ },
+ Err(RegistryError::Other(msg)) => return Err(msg),
+ };
+
+ let fetched = response.headers()["docker-content-digest"].to_str().unwrap().to_string();
+
+ {
+ let _timer = metrics::get().db_query_duration.start_timer();
+ db.execute("
+ INSERT INTO tags(image_id, tag, digest, fetched_at) VALUES (?1, ?2, ?3, ?4)
+ ON CONFLICT(image_id, tag) DO UPDATE SET digest = ?3, fetched_at = ?4
+ ", (image_id, tag_for_digest, &fetched, now)).unwrap();
+ }
+
+ digest_string = Some(fetched);
+ }
+
+ let digest_string = digest_string.unwrap();
if file[digest.clone()] != digest_string {
- edits.push(FilePatch{
+ edits.push(FilePatch {
position: img.digest.unwrap(),
content: digest_string,
});
@@ -441,7 +488,7 @@ fn is_dockerfile(path: &std::path::Path) -> bool {
return name == "Dockerfile" || name == "dockerfile";
}
}
- return false
+ return false;
}
fn is_yaml(path: &std::path::Path) -> bool {
@@ -450,7 +497,7 @@ fn is_yaml(path: &std::path::Path) -> bool {
return ext == "yaml";
}
}
- return false
+ return false;
}
fn run_tool(db: &Connection, auth: &mut Auth, repos: &Vec<Repository>, mut infile_paths: Vec<std::path::PathBuf>, overwrite: bool) {
@@ -526,7 +573,7 @@ fn run_tool(db: &Connection, auth: &mut Auth, repos: &Vec<Repository>, mut infil
infile_paths.push(repo.dest.clone());
}
- let mut inpaths = vec!();
+ let mut inpaths = vec![];
for infile_path in infile_paths {
if infile_path.is_dir() {
let mut unsearched = vec![infile_path.clone()];
@@ -807,6 +854,25 @@ fn main() {
db.execute("INSERT INTO migrations(id) VALUES (?1)", (5, )).unwrap();
}
+
+ if newest_migration < 6 {
+ db.execute("
+ CREATE TABLE tags (
+ id INTEGER PRIMARY KEY,
+ image_id INTEGER NOT NULL REFERENCES images(id),
+ tag TEXT NOT NULL,
+ digest TEXT,
+ fetched_at DATETIME NOT NULL
+ )
+ ", ()).unwrap();
+
+ db.execute("
+ CREATE UNIQUE INDEX tags__image_id_tag
+ ON tags(image_id, tag)
+ ", ()).unwrap();
+
+ db.execute("INSERT INTO migrations(id) VALUES (?1)", (6, )).unwrap();
+ }
}
let mut repos = vec![];
@@ -962,7 +1028,7 @@ fn main() {
_ => panic!("Invalid config json"),
}
- repos.push(Repository{
+ repos.push(Repository {
url: k,
key: Some(workdir.join(path.parent().unwrap().join(std::path::PathBuf::from(key_name)))),
host: Some(workdir.join(path.parent().unwrap().join(std::path::PathBuf::from(host_name)))),
diff --git a/src/metrics.rs b/src/metrics.rs
index a6c8286..95a4782 100644
--- a/src/metrics.rs
+++ b/src/metrics.rs
@@ -99,7 +99,7 @@ impl Metrics {
).unwrap();
registry.register(Box::new(db_query_duration.clone())).unwrap();
- Self {
+ return Self {
state,
images_checked,
images_updated,
@@ -111,7 +111,7 @@ impl Metrics {
git_operations,
db_query_duration,
registry,
- }
+ };
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-03 20:06:17 +0000