From a95c094063bb53d35a6186dc9cb32db3a6a2c070 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 13:05:15 -0400 Subject: [PATCH 01/12] cargo update --- Cargo.lock | 72 +++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d16b9d3..224cb94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,9 +129,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.17" +version = "1.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" dependencies = [ "shlex", ] @@ -144,9 +144,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.34" +version = "4.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e958897981290da2a852763fe9cdb89cd36977a5d729023127095fa94d95e2ff" +checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04" dependencies = [ "clap_builder", "clap_derive", @@ -154,9 +154,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.34" +version = "4.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b0f35019843db2160b5bb19ae09b4e6411ac33fc6a712003c33e03090e2489" +checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5" dependencies = [ "anstream", "anstyle", @@ -236,9 +236,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" dependencies = [ "anstream", "anstyle", @@ -255,9 +255,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", "windows-sys 0.59.0", @@ -369,9 +369,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "h2" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" dependencies = [ "atomic-waker", "bytes", @@ -652,9 +652,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown", @@ -680,9 +680,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" +checksum = "e5ad87c89110f55e4cd4dc2893a9790820206729eaf221555f742d540b0724a0" dependencies = [ "jiff-static", "log", @@ -693,9 +693,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" +checksum = "d076d5b64a7e2fe6f0743f02c43ca4a6725c0f904203bfe276a5b3e793103605" dependencies = [ "proc-macro2", "quote", @@ -720,9 +720,9 @@ checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "linux-raw-sys" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" @@ -760,9 +760,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -825,9 +825,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openssl" -version = "0.10.71" +version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ "bitflags", "cfg-if", @@ -857,9 +857,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.106" +version = "0.9.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" dependencies = [ "cc", "libc", @@ -955,9 +955,9 @@ checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ "bitflags", ] @@ -1057,9 +1057,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ "bitflags", "errno", @@ -1070,9 +1070,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.25" +version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" +checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ "once_cell", "rustls-pki-types", @@ -1227,9 +1227,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "socket2" @@ -1336,9 +1336,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.44.1" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", From abdff736f5fd05ae5f0b0e231ff3c9781b540021 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 15:11:01 -0400 Subject: [PATCH 02/12] aws doesn't use defaults --- src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index cf78e63..57d3340 100644 --- a/src/main.rs +++ b/src/main.rs @@ -44,11 +44,11 @@ pub struct PushArgs { #[arg(long)] signing_key: String, - /// If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default + /// If unspecified, will get it form AWS_DEFAULT_REGION envar #[arg(long)] region: Option, - /// If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default + /// If unspecifed, will get it from AWS_ENDPOINT_URL envar /// e.g. https://s3.example.com #[arg(long)] endpoint: Option, From 5111af1d34f3f80c2792f8b237d600f32cb09f6e Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 16:32:12 -0400 Subject: [PATCH 03/12] add a trace --- src/path_info.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/path_info.rs b/src/path_info.rs index 7debc33..24969c9 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -32,6 +32,11 @@ impl PathInfo { .await .context("`nix path-info` failed for {package}")?; + trace!( + "nix path-info output: {}", + String::from_utf8_lossy(&nix_cmd.stdout) + ); + // nix path-info returns an array with one element match serde_json::from_slice::>(&nix_cmd.stdout) .context("parse path info from stdout") From df8d3a0bc6f0441713480c7fbfbd30d004a916c8 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 16:47:59 -0400 Subject: [PATCH 04/12] don't do anything cahash cause serde fails to parse sometimes --- src/path_info.rs | 3 +-- src/uploader.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 24969c9..77a8cdb 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -2,8 +2,8 @@ use std::collections::HashSet; use anyhow::{Context, Error, Result}; use aws_sdk_s3 as s3; +use nix_compat::nixbase32; use nix_compat::store_path::StorePath; -use nix_compat::{nixbase32, nixhash::CAHash}; use regex::Regex; use serde::{Deserialize, Serialize}; use tokio::process::Command; @@ -18,7 +18,6 @@ pub struct PathInfo { pub path: StorePath, signatures: Vec, pub references: Vec>, - pub ca: Option, } impl PathInfo { /// get PathInfo for a package or a store path diff --git a/src/uploader.rs b/src/uploader.rs index e52e4ae..0a25cff 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -157,7 +157,7 @@ impl<'a> Uploader<'a> { nar_size: nar.len() as u64, references: self.path.references.iter().map(StorePath::as_ref).collect(), signatures: Vec::new(), - ca: self.path.ca.clone(), + ca: None, system: None, deriver: Some(self.path.deriver.as_ref()), compression: Some("zstd"), From 6b959f583ba6ec51778b4397906039d654af60bb Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 19:29:33 -0400 Subject: [PATCH 05/12] make deriver optional cause it won't be there if path is a derivation --- src/path_info.rs | 2 +- src/uploader.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 77a8cdb..91c6612 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -14,7 +14,7 @@ use url::Url; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PathInfo { - pub deriver: StorePath, + pub deriver: Option>, pub path: StorePath, signatures: Vec, pub references: Vec>, diff --git a/src/uploader.rs b/src/uploader.rs index 0a25cff..b0520ac 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -159,7 +159,7 @@ impl<'a> Uploader<'a> { signatures: Vec::new(), ca: None, system: None, - deriver: Some(self.path.deriver.as_ref()), + deriver: self.path.deriver.as_ref().map(|x| x.as_ref()), compression: Some("zstd"), file_hash: None, file_size: None, From 307109d9f15d4433e24bbebcb2d9ebc37b394a48 Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 14:13:15 -0400 Subject: [PATCH 06/12] we don't ever serialize pathinfo --- src/path_info.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 91c6612..9694ddd 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -5,14 +5,13 @@ use aws_sdk_s3 as s3; use nix_compat::nixbase32; use nix_compat::store_path::StorePath; use regex::Regex; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use tokio::process::Command; use tracing::{debug, error, trace}; use url::Url; // nix path-info --derivation --json -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, Clone, Deserialize)] pub struct PathInfo { pub deriver: Option>, pub path: StorePath, From c37a05ba9ff6a65be786e2a7ef2c231b59efc604 Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 14:40:18 -0400 Subject: [PATCH 07/12] make signatures optional when deserializing --- src/path_info.rs | 56 +++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 9694ddd..bcae0ae 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -15,7 +15,7 @@ use url::Url; pub struct PathInfo { pub deriver: Option>, pub path: StorePath, - signatures: Vec, + signatures: Option>, pub references: Vec>, } impl PathInfo { @@ -91,13 +91,15 @@ impl PathInfo { } fn signees(&self) -> Vec<&str> { - let signees: Vec<_> = self - .signatures - .iter() - .filter_map(|signature| Some(signature.split_once(":")?.0)) - .collect(); - trace!("signees for {}: {:?}", self.path, signees); - signees + if let Some(signatures) = self.signatures.as_ref() { + let signees: Vec<_> = signatures + .iter() + .filter_map(|signature| Some(signature.split_once(":")?.0)) + .collect(); + trace!("signees for {}: {:?}", self.path, signees); + return signees; + } + Vec::new() } pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { @@ -147,10 +149,10 @@ mod tests { let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#; let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); - path_info.signatures = vec![ + path_info.signatures = Some(vec![ "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), - ]; + ]); let signees = path_info.signees(); assert_eq!(signees, vec!["cache.nixos.org-1", "nixcache.cy7.sh"]); } @@ -160,29 +162,35 @@ mod tests { let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#; let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); - path_info.signatures = vec![ + path_info.signatures = Some(vec![ "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=".to_string(), - ]; - assert_eq!( - path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]), - true + ]); + assert!( + path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]) ); - assert_eq!( - path_info.check_upstream_signature(&[Url::parse("https://nixcache.cy7.sh").unwrap()]), - true + assert!( + path_info.check_upstream_signature(&[Url::parse("https://nixcache.cy7.sh").unwrap()]) ); - assert_eq!( + assert!( path_info.check_upstream_signature(&[ Url::parse("https://nix-community.cachix.org").unwrap() - ]), - true + ]) ); - assert_eq!( - path_info + assert!( + !path_info .check_upstream_signature(&[Url::parse("https://fake-cache.cachix.org").unwrap()]), - false + ); + } + + #[test] + fn path_info_without_signature() { + let path_info_json = r#"{"ca":"fixed:r:sha256:1q10p04pgx9sk6xbvrkn4nvh0ys2lzplgcni5368f4z3cr8ikbmz","narHash":"sha256-v64ZUWbjE4fMKNGyR++nQnsAtyV25r26mTr1dwm4IOA=","narSize":5520,"path":"/nix/store/gj6hz9mj23v01yvq1nn5f655jrcky1qq-nixos-option.nix","references":[],"registrationTime":1744740942,"valid":true}"#; + let path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); + + assert!( + !path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]) ); } } From d4cac6524719fb8c8bf1435e822c4d2a419a512d Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 16:52:26 -0400 Subject: [PATCH 08/12] use lix --- src/path_info.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/path_info.rs b/src/path_info.rs index bcae0ae..0f05a87 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -18,11 +18,17 @@ pub struct PathInfo { signatures: Option>, pub references: Vec>, } + impl PathInfo { - /// get PathInfo for a package or a store path + // get PathInfo for a package or a store path + // we deserialize this as an array of `PathInfo` below pub async fn from_path(path: &str) -> Result { debug!("query nix path-info for {path}"); + // use lix cause nix would return a json map instead of an array let nix_cmd = Command::new("nix") + .arg("run") + .arg("github:nixos/nixpkgs/nixos-unstable#lix") + .arg("--") .arg("path-info") .arg("--json") .arg(path) @@ -193,4 +199,15 @@ mod tests { !path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]) ); } + + /* + #[test] + fn path_info_deserialize_nix_map() { + let path_info_json = r#"{"/nix/store/8vm1jxsc0jphd65vb7r6g5ysgqw0yh9f-home-manager-generation":{"ca":null,"deriver":"/nix/store/h8z25s6arcrns5nmrq1yhgbamywjivpn-home-manager-generation.drv","narHash":"sha256-o4qwqyJ5UVm9cyC/nBNcNYVnIM14Pewgw7fou+wUVSY=","narSize":13608,"references":["/nix/store/40yifhx34v4g4llrdn3v2ag8w02j10fv-gnugrep-3.11","/nix/store/4d0ix5djms3n2jnjdc58l916cwack1rp-empty-directory","/nix/store/56zmgla8443qfrkrh2ch0vz0mh8ywrw1-home-manager-files","/nix/store/58br4vk3q5akf4g8lx0pqzfhn47k3j8d-bash-5.2p37","/nix/store/80l1sb3vcmrkcdd7ihlizkcnv19rq9fj-ncurses-6.5","/nix/store/8vm1jxsc0jphd65vb7r6g5ysgqw0yh9f-home-manager-generation","/nix/store/92as847i10kl6s19fi910ddyk9l83835-check-link-targets.sh","/nix/store/9c90iz95yynyh3vsc67zndch6j01vgz3-home-manager-path","/nix/store/b2cfj7yk3wfg1jdwjzim7306hvsc5gnl-systemd-257.3","/nix/store/bm5fi6wj0w4r2wjll2448k307bzfcjwx-cleanup","/nix/store/c244fsb3a7i5837lzn94m4bmav9i5p9b-link","/nix/store/cvlbhhrvzfkjl2hrrzhq3vr5gzan1r60-bash-interactive-5.2p37","/nix/store/gpxsdrrd4x93fs75395vr2dfys1ki9mq-jq-1.7.1-bin","/nix/store/jlf743lqxbvad6dbgndsgqfg20m2np5i-sd-switch-0.5.3","/nix/store/mhmgm739aagj4x7hr6ag2wjmxhmpy8mf-gettext-0.22.5","/nix/store/w9db12j05yv5hl31s6jndd9cfm1g1gw4-hm-modules-messages","/nix/store/wj1c3gsiajabnq50ifxqnlv60i5rhqj7-diffutils-3.10","/nix/store/xhql0ilzbiqwnmz4z8y0phk611wynxf2-gnused-4.9","/nix/store/xq5f95pp297afc2xjgrmhmf9w631qp7m-findutils-4.10.0","/nix/store/yh6qg1nsi5h2xblcr67030pz58fsaxx3-coreutils-9.6","/nix/store/zhrjg6wxrxmdlpn6iapzpp2z2vylpvw5-home-manager.sh"],"registrationTime":1744742989,"signatures":["nixcache.cy7.sh:Vq4X95kSzum7BwrBhjmmM2yVipfBI3AE3jgZ3b3RoYrP4/ghotbDdlwCvwK3qx4BQdEOLSgrC1tDwiMNb6oRBw=="],"ultimate":false}}"#; + serde_json::from_str::>(path_info_json).expect("must serialize"); + + let path_info_json = r#"{"/nix/store/3a2ahdaprw6df0lml1pj9jhbi038dsjh-nixos-system-chunk-25.05.20250412.2631b0b":{"ca":null,"deriver":"/nix/store/12ssi931481jlkizgfk1c1jnawvwjbhh-nixos-system-chunk-25.05.20250412.2631b0b.drv","narHash":"sha256-CHhBIzMD4v/FKqKgGroq0UC1k3GrK5lcNwQPMpv2xLc=","narSize":20704,"references":["/nix/store/0yjiyixxsr137iw93hnaacdsssy1li9h-switch-to-configuration-0.1.0","/nix/store/14rby7cpwrzjsjym44cl5h6nj6qpn1gs-etc","/nix/store/3a2ahdaprw6df0lml1pj9jhbi038dsjh-nixos-system-chunk-25.05.20250412.2631b0b","/nix/store/3wjljpj30fvv2cdb60apr4126pa5bm87-shadow-4.17.2","/nix/store/40yifhx34v4g4llrdn3v2ag8w02j10fv-gnugrep-3.11","/nix/store/58br4vk3q5akf4g8lx0pqzfhn47k3j8d-bash-5.2p37","/nix/store/5dyh8l59kfvf89zjkbmjfnx7fix93n4f-net-tools-2.10","/nix/store/aq9wdsz12bg9252790l9awiry2bml4ls-sops-install-secrets-0.0.1","/nix/store/b00kq6fjhgisdrykg621vml8505nnmb3-users-groups.json","/nix/store/b2cfj7yk3wfg1jdwjzim7306hvsc5gnl-systemd-257.3","/nix/store/bfr68wi6k8icb3j9fy3fzchva56djfhd-mounts.sh","/nix/store/cjnihsds5hhnji9r85hglph07q9y9hgc-system-path","/nix/store/cvlbhhrvzfkjl2hrrzhq3vr5gzan1r60-bash-interactive-5.2p37","/nix/store/f9jll96j74f5ykvs062718b98lfjbn9g-util-linux-2.40.4-bin","/nix/store/h7zih134d3n5yk8pnhv1fa38n6qkyrn2-pre-switch-checks","/nix/store/idn5n51246piyxcr3v6gxnj5a5l9mzpn-linux-6.14.2","/nix/store/ipn5793y61x2904xqnkgbjnp91svjjzx-perl-5.40.0-env","/nix/store/j1rikvl25pz0b5ham1ijq0nbg1q2fqfy-initrd-linux-6.14.2","/nix/store/jgawnqyh6piwcl79gxpmq5czx9rfr9xh-glibc-locales-2.40-66","/nix/store/jqgmcv8j4gj59218hcbiyn8z951rycdj-install-grub.sh","/nix/store/kpmybhxy3gz6k1znbdirwsp3c6wvsgg9-manifest.json","/nix/store/lgainx4gl6q7mhiwmls81d3n51p5jz7z-linux-6.14.2-modules","/nix/store/mhxn5kwnri3z9hdzi3x0980id65p0icn-lib.sh","/nix/store/n8n0faszqlnf3mdg0fj6abnknrhjsw5j-perl-5.40.0-env","/nix/store/nq61v7a601gjndijq5nndprkzpwz4q9g-glibc-2.40-66-bin","/nix/store/nx27idxpvi3fk3p7admvhipny73nr25n-kmod-31","/nix/store/pggww1d2pg24fcg5v36xn63n53vanyyi-gnupg-2.4.7","/nix/store/rg5rf512szdxmnj9qal3wfdnpfsx38qi-setup-etc.pl","/nix/store/vvlfaafnz3pdhw7lx5kc5gb9pl4zhz5l-local-cmds","/nix/store/w142vx7ij1fz6qwhp5dprkf59cizvv1v-update-users-groups.pl","/nix/store/xq5f95pp297afc2xjgrmhmf9w631qp7m-findutils-4.10.0","/nix/store/yh6qg1nsi5h2xblcr67030pz58fsaxx3-coreutils-9.6","/nix/store/zlsmh0ccgvncg30qb4y0mp5pahnk1wnw-append-initrd-secrets","/nix/store/zs07icpv5ykf8m36xcv717hh26bp09fa-firmware","/nix/store/zy2n4id5gcxcbx2x8jbblkmcpdlpsypk-getent-glibc-2.40-66"],"registrationTime":1744743136,"signatures":["nixcache.cy7.sh:dZ1XiKQNe0fRX48gBj03PIABYJGV6BPwb72YpMqEBONZMF+JrkVKhRCF0ur/4Bf5prHxg6Qfg1ytP/4csRC9DQ=="],"ultimate":false}}"#; + serde_json::from_str::>(path_info_json).expect("must serialize"); + } + */ } From 03530a2afa11c4a2e8c9908b8a17217c138f7fb7 Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 18:53:45 -0400 Subject: [PATCH 09/12] don't limit stuff with semaphore and use println for uploading output --- src/push.rs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/push.rs b/src/push.rs index 5adab30..896b556 100644 --- a/src/push.rs +++ b/src/push.rs @@ -12,7 +12,7 @@ use aws_config::Region; use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::narinfo::{self, SigningKey}; -use tokio::sync::{RwLock, Semaphore, mpsc}; +use tokio::sync::{RwLock, mpsc}; use tracing::{debug, info, trace}; use url::Url; @@ -98,7 +98,6 @@ impl Push { /// filter paths that are on upstream and send to `tx` async fn filter_from_upstream(&'static self, tx: mpsc::Sender) { - let permits = Arc::new(Semaphore::new(10)); let mut handles = Vec::with_capacity(10); let store_paths = self.store_paths.read().await.clone(); @@ -109,11 +108,8 @@ impl Push { continue; } handles.push({ - let permits = permits.clone(); let tx = tx.clone(); tokio::spawn(async move { - let _permit = permits.acquire().await.unwrap(); - if !path .check_upstream_hit(self.upstream_caches.as_slice()) .await @@ -144,15 +140,13 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let upload_count = AtomicUsize::new(0); - let permits = Arc::new(Semaphore::new(10)); let mut uploads = Vec::with_capacity(10); loop { if let Some(path_to_upload) = rx.recv().await { - let permits = Arc::clone(&permits); let absolute_path = path_to_upload.absolute_path(); - info!("uploading: {}", absolute_path); + println!("uploading: {}", absolute_path); let uploader = Uploader::new( &self.signing_key, path_to_upload, @@ -160,10 +154,7 @@ impl Push { self.bucket.clone(), )?; - uploads.push(tokio::spawn(async move { - let _permit = permits.acquire().await.unwrap(); - uploader.upload().await - })); + uploads.push(tokio::spawn(async move { uploader.upload().await })); } else { join_all(uploads) .await From 489572083183aa25902635f306d8f4625cfd46ce Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 20:00:55 -0400 Subject: [PATCH 10/12] fix upload count --- src/push.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/push.rs b/src/push.rs index 896b556..719d3a8 100644 --- a/src/push.rs +++ b/src/push.rs @@ -30,6 +30,8 @@ pub struct Push { upstream_hit_count: AtomicUsize, // paths that we skipped cause they are already on our cache already_exists_count: AtomicUsize, + // paths that we uploaded + upload_count: AtomicUsize, } impl Push { @@ -68,6 +70,7 @@ impl Push { signature_hit_count: AtomicUsize::new(0), upstream_hit_count: AtomicUsize::new(0), already_exists_count: AtomicUsize::new(0), + upload_count: AtomicUsize::new(0), }) } @@ -139,7 +142,6 @@ impl Push { } async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { - let upload_count = AtomicUsize::new(0); let mut uploads = Vec::with_capacity(10); loop { @@ -154,7 +156,11 @@ impl Push { self.bucket.clone(), )?; - uploads.push(tokio::spawn(async move { uploader.upload().await })); + uploads.push(tokio::spawn(async move { + let res = uploader.upload().await; + self.upload_count.fetch_add(1, Ordering::Relaxed); + res + })); } else { join_all(uploads) .await @@ -162,7 +168,7 @@ impl Push { .flatten() .collect::>>()?; - println!("uploaded: {}", upload_count.load(Ordering::Relaxed)); + println!("uploaded: {}", self.upload_count.load(Ordering::Relaxed)); println!( "skipped because of signature match: {}", self.signature_hit_count.load(Ordering::Relaxed) From 20c13a86be3d571d59c1e5c82f3fdb558225d4c0 Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 20:06:46 -0400 Subject: [PATCH 11/12] use experimental-features flag when executing nix run --- src/path_info.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/path_info.rs b/src/path_info.rs index 0f05a87..6dcbb53 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -25,8 +25,12 @@ impl PathInfo { pub async fn from_path(path: &str) -> Result { debug!("query nix path-info for {path}"); // use lix cause nix would return a json map instead of an array + // json output is not stable and could break in future + // TODO figure out a better way let nix_cmd = Command::new("nix") .arg("run") + .arg("--experimental-features") + .arg("nix-command flakes") .arg("github:nixos/nixpkgs/nixos-unstable#lix") .arg("--") .arg("path-info") From 39792cdd403ab785830d65b767af0f4d3417e5f1 Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 20:09:09 -0400 Subject: [PATCH 12/12] cargo update --- Cargo.lock | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 419ee1b..aee2282 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "async-compression" @@ -414,7 +414,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.4.8", + "h2 0.4.9", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -425,7 +425,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -1298,9 +1298,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" dependencies = [ "atomic-waker", "bytes", @@ -1457,7 +1457,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.8", + "h2 0.4.9", "http 1.3.1", "http-body 1.0.1", "httparse", @@ -1494,7 +1494,7 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -1748,9 +1748,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" @@ -1780,9 +1780,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" @@ -2310,7 +2310,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.8", + "h2 0.4.9", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -2404,14 +2404,14 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.9.3", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] @@ -2429,9 +2429,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.25" +version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" +checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ "aws-lc-rs", "once_cell", @@ -2837,7 +2837,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix 1.0.3", + "rustix 1.0.5", "windows-sys 0.59.0", ] @@ -2966,7 +2966,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.25", + "rustls 0.23.26", "tokio", ]