From f51099b911dc0377f145259fe1ec21d7680de4aa Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 13 Apr 2025 01:33:15 -0400 Subject: [PATCH 01/20] reformat and stuff ig --- .gitignore | 1 + Cargo.lock | 10 +++ Cargo.toml | 3 + src/cli.rs | 2 + src/main.rs | 162 +++++++++++++---------------------------------- src/nixcp.rs | 64 +++++++++++++++++++ src/path_info.rs | 155 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 279 insertions(+), 118 deletions(-) create mode 100644 src/cli.rs create mode 100644 src/nixcp.rs create mode 100644 src/path_info.rs diff --git a/.gitignore b/.gitignore index 2d5df85..9624092 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target .direnv +result diff --git a/Cargo.lock b/Cargo.lock index d16b9d3..5cf46a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anyhow" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" + [[package]] name = "atomic-waker" version = "1.1.2" @@ -799,13 +805,16 @@ dependencies = [ name = "nixcp" version = "0.1.0" dependencies = [ + "anyhow", "clap", "env_logger", "log", + "regex", "reqwest", "serde", "serde_json", "tokio", + "url", ] [[package]] @@ -1469,6 +1478,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2428f7e..fe2d773 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,10 +4,13 @@ version = "0.1.0" edition = "2024" [dependencies] +anyhow = "1.0.97" clap = { version = "4.5.34", features = ["derive"] } env_logger = "0.11.7" log = "0.4.27" +regex = "1.11.1" reqwest = "0.12.15" serde = { version = "1.0.219", features = [ "derive" ]} serde_json = "1.0.140" tokio = { version = "1.44.1", features = [ "full" ]} +url = { version = "2.5.4", features = [ "serde" ]} diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..139597f --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,2 @@ + + diff --git a/src/main.rs b/src/main.rs index 27abf93..976efb9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ #![feature(let_chains)] +#![feature(extend_one)] use std::path::Path; use std::sync::{ @@ -6,92 +7,39 @@ use std::sync::{ atomic::{AtomicUsize, Ordering}, }; -use clap::Parser; +use anyhow::Result; +use clap::{Parser, Subcommand}; use log::{debug, trace}; -use serde::{Deserialize, Serialize}; use tokio::process::Command; use tokio::sync::{Semaphore, mpsc}; -const UPSTREAM_CACHES: &[&str] = &["https://cache.nixos.org"]; +use nixcp::NixCp; -// nix path-info --derivation --json -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PathInfo { - ca: String, - nar_hash: String, - nar_size: u32, - path: String, - references: Vec, - registration_time: u32, - valid: bool, -} +mod cli; +mod nixcp; +mod path_info; -impl PathInfo { - // find derivations related to package - async fn from_package(package: &str, recursive: bool) -> Vec { - let mut args = vec!["path-info", "--derivation", "--json"]; - if recursive { - args.push("--recursive"); - } - let path_infos = Command::new("nix") - .args(args) - .arg(package) - .output() - .await - .expect("path-info failed"); - - let path_infos: Vec = serde_json::from_slice(&path_infos.stdout) - .expect("no derivations found for this package"); - debug!("PathInfo's from nix path-info: {:#?}", path_infos); - path_infos - } - - // find store paths related to derivation - async fn get_store_paths(&self) -> Vec { - let mut store_paths: Vec = Vec::new(); - let nix_store_cmd = Command::new("nix-store") - .arg("--query") - .arg("--requisites") - .arg("--include-outputs") - .arg(&self.path) - .output() - .await - .expect("nix-store cmd failed"); - - let nix_store_out = String::from_utf8(nix_store_cmd.stdout).unwrap(); - for store_path in nix_store_out.split_whitespace().map(ToString::to_string) { - store_paths.push(store_path); - } - store_paths - } -} - -#[derive(Parser)] -#[command(version, about, long_about = None)] +#[derive(Parser, Debug)] +#[command(version, name = "nixcp")] struct Cli { - /// Package to upload to the binary cache - package: String, + #[command(subcommand)] + command: Commands, /// Address of the binary cache (passed to nix copy --to) #[arg(long, value_name = "BINARY CACHE")] to: String, /// Upstream cache to check against. Can be specified multiple times. - /// cache.nixos.org is always included - #[arg(long, short)] - upstream_cache: Vec, - - /// Whether to pass --recursive to nix path-info. Can queue a huge number of paths to upload - #[arg(long, short, default_value_t = false)] - recursive: bool, + /// cache.nixos.org is always included (unless --no-nixos-cache is passed) + #[arg(long = "upstream-cache", short)] + upstream_caches: Vec, /// Concurrent upstream cache checkers #[arg(long, default_value_t = 32)] upstream_checker_concurrency: u8, /// Concurrent uploaders - #[arg(long, default_value_t = 16)] + #[arg(long, default_value_t = 4)] uploader_concurrency: u8, /// Concurrent nix-store commands to run @@ -99,67 +47,45 @@ struct Cli { nix_store_concurrency: u8, } +#[derive(Debug, Subcommand)] +enum Commands { + Push { + /// Package or store path to upload + /// e.g. nixpkgs#hello or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 + package: String, + }, +} + #[tokio::main] -async fn main() { +async fn main() -> Result<()> { env_logger::init(); let cli = Cli::parse(); - let package = &cli.package; - let binary_cache = cli.to; - let mut upstream_caches = cli.upstream_cache; - for upstream in UPSTREAM_CACHES { - upstream_caches.push(upstream.to_string()); + let mut nixcp = NixCp::new(); + nixcp.add_upstreams(&cli.upstream_caches)?; + + match &cli.command { + Commands::Push { package } => { + nixcp.paths_from_package(package).await?; + } } - debug!("package: {}", package); - debug!("binary cache: {}", binary_cache); - debug!("upstream caches: {:#?}", upstream_caches); - println!("querying nix path-info"); - let derivations = PathInfo::from_package(package, cli.recursive).await; - println!("got {} derivations", derivations.len()); + Ok(()) - println!("querying nix-store"); - let mut handles = Vec::new(); - let concurrency = Arc::new(Semaphore::new(cli.nix_store_concurrency.into())); - let store_paths = Arc::new(RwLock::new(Vec::new())); + /* + let (cacheable_tx, mut cacheable_rx) = mpsc::channel(cli.uploader_concurrency.into()); - for derivation in derivations { - let store_paths = Arc::clone(&store_paths); - let permit = Arc::clone(&concurrency); + println!("spawning check_upstream"); + + println!("spawning uploader"); handles.push(tokio::spawn(async move { - let _permit = permit.acquire_owned().await.unwrap(); - let paths = derivation.get_store_paths().await; - store_paths.write().unwrap().extend(paths); + uploader(&mut cacheable_rx, binary_cache, cli.uploader_concurrency).await; })); - } - // resolve store paths for all derivations before we move on - for handle in handles { - handle.await.unwrap(); - } - println!("got {} store paths", store_paths.read().unwrap().len()); - let (cacheable_tx, mut cacheable_rx) = mpsc::channel(cli.uploader_concurrency.into()); - - println!("spawning check_upstream"); - handles = Vec::new(); - handles.push(tokio::spawn(async move { - check_upstream( - store_paths, - cacheable_tx, - cli.upstream_checker_concurrency, - Arc::new(upstream_caches), - ) - .await; - })); - - println!("spawning uploader"); - handles.push(tokio::spawn(async move { - uploader(&mut cacheable_rx, binary_cache, cli.uploader_concurrency).await; - })); - - // make sure all threads are done - for handle in handles { - handle.await.unwrap(); - } + // make sure all threads are done + for handle in handles { + handle.await.unwrap(); + } + */ } // filter out store paths that exist in upstream caches diff --git a/src/nixcp.rs b/src/nixcp.rs new file mode 100644 index 0000000..74e9fa5 --- /dev/null +++ b/src/nixcp.rs @@ -0,0 +1,64 @@ +use std::sync::Arc; + +use crate::path_info::PathInfo; +use anyhow::{Context, Result}; +use log::info; +use tokio::sync::{Semaphore, mpsc}; +use url::Url; + +pub struct NixCp { + upstream_caches: Arc>, + store_paths: Vec, +} + +impl NixCp { + pub fn new() -> Self { + Self { + upstream_caches: vec![Url::parse("https://cache.nixos.org").unwrap()], + store_paths: Vec::new(), + } + } + + pub fn add_upstreams(&mut self, upstreams: &[String]) -> Result<()> { + self.upstream_caches.reserve(upstreams.len()); + for upstream in upstreams { + self.upstream_caches + .push(Url::parse(upstream).context(format!("failed to parse {upstream} as url"))?); + } + Ok(()) + } + + pub async fn paths_from_package(&mut self, package: &str) -> Result<()> { + let path_info = PathInfo::from_path(package).await?; + self.store_paths = path_info.get_closure().await?; + info!("found {} store paths", self.store_paths.len()); + + Ok(()) + } + + pub async fn run(&mut self) {} + + /// filter paths that are on upstream and send to `tx` + async fn filter_from_upstream(&self, tx: mpsc::Sender<&PathInfo>) { + let permits = Arc::new(Semaphore::new(10)); + let mut handles = Vec::new(); + for path in &self.store_paths { + if path.check_upstream_signature(&self.upstream_caches) { + continue; + } + let permits = permits.clone(); + let tx = tx.clone(); + handles.push(tokio::spawn(async move { + let _permit = permits.acquire().await.unwrap(); + + if !path.check_upstream_hit(&self.upstream_caches).await { + tx.send(path); + } + })); + } + + for handle in handles { + handle.await.unwrap(); + } + } +} diff --git a/src/path_info.rs b/src/path_info.rs new file mode 100644 index 0000000..1213c1b --- /dev/null +++ b/src/path_info.rs @@ -0,0 +1,155 @@ +use std::{collections::HashSet, path::Path}; + +use anyhow::{Context, Result}; +use log::trace; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use tokio::process::Command; +use url::Url; + +// nix path-info --derivation --json +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PathInfo { + deriver: String, + path: String, + signatures: Vec, +} +impl PathInfo { + /// get PathInfo for a package or a store path + pub async fn from_path(path: &str) -> Result { + let path_info = Command::new("nix") + .arg("path-info") + .arg("--json") + .arg(path) + .output() + .await + .context("`nix path-info` failed for {package}")?; + + Ok(serde_json::from_slice(&path_info.stdout)?) + } + + pub async fn get_closure(&self) -> Result> { + let nix_store_cmd = Command::new("nix-store") + .arg("--query") + .arg("--requisites") + .arg("--include-outputs") + .arg(&self.deriver) + .output() + .await + .expect("nix-store cmd failed"); + + let nix_store_paths = String::from_utf8(nix_store_cmd.stdout)?; + let nix_store_paths: Vec<&str> = nix_store_paths.lines().collect(); + let mut closure = Vec::with_capacity(nix_store_paths.len()); + for path in nix_store_paths { + closure.push(Self::from_path(path).await?); + } + Ok(closure) + } + + pub fn get_path(&self) -> &Path { + &Path::new(&self.path) + } + + /// checks if the path is signed by any upstream. if it is, we assume a cache hit. + /// the name of the cache in the signature does not have to be the domain of the cache. + /// in fact, it can be any random string. but, most often it is, and this saves us + /// a request. + pub fn check_upstream_signature(&self, upstreams: &[Url]) -> bool { + let upstreams: HashSet<_> = upstreams.iter().filter_map(|x| x.domain()).collect(); + + // some caches use names prefixed with - + // e.g. cache.nixos.org-1, nix-community.cachix.org-1 + let re = Regex::new(r"-\d+$").expect("regex should be valid"); + for signee in self.signees().iter().map(|&x| re.replace(x, "")) { + if upstreams.contains(signee.as_ref()) { + return true; + } + } + return false; + } + + fn signees(&self) -> Vec<&str> { + let signees: Vec<_> = self + .signatures + .iter() + .filter_map(|signature| Some(signature.split_once(":")?.0)) + .collect(); + trace!("signees for {}: {:?}", self.path, signees); + signees + } + + pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { + let basename = self.get_path().file_name().unwrap().to_str().unwrap(); + let hash = basename.split_once("-").unwrap().0; + + for upstream in upstreams { + let upstream = upstream + .join(format!("{hash}/.narinfo").as_str()) + .expect("adding .narinfo should make a valid url"); + let res_status = reqwest::Client::new() + .head(upstream.as_str()) + .send() + .await + .map(|x| x.status()); + + match &res_status { + Ok(status) => return status.is_success(), + Err(_) => return false, + } + } + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn get_signees_from_path_info() { + let path_info = PathInfo { + deriver: "".to_string(), + path: "".to_string(), + signatures: vec![ + "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), + "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), + ], + }; + let signees = path_info.signees(); + assert_eq!(signees, vec!["cache.nixos.org-1", "nixcache.cy7.sh"]); + } + + #[test] + fn match_upstream_cache_from_signature() { + let path_info = PathInfo { + deriver: "".to_string(), + path: "".to_string(), + signatures: vec![ + "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), + "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=".to_string(), + ], + }; + assert_eq!( + path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]), + true + ); + assert_eq!( + path_info.check_upstream_signature(&[Url::parse("https://nixcache.cy7.sh").unwrap()]), + true + ); + assert_eq!( + path_info.check_upstream_signature(&[ + Url::parse("https://nix-community.cachix.org").unwrap() + ]), + true + ); + assert_eq!( + path_info + .check_upstream_signature(&[Url::parse("https://fake-cache.cachix.org").unwrap()]), + false + ); + } +} From e58cf2bbd0451b598c5bc722b8ab3ce6bc8388e5 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 13 Apr 2025 03:27:55 -0400 Subject: [PATCH 02/20] maybe done refactoring but not tested --- src/main.rs | 137 +---------------------------------------------- src/nixcp.rs | 131 +++++++++++++++++++++++++++++++++----------- src/path_info.rs | 8 ++- 3 files changed, 110 insertions(+), 166 deletions(-) diff --git a/src/main.rs b/src/main.rs index 976efb9..e2773cc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,8 @@ #![feature(let_chains)] #![feature(extend_one)] -use std::path::Path; -use std::sync::{ - Arc, Mutex, RwLock, - atomic::{AtomicUsize, Ordering}, -}; - use anyhow::Result; use clap::{Parser, Subcommand}; -use log::{debug, trace}; -use tokio::process::Command; -use tokio::sync::{Semaphore, mpsc}; use nixcp::NixCp; @@ -60,138 +51,14 @@ enum Commands { async fn main() -> Result<()> { env_logger::init(); let cli = Cli::parse(); - let mut nixcp = NixCp::new(); - nixcp.add_upstreams(&cli.upstream_caches)?; + let nixcp = Box::leak(Box::new(NixCp::with_upstreams(&cli.upstream_caches)?)); match &cli.command { Commands::Push { package } => { nixcp.paths_from_package(package).await?; + nixcp.run().await; } } Ok(()) - - /* - let (cacheable_tx, mut cacheable_rx) = mpsc::channel(cli.uploader_concurrency.into()); - - println!("spawning check_upstream"); - - println!("spawning uploader"); - handles.push(tokio::spawn(async move { - uploader(&mut cacheable_rx, binary_cache, cli.uploader_concurrency).await; - })); - - // make sure all threads are done - for handle in handles { - handle.await.unwrap(); - } - */ -} - -// filter out store paths that exist in upstream caches -async fn check_upstream( - store_paths: Arc>>, - cacheable_tx: mpsc::Sender, - concurrency: u8, - upstream_caches: Arc>, -) { - let concurrency = Arc::new(Semaphore::new(concurrency.into())); - let c_store_paths = Arc::clone(&store_paths); - let store_paths = c_store_paths.read().unwrap().clone(); - - for store_path in store_paths { - let tx = cacheable_tx.clone(); - let upstream_caches = Arc::clone(&upstream_caches); - let concurrency = Arc::clone(&concurrency); - - tokio::spawn(async move { - let _permit = concurrency.acquire().await.unwrap(); - let basename = Path::new(&store_path) - .file_name() - .unwrap() - .to_str() - .unwrap() - .to_string(); - let hash = basename.split("-").next().unwrap(); - - let mut hit = false; - for upstream in upstream_caches.as_ref() { - let mut uri = upstream.clone(); - uri.push_str(format!("/{}.narinfo", hash).as_str()); - - let res_status = reqwest::Client::new() - .head(uri) - .send() - .await - .map(|x| x.status()); - - if let Ok(res_status) = res_status - && res_status.is_success() - { - debug!("{} was a hit upstream: {}", store_path, upstream); - hit = true; - break; - } - } - if !hit { - trace!("sending {}", store_path); - tx.send(store_path).await.unwrap(); - } - }); - } -} - -async fn uploader( - cacheable_rx: &mut mpsc::Receiver, - binary_cache: String, - concurrency: u8, -) { - let upload_count = Arc::new(AtomicUsize::new(0)); - let failures: Arc>> = Arc::new(Mutex::new(Vec::new())); - let concurrency = Arc::new(Semaphore::new(concurrency.into())); - let mut handles = Vec::new(); - - loop { - if let Some(path_to_upload) = cacheable_rx.recv().await { - let concurrency = Arc::clone(&concurrency); - let failures = Arc::clone(&failures); - let binary_cache = binary_cache.clone(); - let upload_count = Arc::clone(&upload_count); - - handles.push(tokio::spawn(async move { - let _permit = concurrency.acquire().await.unwrap(); - println!("uploading: {}", path_to_upload); - if Command::new("nix") - .arg("copy") - .arg("--to") - .arg(&binary_cache) - .arg(&path_to_upload) - .output() - .await - .is_err() - { - println!("WARN: upload failed: {}", path_to_upload); - failures.lock().unwrap().push(path_to_upload); - } else { - upload_count.fetch_add(1, Ordering::Relaxed); - } - })); - } else { - // make sure all threads are done - for handle in handles { - handle.await.unwrap(); - } - println!("uploaded {} paths", upload_count.load(Ordering::Relaxed)); - - let failures = failures.lock().unwrap(); - if !failures.is_empty() { - println!("failed to upload these paths: "); - for failure in failures.iter() { - print!("{}", failure); - } - println!(); - } - break; - } - } } diff --git a/src/nixcp.rs b/src/nixcp.rs index 74e9fa5..057af55 100644 --- a/src/nixcp.rs +++ b/src/nixcp.rs @@ -1,64 +1,135 @@ -use std::sync::Arc; +use std::{ + iter::once, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, +}; use crate::path_info::PathInfo; use anyhow::{Context, Result}; -use log::info; -use tokio::sync::{Semaphore, mpsc}; +use log::{info, warn}; +use tokio::{ + process::Command, + sync::{RwLock, Semaphore, mpsc}, +}; use url::Url; pub struct NixCp { upstream_caches: Arc>, - store_paths: Vec, + store_paths: Arc>>, } impl NixCp { - pub fn new() -> Self { - Self { - upstream_caches: vec![Url::parse("https://cache.nixos.org").unwrap()], - store_paths: Vec::new(), - } - } - - pub fn add_upstreams(&mut self, upstreams: &[String]) -> Result<()> { - self.upstream_caches.reserve(upstreams.len()); - for upstream in upstreams { - self.upstream_caches + pub fn with_upstreams(new_upstreams: &[String]) -> Result { + let mut upstreams = Vec::with_capacity(new_upstreams.len() + 1); + for upstream in new_upstreams + .iter() + .chain(once(&"https://cache.nixos.org".to_string())) + { + upstreams .push(Url::parse(upstream).context(format!("failed to parse {upstream} as url"))?); } - Ok(()) + Ok(Self { + upstream_caches: Arc::new(upstreams), + store_paths: Arc::new(RwLock::new(Vec::new())), + }) } pub async fn paths_from_package(&mut self, package: &str) -> Result<()> { let path_info = PathInfo::from_path(package).await?; - self.store_paths = path_info.get_closure().await?; - info!("found {} store paths", self.store_paths.len()); + self.store_paths + .write() + .await + .extend(path_info.get_closure().await?); + info!("found {} store paths", self.store_paths.read().await.len()); Ok(()) } - pub async fn run(&mut self) {} + pub async fn run(&'static self) { + let (tx, rx) = mpsc::channel(10); + let tx = Arc::new(tx); + tokio::spawn(self.filter_from_upstream(tx)); + tokio::spawn(self.uploader("".to_string(), rx)); + } /// filter paths that are on upstream and send to `tx` - async fn filter_from_upstream(&self, tx: mpsc::Sender<&PathInfo>) { + async fn filter_from_upstream(&self, tx: Arc>) { let permits = Arc::new(Semaphore::new(10)); - let mut handles = Vec::new(); - for path in &self.store_paths { + let mut handles = Vec::with_capacity(10); + let store_paths = self.store_paths.read().await.clone(); + + for path in store_paths.into_iter() { if path.check_upstream_signature(&self.upstream_caches) { continue; } - let permits = permits.clone(); - let tx = tx.clone(); - handles.push(tokio::spawn(async move { - let _permit = permits.acquire().await.unwrap(); + handles.push({ + let permits = permits.clone(); + let tx = tx.clone(); + let upstream_caches = self.upstream_caches.clone(); + tokio::spawn(async move { + let _permit = permits.acquire().await.unwrap(); - if !path.check_upstream_hit(&self.upstream_caches).await { - tx.send(path); - } - })); + if !path.check_upstream_hit(upstream_caches.as_slice()).await { + tx.send(path.to_string()).await.unwrap(); + } + }) + }); } for handle in handles { handle.await.unwrap(); } } + + async fn uploader(&self, cache: String, mut rx: mpsc::Receiver) { + let upload_count = Arc::new(AtomicUsize::new(0)); + let failures: Arc>> = Arc::new(Mutex::new(Vec::new())); + let permits = Arc::new(Semaphore::new(10)); + let mut handles = Vec::with_capacity(10); + + loop { + if let Some(path_to_upload) = rx.recv().await { + let permits = Arc::clone(&permits); + let failures = Arc::clone(&failures); + let binary_cache = cache.clone(); + let upload_count = Arc::clone(&upload_count); + + handles.push(tokio::spawn(async move { + let _permit = permits.acquire().await.unwrap(); + info!("uploading: {}", path_to_upload.to_string()); + if Command::new("nix") + .arg("copy") + .arg("--to") + .arg(&binary_cache) + .arg(&path_to_upload.to_string()) + .output() + .await + .is_err() + { + warn!("upload failed: {}", path_to_upload); + failures.lock().unwrap().push(path_to_upload); + } else { + upload_count.fetch_add(1, Ordering::Relaxed); + } + })); + } else { + // make sure all threads are done + for handle in handles { + handle.await.unwrap(); + } + println!("uploaded {} paths", upload_count.load(Ordering::Relaxed)); + + let failures = failures.lock().unwrap(); + if !failures.is_empty() { + warn!("failed to upload these paths: "); + for failure in failures.iter() { + warn!("{}", failure); + } + } + break; + } + } + } } diff --git a/src/path_info.rs b/src/path_info.rs index 1213c1b..e1cf25a 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -8,7 +8,7 @@ use tokio::process::Command; use url::Url; // nix path-info --derivation --json -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PathInfo { deriver: String, @@ -103,6 +103,12 @@ impl PathInfo { } } +impl ToString for PathInfo { + fn to_string(&self) -> String { + self.path.clone() + } +} + #[cfg(test)] mod tests { use super::*; From 681ee5e8269d89d6d253cb4f0c8f6816a0524b54 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 13 Apr 2025 14:52:12 -0400 Subject: [PATCH 03/20] make nar --- Cargo.lock | 508 +++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 4 + src/main.rs | 15 +- src/nixcp.rs | 2 +- src/path_info.rs | 26 +-- src/uploader.rs | 79 ++++++++ 6 files changed, 607 insertions(+), 27 deletions(-) create mode 100644 src/uploader.rs diff --git a/Cargo.lock b/Cargo.lock index 5cf46a7..0e910ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,6 +82,20 @@ version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +[[package]] +name = "async-compression" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a194f9d963d8099596278594b3107448656ba73831c9d8c783e613ce86da64" +dependencies = [ + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -115,12 +129,38 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" + [[package]] name = "bitflags" version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -139,6 +179,8 @@ version = "1.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -194,6 +236,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" version = "0.9.4" @@ -210,6 +258,78 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -221,6 +341,30 @@ dependencies = [ "syn", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -230,6 +374,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-primitive-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" +dependencies = [ + "num-traits", + "quote", + "syn", +] + [[package]] name = "env_filter" version = "0.1.3" @@ -275,6 +430,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "fnv" version = "1.0.7" @@ -305,6 +466,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -312,6 +488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -320,6 +497,34 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -338,10 +543,26 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", ] [[package]] @@ -373,6 +594,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "h2" version = "0.4.8" @@ -708,6 +935,16 @@ dependencies = [ "syn", ] +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.2", + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -724,6 +961,16 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libmimalloc-sys" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d6fac27761dabcd4ee73571cdb06b7022dc99089acbe5435691edffaac0f4" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.9.3" @@ -758,6 +1005,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mimalloc" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "995942f432bbb4822a7e9c3faa87a695185b0d09273ba85f097b54f4e458f2af" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" @@ -801,22 +1057,103 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix-compat" +version = "0.1.0" +source = "git+https://github.com/tvlfyi/tvix.git#ac27df9ae51f69c1b746b7c8c2ad22f5a857ca52" +dependencies = [ + "bitflags", + "bstr", + "bytes", + "data-encoding", + "ed25519", + "ed25519-dalek", + "enum-primitive-derive", + "futures", + "glob", + "mimalloc", + "nix-compat-derive", + "nom", + "num-traits", + "num_enum", + "pin-project-lite", + "serde", + "serde_json", + "sha2", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "nix-compat-derive" +version = "0.1.0" +source = "git+https://github.com/tvlfyi/tvix.git#ac27df9ae51f69c1b746b7c8c2ad22f5a857ca52" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "nixcp" version = "0.1.0" dependencies = [ "anyhow", + "async-compression", "clap", + "ed25519-dalek", "env_logger", "log", + "nix-compat", "regex", "reqwest", "serde", "serde_json", + "sha2", "tokio", "url", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "object" version = "0.36.7" @@ -917,6 +1254,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -938,6 +1285,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.94" @@ -962,6 +1318,15 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "redox_syscall" version = "0.5.10" @@ -1064,6 +1429,15 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.0.3" @@ -1166,6 +1540,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + [[package]] name = "serde" version = "1.0.219" @@ -1210,6 +1590,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1225,6 +1616,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core", +] + [[package]] name = "slab" version = "0.4.9" @@ -1250,6 +1650,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -1333,6 +1743,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -1405,6 +1835,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.22.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.5.2" @@ -1439,9 +1886,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.33" @@ -1457,6 +1916,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -1505,6 +1970,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" @@ -1791,6 +2262,15 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -1884,3 +2364,31 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index fe2d773..039a587 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,12 +5,16 @@ edition = "2024" [dependencies] anyhow = "1.0.97" +async-compression = { version = "0.4.22", features = ["tokio", "zstd"] } clap = { version = "4.5.34", features = ["derive"] } +ed25519-dalek = "2.1.1" env_logger = "0.11.7" log = "0.4.27" +nix-compat = { git = "https://github.com/tvlfyi/tvix.git", version = "0.1.0" } regex = "1.11.1" reqwest = "0.12.15" serde = { version = "1.0.219", features = [ "derive" ]} serde_json = "1.0.140" +sha2 = "0.10.8" tokio = { version = "1.44.1", features = [ "full" ]} url = { version = "2.5.4", features = [ "serde" ]} diff --git a/src/main.rs b/src/main.rs index e2773cc..5df47d8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ use nixcp::NixCp; mod cli; mod nixcp; mod path_info; +mod uploader; #[derive(Parser, Debug)] #[command(version, name = "nixcp")] @@ -21,21 +22,9 @@ struct Cli { to: String, /// Upstream cache to check against. Can be specified multiple times. - /// cache.nixos.org is always included (unless --no-nixos-cache is passed) + /// cache.nixos.org is always included #[arg(long = "upstream-cache", short)] upstream_caches: Vec, - - /// Concurrent upstream cache checkers - #[arg(long, default_value_t = 32)] - upstream_checker_concurrency: u8, - - /// Concurrent uploaders - #[arg(long, default_value_t = 4)] - uploader_concurrency: u8, - - /// Concurrent nix-store commands to run - #[arg(long, default_value_t = 32)] - nix_store_concurrency: u8, } #[derive(Debug, Subcommand)] diff --git a/src/nixcp.rs b/src/nixcp.rs index 057af55..01db3eb 100644 --- a/src/nixcp.rs +++ b/src/nixcp.rs @@ -72,7 +72,7 @@ impl NixCp { let _permit = permits.acquire().await.unwrap(); if !path.check_upstream_hit(upstream_caches.as_slice()).await { - tx.send(path.to_string()).await.unwrap(); + tx.send(path.absolute_path()).await.unwrap(); } }) }); diff --git a/src/path_info.rs b/src/path_info.rs index e1cf25a..0afb00c 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -2,6 +2,8 @@ use std::{collections::HashSet, path::Path}; use anyhow::{Context, Result}; use log::trace; +use nix_compat::nixhash::CAHash; +use nix_compat::store_path::StorePath; use regex::Regex; use serde::{Deserialize, Serialize}; use tokio::process::Command; @@ -11,9 +13,11 @@ use url::Url; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PathInfo { - deriver: String, - path: String, + pub deriver: StorePath, + pub path: StorePath, signatures: Vec, + pub references: Vec>, + pub ca: Option, } impl PathInfo { /// get PathInfo for a package or a store path @@ -34,7 +38,7 @@ impl PathInfo { .arg("--query") .arg("--requisites") .arg("--include-outputs") - .arg(&self.deriver) + .arg(self.deriver.to_string()) .output() .await .expect("nix-store cmd failed"); @@ -48,10 +52,6 @@ impl PathInfo { Ok(closure) } - pub fn get_path(&self) -> &Path { - &Path::new(&self.path) - } - /// checks if the path is signed by any upstream. if it is, we assume a cache hit. /// the name of the cache in the signature does not have to be the domain of the cache. /// in fact, it can be any random string. but, most often it is, and this saves us @@ -81,8 +81,8 @@ impl PathInfo { } pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { - let basename = self.get_path().file_name().unwrap().to_str().unwrap(); - let hash = basename.split_once("-").unwrap().0; + let hash = + String::from_utf8(self.path.digest().to_vec()).expect("should be a valid string"); for upstream in upstreams { let upstream = upstream @@ -101,14 +101,13 @@ impl PathInfo { } false } -} -impl ToString for PathInfo { - fn to_string(&self) -> String { - self.path.clone() + pub fn absolute_path(&self) -> String { + self.path.to_absolute_path() } } +/* #[cfg(test)] mod tests { use super::*; @@ -159,3 +158,4 @@ mod tests { ); } } +*/ diff --git a/src/uploader.rs b/src/uploader.rs new file mode 100644 index 0000000..04c4d2e --- /dev/null +++ b/src/uploader.rs @@ -0,0 +1,79 @@ +use anyhow::Result; +use async_compression::{Level, tokio::bufread::ZstdEncoder}; +use ed25519_dalek; +use nix_compat::{ + narinfo::{self, NarInfo}, + nixbase32, + store_path::StorePath, +}; +use sha2::{Digest, Sha256}; +use std::fs; +use tokio::{io::AsyncReadExt, process::Command}; + +use crate::path_info::PathInfo; + +pub struct Uploader { + signing_key: narinfo::SigningKey, + path: PathInfo, + compression: Option, +} + +impl Uploader { + pub fn new(key_file: &str, path: PathInfo) -> Result { + let key = fs::read_to_string(key_file)?; + let signing_key = narinfo::parse_keypair(key.as_str())?.0; + Ok(Self { + signing_key, + path, + // TODO: support other algorithms + compression: Some("zstd".to_string()), + }) + } + + pub async fn make_nar(&self) -> Result> { + Ok(Command::new("nix") + .arg("nar") + .arg("dump-path") + .arg(self.path.absolute_path()) + .output() + .await? + .stdout) + } + + pub fn narinfo_from_nar(&self, nar: &[u8]) -> Result { + let mut hasher = Sha256::new(); + hasher.update(nar); + let nar_hash: [u8; 32] = hasher.finalize().into(); + let nar_info = NarInfo { + flags: narinfo::Flags::empty(), + store_path: self.path.path.as_ref(), + nar_hash, + nar_size: nar.len() as u64, + references: self.path.references.iter().map(StorePath::as_ref).collect(), + signatures: Vec::new(), + ca: self.path.ca.clone(), + system: None, + deriver: Some(self.path.deriver.as_ref()), + compression: self.compression.as_ref().map(String::as_str), + file_hash: None, + file_size: None, + url: "", + }; + Ok(nar_info) + } + + fn nar_url(&self, compressed_nar_hash: &[u8]) -> String { + let compressed_nar_hash = nixbase32::encode(compressed_nar_hash); + format!("nar/{compressed_nar_hash}.nar.zst") + } + + async fn compress_nar(&self, nar: &[u8]) -> Vec { + let mut encoder = ZstdEncoder::with_quality(nar, Level::Default); + let mut compressed = Vec::with_capacity(nar.len()); + encoder + .read_to_end(&mut compressed) + .await + .expect("should compress just fine"); + compressed + } +} From 2c252a42c5d2d103adfd9e527d35aaeae92f2abd Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 13 Apr 2025 20:17:54 -0400 Subject: [PATCH 04/20] many changes --- Cargo.lock | 1257 ++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 + src/main.rs | 41 +- src/nixcp.rs | 110 ++-- src/path_info.rs | 39 +- src/uploader.rs | 133 ++++- 6 files changed, 1470 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e910ad..86744e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstream" version = "0.6.18" @@ -108,6 +114,445 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.3.1", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.82.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6eab2900764411ab01c8e91a76fd11a63b4e12bc3da97d9e14a0ce1343d86d3" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.63.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65d21e1ba6f2cdec92044f904356a19f5ad86961acf015741106cdfafd747c0" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "crc64fast-nvme", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.4.8", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.6.0", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.5", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.25", + "rustls-native-certs 0.8.1", + "rustls-pki-types", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0" +dependencies = [ + "aws-smithy-runtime-api", + "once_cell", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "once_cell", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.3.1", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -123,18 +568,63 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + [[package]] name = "bitflags" version = "2.9.0" @@ -173,6 +663,16 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.2.17" @@ -184,12 +684,32 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.34" @@ -230,6 +750,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -252,6 +781,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -267,6 +806,70 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crc64fast-nvme" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" +dependencies = [ + "crc", +] + +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -310,6 +913,16 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.9" @@ -320,6 +933,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -328,6 +950,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -341,14 +964,32 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "ed25519" version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -365,6 +1006,32 @@ dependencies = [ "zeroize", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff", + "generic-array", + "group", + "pkcs8 0.9.0", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -430,6 +1097,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -442,6 +1119,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -466,6 +1149,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -600,6 +1289,36 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.8" @@ -611,7 +1330,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.3.1", "indexmap", "slab", "tokio", @@ -624,6 +1343,11 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -631,6 +1355,41 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" @@ -642,6 +1401,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -649,7 +1419,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -660,8 +1430,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -671,6 +1441,36 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.6.0" @@ -680,9 +1480,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.8", + "http 1.3.1", + "http-body 1.0.1", "httparse", "itoa", "pin-project-lite", @@ -691,6 +1491,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.5" @@ -698,13 +1514,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.3.1", + "hyper 1.6.0", "hyper-util", - "rustls", + "rustls 0.23.25", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tower-service", ] @@ -716,7 +1533,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-util", "native-tls", "tokio", @@ -733,9 +1550,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", "libc", "pin-project-lite", "socket2", @@ -905,6 +1722,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -955,12 +1781,34 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libmimalloc-sys" version = "0.1.42" @@ -971,6 +1819,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.9.3" @@ -999,6 +1853,25 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -1020,6 +1893,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.5" @@ -1052,7 +1931,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -1073,7 +1952,7 @@ dependencies = [ "glob", "mimalloc", "nix-compat-derive", - "nom", + "nom 8.0.0", "num-traits", "num_enum", "pin-project-lite", @@ -1101,9 +1980,12 @@ version = "0.1.0" dependencies = [ "anyhow", "async-compression", + "aws-config", + "aws-sdk-s3", "clap", "ed25519-dalek", "env_logger", + "futures", "log", "nix-compat", "regex", @@ -1115,6 +1997,16 @@ dependencies = [ "url", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -1124,6 +2016,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1213,6 +2120,23 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -1254,14 +2178,24 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", +] + [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.9", + "spki 0.7.3", ] [[package]] @@ -1285,6 +2219,22 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "prettyplease" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -1359,6 +2309,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -1371,17 +2327,17 @@ version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.8", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.6.0", + "hyper-rustls 0.27.5", "hyper-tls", "hyper-util", "ipnet", @@ -1392,7 +2348,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "serde", "serde_json", "serde_urlencoded", @@ -1409,6 +2365,17 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.14" @@ -1429,6 +2396,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.1" @@ -1438,6 +2411,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.0.3" @@ -1447,23 +2433,69 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.9.3", "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "aws-lc-rs", "once_cell", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.1", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.2.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -1479,12 +2511,23 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1517,6 +2560,30 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -1524,7 +2591,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -1590,6 +2670,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" @@ -1616,6 +2707,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "signature" version = "2.2.0" @@ -1650,6 +2751,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -1657,7 +2768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.9", ] [[package]] @@ -1716,7 +2827,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -1739,7 +2850,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix", + "rustix 1.0.3", "windows-sys 0.59.0", ] @@ -1763,6 +2874,36 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -1812,13 +2953,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls", + "rustls 0.23.25", "tokio", ] @@ -1946,6 +3097,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -1964,6 +3121,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1976,6 +3139,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "want" version = "0.3.1" @@ -2081,6 +3250,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "windows-link" version = "0.1.1" @@ -2292,6 +3473,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index 039a587..71b2a41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,12 @@ edition = "2024" [dependencies] anyhow = "1.0.97" async-compression = { version = "0.4.22", features = ["tokio", "zstd"] } +aws-config = { version = "1.6.1", features = ["behavior-version-latest"] } +aws-sdk-s3 = "1.82.0" clap = { version = "4.5.34", features = ["derive"] } ed25519-dalek = "2.1.1" env_logger = "0.11.7" +futures = "0.3.31" log = "0.4.27" nix-compat = { git = "https://github.com/tvlfyi/tvix.git", version = "0.1.0" } regex = "1.11.1" diff --git a/src/main.rs b/src/main.rs index 5df47d8..c4e3d05 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ #![feature(let_chains)] #![feature(extend_one)] +#![feature(array_chunks)] -use anyhow::Result; +use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use nixcp::NixCp; @@ -17,14 +18,32 @@ struct Cli { #[command(subcommand)] command: Commands, - /// Address of the binary cache (passed to nix copy --to) - #[arg(long, value_name = "BINARY CACHE")] - to: String, + /// The s3 bucket to upload to + #[arg(long, value_name = "bucket name")] + bucket: String, /// Upstream cache to check against. Can be specified multiple times. /// cache.nixos.org is always included - #[arg(long = "upstream-cache", short)] - upstream_caches: Vec, + #[arg(long = "upstream", short, value_name = "nixcache.example.com")] + upstreams: Vec, + + /// Path to the file containing signing key + /// e.g. ~/cache-priv-key.pem + #[arg(long)] + signing_key: String, + + /// If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default + #[arg(long)] + region: Option, + + /// If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default + /// e.g. s3.example.com + #[arg(long)] + endpoint: Option, + + /// AWS profile to use + #[arg(long)] + profile: Option, } #[derive(Debug, Subcommand)] @@ -32,6 +51,7 @@ enum Commands { Push { /// Package or store path to upload /// e.g. nixpkgs#hello or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 + #[arg(value_name = "package or store path")] package: String, }, } @@ -40,12 +60,15 @@ enum Commands { async fn main() -> Result<()> { env_logger::init(); let cli = Cli::parse(); - let nixcp = Box::leak(Box::new(NixCp::with_upstreams(&cli.upstream_caches)?)); + let nixcp = Box::leak(Box::new(NixCp::new(&cli).await?)); match &cli.command { Commands::Push { package } => { - nixcp.paths_from_package(package).await?; - nixcp.run().await; + nixcp + .paths_from_package(package) + .await + .context("nixcp get paths from package")?; + nixcp.run().await.context("nixcp run")?; } } diff --git a/src/nixcp.rs b/src/nixcp.rs index 01db3eb..279c20d 100644 --- a/src/nixcp.rs +++ b/src/nixcp.rs @@ -1,4 +1,5 @@ use std::{ + fs, iter::once, sync::{ Arc, Mutex, @@ -6,56 +7,86 @@ use std::{ }, }; -use crate::path_info::PathInfo; use anyhow::{Context, Result}; -use log::{info, warn}; -use tokio::{ - process::Command, - sync::{RwLock, Semaphore, mpsc}, -}; +use aws_config::Region; +use aws_sdk_s3 as s3; +use futures::future::join_all; +use log::{debug, info, warn}; +use nix_compat::narinfo::{self, SigningKey}; +use tokio::sync::{RwLock, Semaphore, mpsc}; use url::Url; +use crate::{Cli, path_info::PathInfo, uploader::Uploader}; + pub struct NixCp { upstream_caches: Arc>, store_paths: Arc>>, + s3_client: s3::Client, + signing_key: SigningKey, + bucket: String, } impl NixCp { - pub fn with_upstreams(new_upstreams: &[String]) -> Result { - let mut upstreams = Vec::with_capacity(new_upstreams.len() + 1); - for upstream in new_upstreams + pub async fn new(cli: &Cli) -> Result { + let mut upstreams = Vec::with_capacity(cli.upstreams.len() + 1); + for upstream in cli + .upstreams .iter() .chain(once(&"https://cache.nixos.org".to_string())) { upstreams .push(Url::parse(upstream).context(format!("failed to parse {upstream} as url"))?); } + + let key = fs::read_to_string(&cli.signing_key)?; + let signing_key = narinfo::parse_keypair(key.as_str())?.0; + + let mut s3_config = aws_config::from_env(); + if let Some(region) = &cli.region { + s3_config = s3_config.region(Region::new(region.clone())); + } + if let Some(endpoint) = &cli.endpoint { + s3_config = s3_config.endpoint_url(endpoint); + } + if let Some(profile) = &cli.profile { + s3_config = s3_config.profile_name(profile); + } + + let s3_client = s3::Client::new(&s3_config.load().await); Ok(Self { upstream_caches: Arc::new(upstreams), store_paths: Arc::new(RwLock::new(Vec::new())), + s3_client, + signing_key, + bucket: cli.bucket.clone(), }) } pub async fn paths_from_package(&mut self, package: &str) -> Result<()> { - let path_info = PathInfo::from_path(package).await?; - self.store_paths - .write() + let path_info = PathInfo::from_path(package) .await - .extend(path_info.get_closure().await?); + .context("get path info for package")?; + debug!("path-info for {package}: {:?}", path_info); + self.store_paths.write().await.extend( + path_info + .get_closure() + .await + .context("closure from path info")?, + ); info!("found {} store paths", self.store_paths.read().await.len()); Ok(()) } - pub async fn run(&'static self) { + pub async fn run(&'static self) -> Result<()> { let (tx, rx) = mpsc::channel(10); let tx = Arc::new(tx); tokio::spawn(self.filter_from_upstream(tx)); - tokio::spawn(self.uploader("".to_string(), rx)); + self.upload(rx).await } /// filter paths that are on upstream and send to `tx` - async fn filter_from_upstream(&self, tx: Arc>) { + async fn filter_from_upstream(&self, tx: Arc>) { let permits = Arc::new(Semaphore::new(10)); let mut handles = Vec::with_capacity(10); let store_paths = self.store_paths.read().await.clone(); @@ -72,7 +103,7 @@ impl NixCp { let _permit = permits.acquire().await.unwrap(); if !path.check_upstream_hit(upstream_caches.as_slice()).await { - tx.send(path.absolute_path()).await.unwrap(); + tx.send(path).await.unwrap(); } }) }); @@ -83,42 +114,32 @@ impl NixCp { } } - async fn uploader(&self, cache: String, mut rx: mpsc::Receiver) { - let upload_count = Arc::new(AtomicUsize::new(0)); + async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { + let upload_count = AtomicUsize::new(0); let failures: Arc>> = Arc::new(Mutex::new(Vec::new())); let permits = Arc::new(Semaphore::new(10)); - let mut handles = Vec::with_capacity(10); + let mut uploads = Vec::with_capacity(10); loop { if let Some(path_to_upload) = rx.recv().await { let permits = Arc::clone(&permits); - let failures = Arc::clone(&failures); - let binary_cache = cache.clone(); - let upload_count = Arc::clone(&upload_count); + let absolute_path = path_to_upload.absolute_path(); - handles.push(tokio::spawn(async move { + info!("uploading: {}", absolute_path); + let uploader = Uploader::new( + &self.signing_key, + path_to_upload, + &self.s3_client, + self.bucket.clone(), + )?; + + let fut = tokio::spawn({ let _permit = permits.acquire().await.unwrap(); - info!("uploading: {}", path_to_upload.to_string()); - if Command::new("nix") - .arg("copy") - .arg("--to") - .arg(&binary_cache) - .arg(&path_to_upload.to_string()) - .output() - .await - .is_err() - { - warn!("upload failed: {}", path_to_upload); - failures.lock().unwrap().push(path_to_upload); - } else { - upload_count.fetch_add(1, Ordering::Relaxed); - } - })); + async move { uploader.upload().await } + }); + uploads.push(fut); } else { - // make sure all threads are done - for handle in handles { - handle.await.unwrap(); - } + join_all(uploads).await; println!("uploaded {} paths", upload_count.load(Ordering::Relaxed)); let failures = failures.lock().unwrap(); @@ -131,5 +152,6 @@ impl NixCp { break; } } + Ok(()) } } diff --git a/src/path_info.rs b/src/path_info.rs index 0afb00c..746c344 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,7 +1,7 @@ -use std::{collections::HashSet, path::Path}; +use std::collections::HashSet; -use anyhow::{Context, Result}; -use log::trace; +use anyhow::{Context, Error, Result}; +use log::{debug, error, trace}; use nix_compat::nixhash::CAHash; use nix_compat::store_path::StorePath; use regex::Regex; @@ -22,7 +22,8 @@ pub struct PathInfo { impl PathInfo { /// get PathInfo for a package or a store path pub async fn from_path(path: &str) -> Result { - let path_info = Command::new("nix") + debug!("query nix path-info for {path}"); + let nix_cmd = Command::new("nix") .arg("path-info") .arg("--json") .arg(path) @@ -30,15 +31,30 @@ impl PathInfo { .await .context("`nix path-info` failed for {package}")?; - Ok(serde_json::from_slice(&path_info.stdout)?) + // nix path-info returns an array with one element + match serde_json::from_slice::>(&nix_cmd.stdout) + .context("parse path info from stdout") + { + Ok(path_info) => path_info + .into_iter() + .next() + .ok_or_else(|| Error::msg("nix path-info returned empty")), + Err(e) => { + error!( + "Failed to parse data from `nix path-info`. The path may not exist on your system." + ); + Err(e) + } + } } pub async fn get_closure(&self) -> Result> { + debug!("query nix-store for {}", self.absolute_path()); let nix_store_cmd = Command::new("nix-store") .arg("--query") .arg("--requisites") .arg("--include-outputs") - .arg(self.deriver.to_string()) + .arg(self.absolute_path()) .output() .await .expect("nix-store cmd failed"); @@ -67,7 +83,7 @@ impl PathInfo { return true; } } - return false; + false } fn signees(&self) -> Vec<&str> { @@ -94,9 +110,8 @@ impl PathInfo { .await .map(|x| x.status()); - match &res_status { - Ok(status) => return status.is_success(), - Err(_) => return false, + if res_status.map(|code| code.is_success()).unwrap_or_default() { + return true; } } false @@ -105,6 +120,10 @@ impl PathInfo { pub fn absolute_path(&self) -> String { self.path.to_absolute_path() } + + pub fn digest(&self) -> &str { + str::from_utf8(self.path.digest()).expect("digest should be valid string") + } } /* diff --git a/src/uploader.rs b/src/uploader.rs index 04c4d2e..81f4be8 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -1,36 +1,139 @@ use anyhow::Result; use async_compression::{Level, tokio::bufread::ZstdEncoder}; -use ed25519_dalek; +use aws_sdk_s3::{ + self as s3, + types::{CompletedMultipartUpload, CompletedPart}, +}; +use futures::future::join_all; +use log::debug; use nix_compat::{ - narinfo::{self, NarInfo}, + narinfo::{self, NarInfo, SigningKey}, nixbase32, store_path::StorePath, }; use sha2::{Digest, Sha256}; -use std::fs; use tokio::{io::AsyncReadExt, process::Command}; use crate::path_info::PathInfo; -pub struct Uploader { - signing_key: narinfo::SigningKey, +const MULTIPART_CUTOFF: usize = 1024 * 1024 * 5; + +pub struct Uploader<'a> { + signing_key: &'a SigningKey, path: PathInfo, - compression: Option, + s3_client: &'a s3::Client, + bucket: String, } -impl Uploader { - pub fn new(key_file: &str, path: PathInfo) -> Result { - let key = fs::read_to_string(key_file)?; - let signing_key = narinfo::parse_keypair(key.as_str())?.0; +impl<'a> Uploader<'a> { + pub fn new( + signing_key: &'a SigningKey, + path: PathInfo, + s3_client: &'a s3::Client, + bucket: String, + ) -> Result { Ok(Self { signing_key, path, - // TODO: support other algorithms - compression: Some("zstd".to_string()), + s3_client, + bucket, }) } - pub async fn make_nar(&self) -> Result> { + pub async fn upload(&self) -> Result<()> { + let nar = self.make_nar().await?; + let mut nar_info = self.narinfo_from_nar(&nar)?; + let nar = self.compress_nar(&nar).await; + + // update fields that we know after compression + nar_info.file_size = Some(nar.len() as u64); + let mut hasher = Sha256::new(); + hasher.update(&nar); + nar_info.file_hash = Some(hasher.finalize().into()); + let nar_url = self.nar_url(&nar); + + if nar.len() < MULTIPART_CUTOFF { + let put_object = self + .s3_client + .put_object() + .bucket(&self.bucket) + .key(&nar_url) + .body(nar.into()) + .send() + .await?; + debug!("put object: {:#?}", put_object); + } else { + let multipart = self + .s3_client + .create_multipart_upload() + .bucket(&self.bucket) + .key(&nar_url) + .send() + .await?; + let upload_id = multipart.upload_id().unwrap(); + + let mut parts = Vec::with_capacity(nar.len() / MULTIPART_CUTOFF); + let chunks = nar.array_chunks::(); + for (i, chunk) in chunks.enumerate() { + parts.push(tokio::task::spawn( + self.s3_client + .upload_part() + .bucket(&self.bucket) + .key(&nar_url) + .upload_id(upload_id) + .part_number(i as i32 + 1) + .body(chunk.to_vec().into()) + .send(), + )); + } + + let completed_parts = join_all(parts) + .await + .into_iter() + .flatten() + .collect::, _>>()? + .into_iter() + .enumerate() + .map(|(i, part)| { + CompletedPart::builder() + .set_e_tag(part.e_tag().map(ToString::to_string)) + .set_part_number(Some(i as i32 + 1)) + .set_checksum_sha256(part.checksum_sha256().map(ToString::to_string)) + .build() + }) + .collect::>(); + + let completed_mp_upload = CompletedMultipartUpload::builder() + .set_parts(Some(completed_parts)) + .build(); + + let complete_mp_upload = self + .s3_client + .complete_multipart_upload() + .bucket(&self.bucket) + .key(&nar_url) + .upload_id(upload_id) + .multipart_upload(completed_mp_upload) + .send() + .await?; + + debug!("complete multipart upload: {:#?}", complete_mp_upload); + } + + nar_info.add_signature(self.signing_key); + + self.s3_client + .put_object() + .bucket(&self.bucket) + .key(format!("{}.narinfo", self.path.digest())) + .body(nar_info.to_string().as_bytes().to_vec().into()) + .send() + .await?; + + Ok(()) + } + + async fn make_nar(&self) -> Result> { Ok(Command::new("nix") .arg("nar") .arg("dump-path") @@ -40,7 +143,7 @@ impl Uploader { .stdout) } - pub fn narinfo_from_nar(&self, nar: &[u8]) -> Result { + fn narinfo_from_nar(&self, nar: &[u8]) -> Result { let mut hasher = Sha256::new(); hasher.update(nar); let nar_hash: [u8; 32] = hasher.finalize().into(); @@ -54,7 +157,7 @@ impl Uploader { ca: self.path.ca.clone(), system: None, deriver: Some(self.path.deriver.as_ref()), - compression: self.compression.as_ref().map(String::as_str), + compression: Some("zstd"), file_hash: None, file_size: None, url: "", From 202b222b83484fb95a28928ce9fdc670dce31b9a Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 13 Apr 2025 20:56:33 -0400 Subject: [PATCH 05/20] fix some silly mistakes --- src/path_info.rs | 11 ++++------- src/uploader.rs | 12 +++++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 746c344..e67dc4a 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -2,8 +2,8 @@ use std::collections::HashSet; use anyhow::{Context, Error, Result}; use log::{debug, error, trace}; -use nix_compat::nixhash::CAHash; use nix_compat::store_path::StorePath; +use nix_compat::{nixbase32, nixhash::CAHash}; use regex::Regex; use serde::{Deserialize, Serialize}; use tokio::process::Command; @@ -97,12 +97,9 @@ impl PathInfo { } pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { - let hash = - String::from_utf8(self.path.digest().to_vec()).expect("should be a valid string"); - for upstream in upstreams { let upstream = upstream - .join(format!("{hash}/.narinfo").as_str()) + .join(format!("{}/.narinfo", self.digest()).as_str()) .expect("adding .narinfo should make a valid url"); let res_status = reqwest::Client::new() .head(upstream.as_str()) @@ -121,8 +118,8 @@ impl PathInfo { self.path.to_absolute_path() } - pub fn digest(&self) -> &str { - str::from_utf8(self.path.digest()).expect("digest should be valid string") + pub fn digest(&self) -> String { + nixbase32::encode(self.path.digest()) } } diff --git a/src/uploader.rs b/src/uploader.rs index 81f4be8..90ce70b 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -49,8 +49,10 @@ impl<'a> Uploader<'a> { nar_info.file_size = Some(nar.len() as u64); let mut hasher = Sha256::new(); hasher.update(&nar); - nar_info.file_hash = Some(hasher.finalize().into()); - let nar_url = self.nar_url(&nar); + let hash: [u8; 32] = hasher.finalize().into(); + let nar_url = self.nar_url(&hash); + nar_info.file_hash = Some(hash); + debug!("uploading to bucket with key: {nar_url}"); if nar.len() < MULTIPART_CUTOFF { let put_object = self @@ -120,8 +122,6 @@ impl<'a> Uploader<'a> { debug!("complete multipart upload: {:#?}", complete_mp_upload); } - nar_info.add_signature(self.signing_key); - self.s3_client .put_object() .bucket(&self.bucket) @@ -147,7 +147,7 @@ impl<'a> Uploader<'a> { let mut hasher = Sha256::new(); hasher.update(nar); let nar_hash: [u8; 32] = hasher.finalize().into(); - let nar_info = NarInfo { + let mut nar_info = NarInfo { flags: narinfo::Flags::empty(), store_path: self.path.path.as_ref(), nar_hash, @@ -162,6 +162,8 @@ impl<'a> Uploader<'a> { file_size: None, url: "", }; + // signature consists of: store_path, nar_hash, nar_size, and references + nar_info.add_signature(self.signing_key); Ok(nar_info) } From b1134d5d6e8a5e33e538b8d5dc2e765557079420 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 13 Apr 2025 23:10:50 -0400 Subject: [PATCH 06/20] use tracing for logs --- Cargo.lock | 191 ++++++++++++++++++++++++++++++----------------- Cargo.toml | 4 +- src/main.rs | 5 +- src/nixcp.rs | 40 ++++++++-- src/path_info.rs | 2 +- src/uploader.rs | 2 +- 6 files changed, 163 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86744e5..901bf4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -647,7 +647,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", - "regex-automata", + "regex-automata 0.4.9", "serde", ] @@ -1052,29 +1052,6 @@ dependencies = [ "syn", ] -[[package]] -name = "env_filter" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -1737,30 +1714,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" -[[package]] -name = "jiff" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde", -] - -[[package]] -name = "jiff-static" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "jobserver" version = "0.1.33" @@ -1862,6 +1815,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1984,9 +1946,7 @@ dependencies = [ "aws-sdk-s3", "clap", "ed25519-dalek", - "env_logger", "futures", - "log", "nix-compat", "regex", "reqwest", @@ -1994,6 +1954,8 @@ dependencies = [ "serde_json", "sha2", "tokio", + "tracing", + "tracing-subscriber", "url", ] @@ -2016,6 +1978,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -2126,6 +2098,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "p256" version = "0.11.1" @@ -2204,21 +2182,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" -[[package]] -name = "portable-atomic" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" - -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -2294,8 +2257,17 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2306,7 +2278,7 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.5", ] [[package]] @@ -2315,6 +2287,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -2692,6 +2670,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -2874,6 +2861,16 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "time" version = "0.3.41" @@ -3059,6 +3056,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -3127,6 +3154,12 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3262,6 +3295,28 @@ dependencies = [ "rustix 0.38.44", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index 71b2a41..f7bc3f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,9 +10,7 @@ aws-config = { version = "1.6.1", features = ["behavior-version-latest"] } aws-sdk-s3 = "1.82.0" clap = { version = "4.5.34", features = ["derive"] } ed25519-dalek = "2.1.1" -env_logger = "0.11.7" futures = "0.3.31" -log = "0.4.27" nix-compat = { git = "https://github.com/tvlfyi/tvix.git", version = "0.1.0" } regex = "1.11.1" reqwest = "0.12.15" @@ -20,4 +18,6 @@ serde = { version = "1.0.219", features = [ "derive" ]} serde_json = "1.0.140" sha2 = "0.10.8" tokio = { version = "1.44.1", features = [ "full" ]} +tracing = "0.1.41" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"]} url = { version = "2.5.4", features = [ "serde" ]} diff --git a/src/main.rs b/src/main.rs index c4e3d05..45348b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; +use tracing_subscriber::{EnvFilter, FmtSubscriber}; use nixcp::NixCp; @@ -58,7 +59,9 @@ enum Commands { #[tokio::main] async fn main() -> Result<()> { - env_logger::init(); + let filter = EnvFilter::from_default_env(); + let subscriber = FmtSubscriber::builder().with_env_filter(filter).finish(); + tracing::subscriber::set_global_default(subscriber)?; let cli = Cli::parse(); let nixcp = Box::leak(Box::new(NixCp::new(&cli).await?)); diff --git a/src/nixcp.rs b/src/nixcp.rs index 279c20d..c0298bd 100644 --- a/src/nixcp.rs +++ b/src/nixcp.rs @@ -11,9 +11,9 @@ use anyhow::{Context, Result}; use aws_config::Region; use aws_sdk_s3 as s3; use futures::future::join_all; -use log::{debug, info, warn}; use nix_compat::narinfo::{self, SigningKey}; use tokio::sync::{RwLock, Semaphore, mpsc}; +use tracing::{debug, info, trace, warn}; use url::Url; use crate::{Cli, path_info::PathInfo, uploader::Uploader}; @@ -24,6 +24,10 @@ pub struct NixCp { s3_client: s3::Client, signing_key: SigningKey, bucket: String, + // paths that we skipped cause of a signature match + signature_hit_count: AtomicUsize, + // paths that we skipped cause we found it on an upstream + upstream_hit_count: AtomicUsize, } impl NixCp { @@ -59,6 +63,8 @@ impl NixCp { s3_client, signing_key, bucket: cli.bucket.clone(), + signature_hit_count: AtomicUsize::new(0), + upstream_hit_count: AtomicUsize::new(0), }) } @@ -81,18 +87,23 @@ impl NixCp { pub async fn run(&'static self) -> Result<()> { let (tx, rx) = mpsc::channel(10); let tx = Arc::new(tx); - tokio::spawn(self.filter_from_upstream(tx)); - self.upload(rx).await + let filter = tokio::spawn(self.filter_from_upstream(tx)); + let upload = tokio::spawn(self.upload(rx)); + filter.await?; + upload.await??; + Ok(()) } /// filter paths that are on upstream and send to `tx` - async fn filter_from_upstream(&self, tx: Arc>) { + async fn filter_from_upstream(&'static self, tx: Arc>) { let permits = Arc::new(Semaphore::new(10)); let mut handles = Vec::with_capacity(10); let store_paths = self.store_paths.read().await.clone(); for path in store_paths.into_iter() { if path.check_upstream_signature(&self.upstream_caches) { + trace!("skip {} (signature match)", path.absolute_path()); + self.signature_hit_count.fetch_add(1, Ordering::Release); continue; } handles.push({ @@ -104,14 +115,19 @@ impl NixCp { if !path.check_upstream_hit(upstream_caches.as_slice()).await { tx.send(path).await.unwrap(); + } else { + trace!("skip {} (upstream hit)", path.absolute_path()); + self.upstream_hit_count.fetch_add(1, Ordering::Relaxed); } }) }); } - for handle in handles { - handle.await.unwrap(); - } + join_all(handles) + .await + .into_iter() + .collect::>() + .unwrap(); } async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { @@ -140,7 +156,15 @@ impl NixCp { uploads.push(fut); } else { join_all(uploads).await; - println!("uploaded {} paths", upload_count.load(Ordering::Relaxed)); + println!("uploaded: {}", upload_count.load(Ordering::Relaxed)); + println!( + "skipped because of signature match: {}", + self.signature_hit_count.load(Ordering::Relaxed) + ); + println!( + "skipped because of upstream hit: {}", + self.upstream_hit_count.load(Ordering::Relaxed) + ); let failures = failures.lock().unwrap(); if !failures.is_empty() { diff --git a/src/path_info.rs b/src/path_info.rs index e67dc4a..57e0a5b 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,12 +1,12 @@ use std::collections::HashSet; use anyhow::{Context, Error, Result}; -use log::{debug, error, trace}; use nix_compat::store_path::StorePath; use nix_compat::{nixbase32, nixhash::CAHash}; use regex::Regex; use serde::{Deserialize, Serialize}; use tokio::process::Command; +use tracing::{debug, error, trace}; use url::Url; // nix path-info --derivation --json diff --git a/src/uploader.rs b/src/uploader.rs index 90ce70b..558e411 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -5,7 +5,6 @@ use aws_sdk_s3::{ types::{CompletedMultipartUpload, CompletedPart}, }; use futures::future::join_all; -use log::debug; use nix_compat::{ narinfo::{self, NarInfo, SigningKey}, nixbase32, @@ -13,6 +12,7 @@ use nix_compat::{ }; use sha2::{Digest, Sha256}; use tokio::{io::AsyncReadExt, process::Command}; +use tracing::debug; use crate::path_info::PathInfo; From 57a7ab944b4b98a548daf4267ef29bfe458b1578 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 13 Apr 2025 23:52:31 -0400 Subject: [PATCH 07/20] fix chunking bug --- src/main.rs | 4 ++-- src/nixcp.rs | 20 ++++++++------------ src/uploader.rs | 11 +++++++---- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/main.rs b/src/main.rs index 45348b3..3a7cc78 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,5 @@ #![feature(let_chains)] #![feature(extend_one)] -#![feature(array_chunks)] use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; @@ -38,7 +37,7 @@ struct Cli { region: Option, /// If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default - /// e.g. s3.example.com + /// e.g. https://s3.example.com #[arg(long)] endpoint: Option, @@ -62,6 +61,7 @@ async fn main() -> Result<()> { let filter = EnvFilter::from_default_env(); let subscriber = FmtSubscriber::builder().with_env_filter(filter).finish(); tracing::subscriber::set_global_default(subscriber)?; + let cli = Cli::parse(); let nixcp = Box::leak(Box::new(NixCp::new(&cli).await?)); diff --git a/src/nixcp.rs b/src/nixcp.rs index c0298bd..a215923 100644 --- a/src/nixcp.rs +++ b/src/nixcp.rs @@ -2,7 +2,7 @@ use std::{ fs, iter::once, sync::{ - Arc, Mutex, + Arc, atomic::{AtomicUsize, Ordering}, }, }; @@ -13,7 +13,7 @@ use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::narinfo::{self, SigningKey}; use tokio::sync::{RwLock, Semaphore, mpsc}; -use tracing::{debug, info, trace, warn}; +use tracing::{debug, info, trace}; use url::Url; use crate::{Cli, path_info::PathInfo, uploader::Uploader}; @@ -132,7 +132,6 @@ impl NixCp { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let upload_count = AtomicUsize::new(0); - let failures: Arc>> = Arc::new(Mutex::new(Vec::new())); let permits = Arc::new(Semaphore::new(10)); let mut uploads = Vec::with_capacity(10); @@ -155,7 +154,12 @@ impl NixCp { }); uploads.push(fut); } else { - join_all(uploads).await; + join_all(uploads) + .await + .into_iter() + .flatten() + .collect::>>()?; + println!("uploaded: {}", upload_count.load(Ordering::Relaxed)); println!( "skipped because of signature match: {}", @@ -165,14 +169,6 @@ impl NixCp { "skipped because of upstream hit: {}", self.upstream_hit_count.load(Ordering::Relaxed) ); - - let failures = failures.lock().unwrap(); - if !failures.is_empty() { - warn!("failed to upload these paths: "); - for failure in failures.iter() { - warn!("{}", failure); - } - } break; } } diff --git a/src/uploader.rs b/src/uploader.rs index 558e411..e52e4ae 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -46,13 +46,14 @@ impl<'a> Uploader<'a> { let nar = self.compress_nar(&nar).await; // update fields that we know after compression - nar_info.file_size = Some(nar.len() as u64); let mut hasher = Sha256::new(); hasher.update(&nar); let hash: [u8; 32] = hasher.finalize().into(); let nar_url = self.nar_url(&hash); nar_info.file_hash = Some(hash); - debug!("uploading to bucket with key: {nar_url}"); + nar_info.file_size = Some(nar.len() as u64); + nar_info.url = nar_url.as_str(); + debug!("uploading nar with key: {nar_url}"); if nar.len() < MULTIPART_CUTOFF { let put_object = self @@ -75,7 +76,7 @@ impl<'a> Uploader<'a> { let upload_id = multipart.upload_id().unwrap(); let mut parts = Vec::with_capacity(nar.len() / MULTIPART_CUTOFF); - let chunks = nar.array_chunks::(); + let chunks = nar.chunks(MULTIPART_CUTOFF); for (i, chunk) in chunks.enumerate() { parts.push(tokio::task::spawn( self.s3_client @@ -122,10 +123,12 @@ impl<'a> Uploader<'a> { debug!("complete multipart upload: {:#?}", complete_mp_upload); } + let narinfo_url = format!("{}.narinfo", self.path.digest()); + debug!("uploading narinfo with key {narinfo_url}"); self.s3_client .put_object() .bucket(&self.bucket) - .key(format!("{}.narinfo", self.path.digest())) + .key(narinfo_url) .body(nar_info.to_string().as_bytes().to_vec().into()) .send() .await?; From 9a4d237235b2d284ec982f6c283e6175f2487a4c Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 11:44:20 -0400 Subject: [PATCH 08/20] add check for already exist --- src/nixcp.rs | 30 +++++++++++++++++++++++------- src/path_info.rs | 14 +++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/nixcp.rs b/src/nixcp.rs index a215923..f5b3dce 100644 --- a/src/nixcp.rs +++ b/src/nixcp.rs @@ -19,7 +19,7 @@ use url::Url; use crate::{Cli, path_info::PathInfo, uploader::Uploader}; pub struct NixCp { - upstream_caches: Arc>, + upstream_caches: Vec, store_paths: Arc>>, s3_client: s3::Client, signing_key: SigningKey, @@ -28,6 +28,8 @@ pub struct NixCp { signature_hit_count: AtomicUsize, // paths that we skipped cause we found it on an upstream upstream_hit_count: AtomicUsize, + // paths that we skipped cause they are already on our cache + already_exists_count: AtomicUsize, } impl NixCp { @@ -58,13 +60,14 @@ impl NixCp { let s3_client = s3::Client::new(&s3_config.load().await); Ok(Self { - upstream_caches: Arc::new(upstreams), + upstream_caches: upstreams, store_paths: Arc::new(RwLock::new(Vec::new())), s3_client, signing_key, bucket: cli.bucket.clone(), signature_hit_count: AtomicUsize::new(0), upstream_hit_count: AtomicUsize::new(0), + already_exists_count: AtomicUsize::new(0), }) } @@ -86,7 +89,6 @@ impl NixCp { pub async fn run(&'static self) -> Result<()> { let (tx, rx) = mpsc::channel(10); - let tx = Arc::new(tx); let filter = tokio::spawn(self.filter_from_upstream(tx)); let upload = tokio::spawn(self.upload(rx)); filter.await?; @@ -95,7 +97,7 @@ impl NixCp { } /// filter paths that are on upstream and send to `tx` - async fn filter_from_upstream(&'static self, tx: Arc>) { + async fn filter_from_upstream(&'static self, tx: mpsc::Sender) { let permits = Arc::new(Semaphore::new(10)); let mut handles = Vec::with_capacity(10); let store_paths = self.store_paths.read().await.clone(); @@ -109,12 +111,22 @@ impl NixCp { handles.push({ let permits = permits.clone(); let tx = tx.clone(); - let upstream_caches = self.upstream_caches.clone(); tokio::spawn(async move { let _permit = permits.acquire().await.unwrap(); - if !path.check_upstream_hit(upstream_caches.as_slice()).await { - tx.send(path).await.unwrap(); + if !path + .check_upstream_hit(self.upstream_caches.as_slice()) + .await + { + if path + .check_if_already_exists(&self.s3_client, self.bucket.clone()) + .await + { + trace!("skip {} (already exists)", path.absolute_path()); + self.already_exists_count.fetch_add(1, Ordering::Relaxed); + } else { + tx.send(path).await.unwrap(); + } } else { trace!("skip {} (upstream hit)", path.absolute_path()); self.upstream_hit_count.fetch_add(1, Ordering::Relaxed); @@ -169,6 +181,10 @@ impl NixCp { "skipped because of upstream hit: {}", self.upstream_hit_count.load(Ordering::Relaxed) ); + println!( + "skipped because already exist: {}", + self.already_exists_count.load(Ordering::Relaxed) + ); break; } } diff --git a/src/path_info.rs b/src/path_info.rs index 57e0a5b..2bf05ea 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,6 +1,7 @@ use std::collections::HashSet; use anyhow::{Context, Error, Result}; +use aws_sdk_s3 as s3; use nix_compat::store_path::StorePath; use nix_compat::{nixbase32, nixhash::CAHash}; use regex::Regex; @@ -99,8 +100,9 @@ impl PathInfo { pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { for upstream in upstreams { let upstream = upstream - .join(format!("{}/.narinfo", self.digest()).as_str()) + .join(format!("{}.narinfo", self.digest()).as_str()) .expect("adding .narinfo should make a valid url"); + debug!("querying {}", upstream); let res_status = reqwest::Client::new() .head(upstream.as_str()) .send() @@ -121,6 +123,16 @@ impl PathInfo { pub fn digest(&self) -> String { nixbase32::encode(self.path.digest()) } + + pub async fn check_if_already_exists(&self, s3_client: &s3::Client, bucket: String) -> bool { + !s3_client + .head_object() + .bucket(bucket) + .key(format!("{}.narinfo", self.digest())) + .send() + .await + .is_err() + } } /* From 48e44628fb3246cf06947295604d041791a00597 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 12:08:43 -0400 Subject: [PATCH 09/20] update readme --- README.md | 52 +++++++++++++++++++++++++++++++++------------------- src/main.rs | 2 +- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 9265a87..ff71137 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,38 @@ -Runs `nix copy` under the hood but only uploads paths that don't exist in upstream caches. It's async so may also be somewhat faster. Unlike `nix copy`, we also upload build dependencies. You may also pass the `--recursive` flag to absolutely not miss anything (be warned though, it queues up a lot of paths to check against upstream caches (also idk why you'd ever want to use this honestly)). Specify upstream caches to check against with `--upstream-cache` (can be specified multiple times, `cache.nixos.org` is always included). +Uploads stuff to your s3 binary cache, but skip stuff that exist on upstream caches to save you space and time. Unlike `nix copy`, we also upload build dependencies meaning you just say the package or store path and we figure out the rest. Specify upstream caches to check against with `-u` (can be specified multiple times, `cache.nixos.org` is always included). + +## Usage + +Example: +``` +nixcp --bucket nixcache --signing-key ~/cache-priv-key.pem --endpoint https://s3.cy7.sh -u https://nix-community.cachix.org push github:cything/nixcp/2025-04-12 +``` +The signing key is generated with: +``` +nix-store --generate-binary-cache-key nixcache.cy7.sh cache-priv-key.pem cache-pub-key.pem +``` + +`AWS_ACCESS_KEY_ID` and `AWS_ENDPOINT_URL` environment variables should be set with your s3 credentials. ``` -Usage: nixcp [OPTIONS] --to +Usage: nixcp [OPTIONS] --bucket --signing-key -Arguments: - Package to upload to the binary cache +Commands: + push + help Print this message or the help of the given subcommand(s) Options: - --to - Address of the binary cache (passed to nix copy --to) - -u, --upstream-cache + --bucket + The s3 bucket to upload to + -u, --upstream Upstream cache to check against. Can be specified multiple times. cache.nixos.org is always included - -r, --recursive - Whether to pass --recursive to nix path-info. Can queue a huge number of paths to upload - --upstream-checker-concurrency - Concurrent upstream cache checkers [default: 32] - --uploader-concurrency - Concurrent uploaders [default: 16] - --nix-store-concurrency - Concurrent nix-store commands to run [default: 32] + --signing-key + Path to the file containing signing key e.g. ~/cache-priv-key.pem + --region + If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default + --endpoint + If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default e.g. https://s3.example.com + --profile + AWS profile to use -h, --help Print help -V, --version @@ -27,13 +41,13 @@ Options: ## Install with nix ``` -nix profile install git+https://git.cy7.sh/cy/nixcp.git +nix profile install github:cything/nixcp ``` Or run without installing: ``` -nix run git+https://git.cy7.sh/cy/nixcp.git +nix run github:cything/nixcp ``` Separate arguments with `--` to pass them through to `nixcp` like so: ``` -nix run git+https://git.cy7.sh/cy/nixcp.git -- --help -``` \ No newline at end of file +nix run github:cything/nixcp -- --help +``` diff --git a/src/main.rs b/src/main.rs index 3a7cc78..f6f10df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,7 +23,7 @@ struct Cli { bucket: String, /// Upstream cache to check against. Can be specified multiple times. - /// cache.nixos.org is always included + /// cache.nixos.org is always included. #[arg(long = "upstream", short, value_name = "nixcache.example.com")] upstreams: Vec, From 2f9cc77b4288b1db936c3f5f474111b9ab6dc1ab Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 13:20:09 -0400 Subject: [PATCH 10/20] refactor cli and clap stuff --- src/main.rs | 45 +++++++++++++++++++++++---------------- src/{nixcp.rs => push.rs} | 8 +++---- 2 files changed, 31 insertions(+), 22 deletions(-) rename src/{nixcp.rs => push.rs} (97%) diff --git a/src/main.rs b/src/main.rs index f6f10df..cf78e63 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,22 +2,34 @@ #![feature(extend_one)] use anyhow::{Context, Result}; -use clap::{Parser, Subcommand}; +use clap::{Args, Parser, Subcommand}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; -use nixcp::NixCp; +use push::Push; mod cli; -mod nixcp; mod path_info; +mod push; mod uploader; #[derive(Parser, Debug)] -#[command(version, name = "nixcp")] +#[command(version)] +#[command(name = "nixcp")] +#[command(about = "Upload store paths to a s3 binary cache")] +#[command(long_about = None)] struct Cli { #[command(subcommand)] command: Commands, +} +#[derive(Debug, Subcommand)] +enum Commands { + #[command(arg_required_else_help = true)] + Push(PushArgs), +} + +#[derive(Debug, Args)] +pub struct PushArgs { /// The s3 bucket to upload to #[arg(long, value_name = "bucket name")] bucket: String, @@ -44,16 +56,14 @@ struct Cli { /// AWS profile to use #[arg(long)] profile: Option, -} -#[derive(Debug, Subcommand)] -enum Commands { - Push { - /// Package or store path to upload - /// e.g. nixpkgs#hello or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 - #[arg(value_name = "package or store path")] - package: String, - }, + #[arg(long)] + skip_signature_check: bool, + + /// Package or store path to upload + /// e.g. nixpkgs#hello or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 + #[arg(value_name = "package or store path")] + package: String, } #[tokio::main] @@ -63,15 +73,14 @@ async fn main() -> Result<()> { tracing::subscriber::set_global_default(subscriber)?; let cli = Cli::parse(); - let nixcp = Box::leak(Box::new(NixCp::new(&cli).await?)); match &cli.command { - Commands::Push { package } => { - nixcp - .paths_from_package(package) + Commands::Push(cli) => { + let push = Box::leak(Box::new(Push::new(cli).await?)); + push.paths_from_package(&cli.package) .await .context("nixcp get paths from package")?; - nixcp.run().await.context("nixcp run")?; + push.run().await.context("nixcp run")?; } } diff --git a/src/nixcp.rs b/src/push.rs similarity index 97% rename from src/nixcp.rs rename to src/push.rs index f5b3dce..074324c 100644 --- a/src/nixcp.rs +++ b/src/push.rs @@ -16,9 +16,9 @@ use tokio::sync::{RwLock, Semaphore, mpsc}; use tracing::{debug, info, trace}; use url::Url; -use crate::{Cli, path_info::PathInfo, uploader::Uploader}; +use crate::{PushArgs, path_info::PathInfo, uploader::Uploader}; -pub struct NixCp { +pub struct Push { upstream_caches: Vec, store_paths: Arc>>, s3_client: s3::Client, @@ -32,8 +32,8 @@ pub struct NixCp { already_exists_count: AtomicUsize, } -impl NixCp { - pub async fn new(cli: &Cli) -> Result { +impl Push { + pub async fn new(cli: &PushArgs) -> Result { let mut upstreams = Vec::with_capacity(cli.upstreams.len() + 1); for upstream in cli .upstreams From f304dae207bdd8eea1e0769577aca8251f8fe332 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 13:27:00 -0400 Subject: [PATCH 11/20] just a little --- src/push.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/push.rs b/src/push.rs index 074324c..5adab30 100644 --- a/src/push.rs +++ b/src/push.rs @@ -160,11 +160,10 @@ impl Push { self.bucket.clone(), )?; - let fut = tokio::spawn({ + uploads.push(tokio::spawn(async move { let _permit = permits.acquire().await.unwrap(); - async move { uploader.upload().await } - }); - uploads.push(fut); + uploader.upload().await + })); } else { join_all(uploads) .await From b47a778b9eeee968c53f103e67344548ed2d032d Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 13:28:04 -0400 Subject: [PATCH 12/20] ok clippy --- src/path_info.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 2bf05ea..44beac1 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -125,13 +125,13 @@ impl PathInfo { } pub async fn check_if_already_exists(&self, s3_client: &s3::Client, bucket: String) -> bool { - !s3_client + s3_client .head_object() .bucket(bucket) .key(format!("{}.narinfo", self.digest())) .send() .await - .is_err() + .is_ok() } } From e5a7dfa68370e28699663966c5a81c20e78d7f9c Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 13:56:28 -0400 Subject: [PATCH 13/20] fix pathinfo tests --- src/path_info.rs | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 44beac1..7debc33 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -135,36 +135,33 @@ impl PathInfo { } } -/* #[cfg(test)] mod tests { use super::*; #[test] fn get_signees_from_path_info() { - let path_info = PathInfo { - deriver: "".to_string(), - path: "".to_string(), - signatures: vec![ - "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), - "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), - ], - }; + let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#; + let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); + + path_info.signatures = vec![ + "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), + "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), + ]; let signees = path_info.signees(); assert_eq!(signees, vec!["cache.nixos.org-1", "nixcache.cy7.sh"]); } #[test] fn match_upstream_cache_from_signature() { - let path_info = PathInfo { - deriver: "".to_string(), - path: "".to_string(), - signatures: vec![ - "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), - "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), - "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=".to_string(), - ], - }; + let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#; + let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); + + path_info.signatures = vec![ + "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), + "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=".to_string(), + ]; assert_eq!( path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]), true @@ -186,4 +183,3 @@ mod tests { ); } } -*/ From abdff736f5fd05ae5f0b0e231ff3c9781b540021 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 15:11:01 -0400 Subject: [PATCH 14/20] aws doesn't use defaults --- src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index cf78e63..57d3340 100644 --- a/src/main.rs +++ b/src/main.rs @@ -44,11 +44,11 @@ pub struct PushArgs { #[arg(long)] signing_key: String, - /// If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default + /// If unspecified, will get it form AWS_DEFAULT_REGION envar #[arg(long)] region: Option, - /// If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default + /// If unspecifed, will get it from AWS_ENDPOINT_URL envar /// e.g. https://s3.example.com #[arg(long)] endpoint: Option, From 5111af1d34f3f80c2792f8b237d600f32cb09f6e Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 16:32:12 -0400 Subject: [PATCH 15/20] add a trace --- src/path_info.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/path_info.rs b/src/path_info.rs index 7debc33..24969c9 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -32,6 +32,11 @@ impl PathInfo { .await .context("`nix path-info` failed for {package}")?; + trace!( + "nix path-info output: {}", + String::from_utf8_lossy(&nix_cmd.stdout) + ); + // nix path-info returns an array with one element match serde_json::from_slice::>(&nix_cmd.stdout) .context("parse path info from stdout") From df8d3a0bc6f0441713480c7fbfbd30d004a916c8 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 16:47:59 -0400 Subject: [PATCH 16/20] don't do anything cahash cause serde fails to parse sometimes --- src/path_info.rs | 3 +-- src/uploader.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 24969c9..77a8cdb 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -2,8 +2,8 @@ use std::collections::HashSet; use anyhow::{Context, Error, Result}; use aws_sdk_s3 as s3; +use nix_compat::nixbase32; use nix_compat::store_path::StorePath; -use nix_compat::{nixbase32, nixhash::CAHash}; use regex::Regex; use serde::{Deserialize, Serialize}; use tokio::process::Command; @@ -18,7 +18,6 @@ pub struct PathInfo { pub path: StorePath, signatures: Vec, pub references: Vec>, - pub ca: Option, } impl PathInfo { /// get PathInfo for a package or a store path diff --git a/src/uploader.rs b/src/uploader.rs index e52e4ae..0a25cff 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -157,7 +157,7 @@ impl<'a> Uploader<'a> { nar_size: nar.len() as u64, references: self.path.references.iter().map(StorePath::as_ref).collect(), signatures: Vec::new(), - ca: self.path.ca.clone(), + ca: None, system: None, deriver: Some(self.path.deriver.as_ref()), compression: Some("zstd"), From 6b959f583ba6ec51778b4397906039d654af60bb Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 14 Apr 2025 19:29:33 -0400 Subject: [PATCH 17/20] make deriver optional cause it won't be there if path is a derivation --- src/path_info.rs | 2 +- src/uploader.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 77a8cdb..91c6612 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -14,7 +14,7 @@ use url::Url; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PathInfo { - pub deriver: StorePath, + pub deriver: Option>, pub path: StorePath, signatures: Vec, pub references: Vec>, diff --git a/src/uploader.rs b/src/uploader.rs index 0a25cff..b0520ac 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -159,7 +159,7 @@ impl<'a> Uploader<'a> { signatures: Vec::new(), ca: None, system: None, - deriver: Some(self.path.deriver.as_ref()), + deriver: self.path.deriver.as_ref().map(|x| x.as_ref()), compression: Some("zstd"), file_hash: None, file_size: None, From 307109d9f15d4433e24bbebcb2d9ebc37b394a48 Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 14:13:15 -0400 Subject: [PATCH 18/20] we don't ever serialize pathinfo --- src/path_info.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 91c6612..9694ddd 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -5,14 +5,13 @@ use aws_sdk_s3 as s3; use nix_compat::nixbase32; use nix_compat::store_path::StorePath; use regex::Regex; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use tokio::process::Command; use tracing::{debug, error, trace}; use url::Url; // nix path-info --derivation --json -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, Clone, Deserialize)] pub struct PathInfo { pub deriver: Option>, pub path: StorePath, From c37a05ba9ff6a65be786e2a7ef2c231b59efc604 Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 14:40:18 -0400 Subject: [PATCH 19/20] make signatures optional when deserializing --- src/path_info.rs | 56 +++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 9694ddd..bcae0ae 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -15,7 +15,7 @@ use url::Url; pub struct PathInfo { pub deriver: Option>, pub path: StorePath, - signatures: Vec, + signatures: Option>, pub references: Vec>, } impl PathInfo { @@ -91,13 +91,15 @@ impl PathInfo { } fn signees(&self) -> Vec<&str> { - let signees: Vec<_> = self - .signatures - .iter() - .filter_map(|signature| Some(signature.split_once(":")?.0)) - .collect(); - trace!("signees for {}: {:?}", self.path, signees); - signees + if let Some(signatures) = self.signatures.as_ref() { + let signees: Vec<_> = signatures + .iter() + .filter_map(|signature| Some(signature.split_once(":")?.0)) + .collect(); + trace!("signees for {}: {:?}", self.path, signees); + return signees; + } + Vec::new() } pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { @@ -147,10 +149,10 @@ mod tests { let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#; let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); - path_info.signatures = vec![ + path_info.signatures = Some(vec![ "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), - ]; + ]); let signees = path_info.signees(); assert_eq!(signees, vec!["cache.nixos.org-1", "nixcache.cy7.sh"]); } @@ -160,29 +162,35 @@ mod tests { let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#; let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); - path_info.signatures = vec![ + path_info.signatures = Some(vec![ "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=".to_string(), - ]; - assert_eq!( - path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]), - true + ]); + assert!( + path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]) ); - assert_eq!( - path_info.check_upstream_signature(&[Url::parse("https://nixcache.cy7.sh").unwrap()]), - true + assert!( + path_info.check_upstream_signature(&[Url::parse("https://nixcache.cy7.sh").unwrap()]) ); - assert_eq!( + assert!( path_info.check_upstream_signature(&[ Url::parse("https://nix-community.cachix.org").unwrap() - ]), - true + ]) ); - assert_eq!( - path_info + assert!( + !path_info .check_upstream_signature(&[Url::parse("https://fake-cache.cachix.org").unwrap()]), - false + ); + } + + #[test] + fn path_info_without_signature() { + let path_info_json = r#"{"ca":"fixed:r:sha256:1q10p04pgx9sk6xbvrkn4nvh0ys2lzplgcni5368f4z3cr8ikbmz","narHash":"sha256-v64ZUWbjE4fMKNGyR++nQnsAtyV25r26mTr1dwm4IOA=","narSize":5520,"path":"/nix/store/gj6hz9mj23v01yvq1nn5f655jrcky1qq-nixos-option.nix","references":[],"registrationTime":1744740942,"valid":true}"#; + let path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); + + assert!( + !path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]) ); } } From d4cac6524719fb8c8bf1435e822c4d2a419a512d Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 16:52:26 -0400 Subject: [PATCH 20/20] use lix --- src/path_info.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/path_info.rs b/src/path_info.rs index bcae0ae..0f05a87 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -18,11 +18,17 @@ pub struct PathInfo { signatures: Option>, pub references: Vec>, } + impl PathInfo { - /// get PathInfo for a package or a store path + // get PathInfo for a package or a store path + // we deserialize this as an array of `PathInfo` below pub async fn from_path(path: &str) -> Result { debug!("query nix path-info for {path}"); + // use lix cause nix would return a json map instead of an array let nix_cmd = Command::new("nix") + .arg("run") + .arg("github:nixos/nixpkgs/nixos-unstable#lix") + .arg("--") .arg("path-info") .arg("--json") .arg(path) @@ -193,4 +199,15 @@ mod tests { !path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]) ); } + + /* + #[test] + fn path_info_deserialize_nix_map() { + let path_info_json = r#"{"/nix/store/8vm1jxsc0jphd65vb7r6g5ysgqw0yh9f-home-manager-generation":{"ca":null,"deriver":"/nix/store/h8z25s6arcrns5nmrq1yhgbamywjivpn-home-manager-generation.drv","narHash":"sha256-o4qwqyJ5UVm9cyC/nBNcNYVnIM14Pewgw7fou+wUVSY=","narSize":13608,"references":["/nix/store/40yifhx34v4g4llrdn3v2ag8w02j10fv-gnugrep-3.11","/nix/store/4d0ix5djms3n2jnjdc58l916cwack1rp-empty-directory","/nix/store/56zmgla8443qfrkrh2ch0vz0mh8ywrw1-home-manager-files","/nix/store/58br4vk3q5akf4g8lx0pqzfhn47k3j8d-bash-5.2p37","/nix/store/80l1sb3vcmrkcdd7ihlizkcnv19rq9fj-ncurses-6.5","/nix/store/8vm1jxsc0jphd65vb7r6g5ysgqw0yh9f-home-manager-generation","/nix/store/92as847i10kl6s19fi910ddyk9l83835-check-link-targets.sh","/nix/store/9c90iz95yynyh3vsc67zndch6j01vgz3-home-manager-path","/nix/store/b2cfj7yk3wfg1jdwjzim7306hvsc5gnl-systemd-257.3","/nix/store/bm5fi6wj0w4r2wjll2448k307bzfcjwx-cleanup","/nix/store/c244fsb3a7i5837lzn94m4bmav9i5p9b-link","/nix/store/cvlbhhrvzfkjl2hrrzhq3vr5gzan1r60-bash-interactive-5.2p37","/nix/store/gpxsdrrd4x93fs75395vr2dfys1ki9mq-jq-1.7.1-bin","/nix/store/jlf743lqxbvad6dbgndsgqfg20m2np5i-sd-switch-0.5.3","/nix/store/mhmgm739aagj4x7hr6ag2wjmxhmpy8mf-gettext-0.22.5","/nix/store/w9db12j05yv5hl31s6jndd9cfm1g1gw4-hm-modules-messages","/nix/store/wj1c3gsiajabnq50ifxqnlv60i5rhqj7-diffutils-3.10","/nix/store/xhql0ilzbiqwnmz4z8y0phk611wynxf2-gnused-4.9","/nix/store/xq5f95pp297afc2xjgrmhmf9w631qp7m-findutils-4.10.0","/nix/store/yh6qg1nsi5h2xblcr67030pz58fsaxx3-coreutils-9.6","/nix/store/zhrjg6wxrxmdlpn6iapzpp2z2vylpvw5-home-manager.sh"],"registrationTime":1744742989,"signatures":["nixcache.cy7.sh:Vq4X95kSzum7BwrBhjmmM2yVipfBI3AE3jgZ3b3RoYrP4/ghotbDdlwCvwK3qx4BQdEOLSgrC1tDwiMNb6oRBw=="],"ultimate":false}}"#; + serde_json::from_str::>(path_info_json).expect("must serialize"); + + let path_info_json = r#"{"/nix/store/3a2ahdaprw6df0lml1pj9jhbi038dsjh-nixos-system-chunk-25.05.20250412.2631b0b":{"ca":null,"deriver":"/nix/store/12ssi931481jlkizgfk1c1jnawvwjbhh-nixos-system-chunk-25.05.20250412.2631b0b.drv","narHash":"sha256-CHhBIzMD4v/FKqKgGroq0UC1k3GrK5lcNwQPMpv2xLc=","narSize":20704,"references":["/nix/store/0yjiyixxsr137iw93hnaacdsssy1li9h-switch-to-configuration-0.1.0","/nix/store/14rby7cpwrzjsjym44cl5h6nj6qpn1gs-etc","/nix/store/3a2ahdaprw6df0lml1pj9jhbi038dsjh-nixos-system-chunk-25.05.20250412.2631b0b","/nix/store/3wjljpj30fvv2cdb60apr4126pa5bm87-shadow-4.17.2","/nix/store/40yifhx34v4g4llrdn3v2ag8w02j10fv-gnugrep-3.11","/nix/store/58br4vk3q5akf4g8lx0pqzfhn47k3j8d-bash-5.2p37","/nix/store/5dyh8l59kfvf89zjkbmjfnx7fix93n4f-net-tools-2.10","/nix/store/aq9wdsz12bg9252790l9awiry2bml4ls-sops-install-secrets-0.0.1","/nix/store/b00kq6fjhgisdrykg621vml8505nnmb3-users-groups.json","/nix/store/b2cfj7yk3wfg1jdwjzim7306hvsc5gnl-systemd-257.3","/nix/store/bfr68wi6k8icb3j9fy3fzchva56djfhd-mounts.sh","/nix/store/cjnihsds5hhnji9r85hglph07q9y9hgc-system-path","/nix/store/cvlbhhrvzfkjl2hrrzhq3vr5gzan1r60-bash-interactive-5.2p37","/nix/store/f9jll96j74f5ykvs062718b98lfjbn9g-util-linux-2.40.4-bin","/nix/store/h7zih134d3n5yk8pnhv1fa38n6qkyrn2-pre-switch-checks","/nix/store/idn5n51246piyxcr3v6gxnj5a5l9mzpn-linux-6.14.2","/nix/store/ipn5793y61x2904xqnkgbjnp91svjjzx-perl-5.40.0-env","/nix/store/j1rikvl25pz0b5ham1ijq0nbg1q2fqfy-initrd-linux-6.14.2","/nix/store/jgawnqyh6piwcl79gxpmq5czx9rfr9xh-glibc-locales-2.40-66","/nix/store/jqgmcv8j4gj59218hcbiyn8z951rycdj-install-grub.sh","/nix/store/kpmybhxy3gz6k1znbdirwsp3c6wvsgg9-manifest.json","/nix/store/lgainx4gl6q7mhiwmls81d3n51p5jz7z-linux-6.14.2-modules","/nix/store/mhxn5kwnri3z9hdzi3x0980id65p0icn-lib.sh","/nix/store/n8n0faszqlnf3mdg0fj6abnknrhjsw5j-perl-5.40.0-env","/nix/store/nq61v7a601gjndijq5nndprkzpwz4q9g-glibc-2.40-66-bin","/nix/store/nx27idxpvi3fk3p7admvhipny73nr25n-kmod-31","/nix/store/pggww1d2pg24fcg5v36xn63n53vanyyi-gnupg-2.4.7","/nix/store/rg5rf512szdxmnj9qal3wfdnpfsx38qi-setup-etc.pl","/nix/store/vvlfaafnz3pdhw7lx5kc5gb9pl4zhz5l-local-cmds","/nix/store/w142vx7ij1fz6qwhp5dprkf59cizvv1v-update-users-groups.pl","/nix/store/xq5f95pp297afc2xjgrmhmf9w631qp7m-findutils-4.10.0","/nix/store/yh6qg1nsi5h2xblcr67030pz58fsaxx3-coreutils-9.6","/nix/store/zlsmh0ccgvncg30qb4y0mp5pahnk1wnw-append-initrd-secrets","/nix/store/zs07icpv5ykf8m36xcv717hh26bp09fa-firmware","/nix/store/zy2n4id5gcxcbx2x8jbblkmcpdlpsypk-getent-glibc-2.40-66"],"registrationTime":1744743136,"signatures":["nixcache.cy7.sh:dZ1XiKQNe0fRX48gBj03PIABYJGV6BPwb72YpMqEBONZMF+JrkVKhRCF0ur/4Bf5prHxg6Qfg1ytP/4csRC9DQ=="],"ultimate":false}}"#; + serde_json::from_str::>(path_info_json).expect("must serialize"); + } + */ }