From a17fa92c78a6355adb70c9645627d9f42dc1a7c4 Mon Sep 17 00:00:00 2001 From: cy Date: Thu, 17 Apr 2025 22:25:36 -0400 Subject: [PATCH 01/44] some progress at an attempt --- src/bindings/mod.rs | 7 +++++++ src/bindings/nix.cpp | 11 +++++++++++ src/bindings/nix.hpp | 1 + src/store.rs | 20 +++++++++++++++++++- src/uploader.rs | 10 ---------- 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 61a32af..701a15e 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -211,6 +211,13 @@ mod ffi { /// Obtains a handle to the Nix store. fn open_nix_store() -> Result>; + /// Creates a NAR dump from a path. + fn nar_from_path( + self: Pin<&mut CNixStore>, + base_name: Vec, + sender: Box, + ) -> Result<()>; + // ========= // CPathInfo // ========= diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp index 3914de1..326e878 100644 --- a/src/bindings/nix.cpp +++ b/src/bindings/nix.cpp @@ -108,6 +108,17 @@ std::unique_ptr> CNixStore::compute_fs_closure(RBasePat return std::make_unique>(result); } +void CNixStore::nar_from_path(RVec base_name, RBox sender) { + RustSink sink(std::move(sender)); + + std::string_view sv((const char *)base_name.data(), base_name.size()); + nix::StorePath store_path(sv); + + // exceptions will be thrown into Rust + this->store->narFromPath(store_path, sink); + sink.eof(); +} + std::unique_ptr open_nix_store() { return std::make_unique(); } diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp index 9f4964a..5c79a33 100644 --- a/src/bindings/nix.hpp +++ b/src/bindings/nix.hpp @@ -79,6 +79,7 @@ public: bool flip_direction, bool include_outputs, bool include_derivers); + void nar_from_path(RVec base_name, RBox sender); }; std::unique_ptr open_nix_store(); diff --git a/src/store.rs b/src/store.rs index 4499243..9aa5b44 100644 --- a/src/store.rs +++ b/src/store.rs @@ -4,7 +4,10 @@ use anyhow::{Context, Result}; use nix_compat::store_path::StorePath; use tokio::task; -use crate::{bindings, path_info::PathInfo}; +use crate::{ + bindings::{self, AsyncWriteAdapter}, + path_info::PathInfo, +}; pub struct Store { inner: Arc, @@ -75,4 +78,19 @@ impl Store { .await .unwrap() } + + pub fn make_nar(&self, path: StorePath) -> AsyncWriteAdapter { + let inner = self.inner.clone(); + let (adapter, mut sender) = AsyncWriteAdapter::new(); + + task::spawn_blocking(move || { + if let Err(e) = inner + .store() + .nar_from_path(path.to_string().as_bytes().to_vec(), sender.clone()) + { + let _ = sender.rust_error(e); + } + }); + adapter + } } diff --git a/src/uploader.rs b/src/uploader.rs index eb955a2..f77a3d3 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -139,16 +139,6 @@ impl<'a> Uploader<'a> { Ok(()) } - async fn make_nar(&self) -> Result> { - Ok(Command::new("nix") - .arg("nar") - .arg("dump-path") - .arg(self.path.absolute_path()) - .output() - .await? - .stdout) - } - fn narinfo_from_nar(&self, nar: &[u8]) -> Result { let mut hasher = Sha256::new(); hasher.update(nar); From 4808671071555f50e979e2cbf35295b16cd259ca Mon Sep 17 00:00:00 2001 From: cy Date: Fri, 18 Apr 2025 21:23:52 -0400 Subject: [PATCH 02/44] some progress on using nix-compat for nar creation --- Cargo.lock | 4 ++ Cargo.toml | 4 ++ src/store.rs | 2 +- src/uploader.rs | 115 +++++++++++++++++++++++++++++++++++------------- 4 files changed, 94 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9fa7a11..2608cf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2228,6 +2228,7 @@ dependencies = [ "async-compression", "aws-config", "aws-sdk-s3", + "bytes", "clap", "console-subscriber", "cxx", @@ -2241,7 +2242,10 @@ dependencies = [ "serde", "serde_json", "sha2", + "tempfile", "tokio", + "tokio-stream", + "tokio-util", "tracing", "url", ] diff --git a/Cargo.toml b/Cargo.toml index ee646cf..2430fdd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,10 @@ tracing = "0.1.41" url = { version = "2.5.4", features = [ "serde" ]} cxx = "1.0" console-subscriber = "0.4.1" +bytes = "1.10.1" +tokio-stream = { version = "0.1.17", features = ["fs"] } +tempfile = "3.19.1" +tokio-util = { version = "0.7.14", features = ["io"] } [build-dependencies] cxx-build = "1.0" diff --git a/src/store.rs b/src/store.rs index 9aa5b44..0433362 100644 --- a/src/store.rs +++ b/src/store.rs @@ -79,7 +79,7 @@ impl Store { .unwrap() } - pub fn make_nar(&self, path: StorePath) -> AsyncWriteAdapter { + pub fn stream_nar(&self, path: StorePath) -> AsyncWriteAdapter { let inner = self.inner.clone(); let (adapter, mut sender) = AsyncWriteAdapter::new(); diff --git a/src/uploader.rs b/src/uploader.rs index f77a3d3..390c3ac 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -1,20 +1,30 @@ +use std::{collections::BTreeMap, os::unix::fs::PermissionsExt, path::PathBuf}; + use anyhow::Result; use async_compression::{Level, tokio::bufread::ZstdEncoder}; use aws_sdk_s3::{ self as s3, types::{CompletedMultipartUpload, CompletedPart}, }; -use futures::future::join_all; +use bytes::{BufMut, Bytes, BytesMut}; +use futures::{future::join_all, stream::TryStreamExt}; use nix_compat::{ + nar::writer::r#async as nar, narinfo::{self, NarInfo, SigningKey}, nixbase32, store_path::StorePath, }; use sha2::{Digest, Sha256}; -use tokio::{io::AsyncReadExt, process::Command}; +use tokio::{ + fs::{File, read_dir, read_link}, + io::{AsyncRead, BufReader}, + pin, +}; +use tokio_stream::wrappers::ReadDirStream; +use tokio_util::io::InspectReader; use tracing::debug; -use crate::path_info::PathInfo; +use crate::{bindings::AsyncWriteAdapter, path_info::PathInfo, store::Store}; const MULTIPART_CUTOFF: usize = 1024 * 1024 * 5; @@ -23,7 +33,7 @@ pub struct Uploader<'a> { path: PathInfo, s3_client: &'a s3::Client, bucket: String, - hash: Sha256, + store: &'a Store, } impl<'a> Uploader<'a> { @@ -32,38 +42,28 @@ impl<'a> Uploader<'a> { path: PathInfo, s3_client: &'a s3::Client, bucket: String, + store: &'a Store, ) -> Result { Ok(Self { signing_key, path, s3_client, bucket, - hash: Sha256::new(), + store, }) } pub async fn upload(&self) -> Result<()> { - let nar = self.make_nar().await?; - let mut nar_info = self.narinfo_from_nar(&nar)?; - let nar = self.compress_nar(&nar).await; + let mut nar_temp = File::open(tempfile::Builder::new().tempfile()?.path()).await?; + self.make_nar(&mut nar_temp); - // update fields that we know after compression - let mut hasher = Sha256::new(); - hasher.update(&nar); - let hash: [u8; 32] = hasher.finalize().into(); - let nar_url = self.nar_url(&hash); - nar_info.file_hash = Some(hash); - nar_info.file_size = Some(nar.len() as u64); - nar_info.url = nar_url.as_str(); - debug!("uploading nar with key: {nar_url}"); - - if nar.len() < MULTIPART_CUTOFF { + if first_chunk.len() < MULTIPART_CUTOFF { let put_object = self .s3_client .put_object() .bucket(&self.bucket) .key(&nar_url) - .body(nar.into()) + .body(first_chunk.into()) .send() .await?; debug!("put object: {:#?}", put_object); @@ -164,17 +164,72 @@ impl<'a> Uploader<'a> { } fn nar_url(&self, compressed_nar_hash: &[u8]) -> String { - let compressed_nar_hash = nixbase32::encode(compressed_nar_hash); - format!("nar/{compressed_nar_hash}.nar.zst") + format!("nar/{}.nar.zst", nixbase32::encode(compressed_nar_hash)) } - async fn compress_nar(&self, nar: &[u8]) -> Vec { - let mut encoder = ZstdEncoder::with_quality(nar, Level::Default); - let mut compressed = Vec::with_capacity(nar.len()); - encoder - .read_to_end(&mut compressed) - .await - .expect("should compress just fine"); - compressed + async fn make_nar(&self, sink: &mut File) -> Result<()> { + let nar = nar::open(sink).await?; + let path = self.path.absolute_path(); + let metadata = File::open(&path).await?.metadata().await?; + + if metadata.is_symlink() { + let target = read_link(&path).await?; + nar.symlink(target.as_os_str().as_encoded_bytes()).await; + } else if metadata.is_dir() { + let mut nar = nar.directory().await?; + nar_from_dir(path.into(), &mut nar).await; + nar.close().await; + } else if metadata.is_file() { + let perms = metadata.permissions().mode(); + let mut executable = false; + if (perms & 0o700) == 0o700 { + executable = true; + } + + let mut file = BufReader::new(File::open(&path).await?); + nar.file(executable, metadata.len(), &mut file).await; + } + + Ok(()) } } + +async fn nar_from_dir(path: PathBuf, node: &mut nar::Directory<'_, '_>) -> Result<()> { + let root = ReadDirStream::new(read_dir(&path).await?); + let entries = root + .map_ok(|x| (x.file_name(), x)) + .try_collect::>() + .await?; + + // directory entries must be written in ascending order of name + for (name, entry) in entries.iter() { + let node = node.entry(name.as_encoded_bytes()).await?; + let metadata = entry.metadata().await?; + + if metadata.is_symlink() { + let target = read_link(entry.path()).await?; + node.symlink(target.as_os_str().as_encoded_bytes()).await; + } else if metadata.is_dir() { + let mut node = node.directory().await?; + Box::pin(nar_from_dir(entry.path(), &mut node)).await; + node.close().await; + } else if metadata.is_file() { + let perms = metadata.permissions().mode(); + let mut executable = false; + if (perms & 0o700) == 0o700 { + executable = true; + } + + let mut file = BufReader::new(File::open(entry.path()).await?); + node.file(executable, metadata.len(), &mut file).await; + } + } + Ok(()) +} + +async fn compress_and_hash_nar(nar: File, nar_hasher: &mut Sha256) -> impl AsyncRead { + let nar_reader = InspectReader::new(nar, |x| nar_hasher.update(x)); + let nar_buf_reader = BufReader::new(nar_reader); + + ZstdEncoder::with_quality(nar_buf_reader, Level::Default) +} From ac4b2ba136e7fa82336aa91c2088ff407e49e603 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 00:39:22 -0400 Subject: [PATCH 03/44] fancy nar cooking --- src/push.rs | 2 +- src/uploader.rs | 43 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/push.rs b/src/push.rs index 8177864..18f74b9 100644 --- a/src/push.rs +++ b/src/push.rs @@ -160,7 +160,7 @@ impl Push { } async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { - let mut uploads = Vec::with_capacity(10); + let mut uploads = Vec::new(); loop { if let Some(path_to_upload) = rx.recv().await { diff --git a/src/uploader.rs b/src/uploader.rs index 390c3ac..8f4efaa 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -21,7 +21,7 @@ use tokio::{ pin, }; use tokio_stream::wrappers::ReadDirStream; -use tokio_util::io::InspectReader; +use tokio_util::io::{InspectReader, read_buf}; use tracing::debug; use crate::{bindings::AsyncWriteAdapter, path_info::PathInfo, store::Store}; @@ -55,7 +55,16 @@ impl<'a> Uploader<'a> { pub async fn upload(&self) -> Result<()> { let mut nar_temp = File::open(tempfile::Builder::new().tempfile()?.path()).await?; - self.make_nar(&mut nar_temp); + self.make_nar(&mut nar_temp).await; + + // this goes to the .narinfo file + let mut nar_hasher = Sha256::new(); + // this is the URL for file .narinfo points to + let mut file_hasher = Sha256::new(); + let nar_reader = compress_and_hash_nar(nar_temp, &mut nar_hasher, &mut file_hasher); + + let buf = BytesMut::with_capacity(MULTIPART_CUTOFF); + let if first_chunk.len() < MULTIPART_CUTOFF { let put_object = self @@ -163,10 +172,6 @@ impl<'a> Uploader<'a> { Ok(nar_info) } - fn nar_url(&self, compressed_nar_hash: &[u8]) -> String { - format!("nar/{}.nar.zst", nixbase32::encode(compressed_nar_hash)) - } - async fn make_nar(&self, sink: &mut File) -> Result<()> { let nar = nar::open(sink).await?; let path = self.path.absolute_path(); @@ -227,9 +232,29 @@ async fn nar_from_dir(path: PathBuf, node: &mut nar::Directory<'_, '_>) -> Resul Ok(()) } -async fn compress_and_hash_nar(nar: File, nar_hasher: &mut Sha256) -> impl AsyncRead { - let nar_reader = InspectReader::new(nar, |x| nar_hasher.update(x)); +fn compress_and_hash_nar( + nar_file: File, + nar_hasher: &mut Sha256, + compressed_nar_hasher: &mut Sha256, +) -> impl AsyncRead { + let nar_reader = InspectReader::new(nar_file, |x| nar_hasher.update(x)); let nar_buf_reader = BufReader::new(nar_reader); - ZstdEncoder::with_quality(nar_buf_reader, Level::Default) + let compressed_nar_reader = ZstdEncoder::with_quality(nar_buf_reader, Level::Default); + InspectReader::new(compressed_nar_reader, |x| compressed_nar_hasher.update(x)) +} + +fn nar_url(compressed_nar_hash: &[u8]) -> String { + format!("nar/{}.nar.zst", nixbase32::encode(compressed_nar_hash)) +} + +async fn read_buf_nar(stream: &mut S, mut buf: BytesMut) -> Result { + while buf.len() < buf.capacity() { + let n = read_buf(stream, &mut buf).await?; + + if n == 0 { + break; + } + } + Ok(buf.freeze()) } From 81ce855dae7cf803111f8543e12e61cde09a2898 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 12:37:07 -0400 Subject: [PATCH 04/44] refactor and bunch more improvements; use object_store for s3 --- Cargo.lock | 1454 +++++++++++----------------------------------- Cargo.toml | 7 +- flake.nix | 1 + src/main.rs | 10 +- src/make_nar.rs | 92 +++ src/path_info.rs | 21 +- src/push.rs | 49 +- src/store.rs | 4 +- src/uploader.rs | 217 ++----- 9 files changed, 535 insertions(+), 1320 deletions(-) create mode 100644 src/make_nar.rs diff --git a/Cargo.lock b/Cargo.lock index 9fa7a11..bbd7534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,10 +27,19 @@ dependencies = [ ] [[package]] -name = "allocator-api2" -version = "0.2.21" +name = "android-tzdata" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] [[package]] name = "anstream" @@ -147,445 +156,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "aws-config" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "hex", - "http 1.3.1", - "ring", - "time", - "tokio", - "tracing", - "url", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "zeroize", -] - -[[package]] -name = "aws-lc-rs" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" -dependencies = [ - "bindgen", - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "aws-runtime" -version = "1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-s3" -version = "1.82.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6eab2900764411ab01c8e91a76fd11a63b4e12bc3da97d9e14a0ce1343d86d3" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "fastrand", - "hex", - "hmac", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "lru", - "once_cell", - "percent-encoding", - "regex-lite", - "sha2", - "tracing", - "url", -] - -[[package]] -name = "aws-sdk-sso" -version = "1.64.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-ssooidc" -version = "1.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "1.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" -dependencies = [ - "aws-credential-types", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "crypto-bigint 0.5.5", - "form_urlencoded", - "hex", - "hmac", - "http 0.2.12", - "http 1.3.1", - "once_cell", - "p256", - "percent-encoding", - "ring", - "sha2", - "subtle", - "time", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-async" -version = "1.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.63.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65d21e1ba6f2cdec92044f904356a19f5ad86961acf015741106cdfafd747c0" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc32c", - "crc32fast", - "crc64fast-nvme", - "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", - "pin-project-lite", - "sha1", - "sha2", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.60.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http-client" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "h2 0.4.9", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper 1.6.0", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.5", - "hyper-util", - "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.26", - "rustls-native-certs 0.8.1", - "rustls-pki-types", - "tokio", - "tower 0.5.2", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-observability" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0" -dependencies = [ - "aws-smithy-runtime-api", - "once_cell", -] - -[[package]] -name = "aws-smithy-query" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-runtime" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-client", - "aws-smithy-observability", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "http-body 1.0.1", - "once_cell", - "pin-project-lite", - "pin-utils", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "1.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "bytes", - "http 0.2.12", - "http 1.3.1", - "pin-project-lite", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-types" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" -dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "itoa", - "num-integer", - "pin-project-lite", - "pin-utils", - "ryu", - "serde", - "time", - "tokio", - "tokio-util", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.60.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "1.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "rustc_version", - "tracing", -] - [[package]] name = "axum" version = "0.7.9" @@ -596,8 +166,8 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "itoa", "matchit", @@ -622,8 +192,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", @@ -648,12 +218,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "base16ct" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" - [[package]] name = "base64" version = "0.21.7" @@ -666,45 +230,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] - [[package]] name = "base64ct" version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn", - "which", -] - [[package]] name = "bitflags" version = "2.9.0" @@ -749,16 +280,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "cc" version = "1.2.19" @@ -770,15 +291,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom 7.1.3", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -786,14 +298,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "clang-sys" -version = "1.8.1" +name = "cfg_aliases" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ - "glob", - "libc", - "libloading", + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-link", ] [[package]] @@ -836,15 +356,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" -[[package]] -name = "cmake" -version = "0.1.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" -dependencies = [ - "cc", -] - [[package]] name = "codespan-reporting" version = "0.12.0" @@ -942,30 +453,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.4.2" @@ -975,15 +462,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crc64fast-nvme" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" -dependencies = [ - "crc", -] - [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -999,28 +477,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array", - "rand_core", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "rand_core", - "subtle", -] - [[package]] name = "crypto-common" version = "0.1.6" @@ -1123,16 +579,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid", - "zeroize", -] - [[package]] name = "der" version = "0.7.9" @@ -1143,15 +589,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "deranged" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" -dependencies = [ - "powerfmt", -] - [[package]] name = "digest" version = "0.10.7" @@ -1160,7 +597,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -1174,32 +610,14 @@ dependencies = [ "syn", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "ecdsa" -version = "0.14.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" -dependencies = [ - "der 0.6.1", - "elliptic-curve", - "rfc6979", - "signature 1.6.4", -] - [[package]] name = "ed25519" version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", + "pkcs8", + "signature", ] [[package]] @@ -1222,26 +640,6 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" -[[package]] -name = "elliptic-curve" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" -dependencies = [ - "base16ct", - "crypto-bigint 0.4.9", - "der 0.6.1", - "digest", - "ff", - "generic-array", - "group", - "pkcs8 0.9.0", - "rand_core", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "encoding_rs" version = "0.8.35" @@ -1284,16 +682,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "ff" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" -dependencies = [ - "rand_core", - "subtle", -] - [[package]] name = "fiat-crypto" version = "0.2.9" @@ -1346,12 +734,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.31" @@ -1458,8 +840,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1469,9 +853,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1486,36 +872,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" -[[package]] -name = "group" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" -dependencies = [ - "ff", - "rand_core", - "subtle", -] - -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.9.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.9" @@ -1527,7 +883,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.3.1", + "http", "indexmap 2.9.0", "slab", "tokio", @@ -1546,11 +902,6 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] [[package]] name = "hdrhistogram" @@ -1571,41 +922,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.3.1" @@ -1617,17 +933,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -1635,7 +940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http", ] [[package]] @@ -1646,8 +951,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -1669,30 +974,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.6.0" @@ -1702,9 +983,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.9", - "http 1.3.1", - "http-body 1.0.1", + "h2", + "http", + "http-body", "httparse", "httpdate", "itoa", @@ -1714,22 +995,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "rustls-native-certs 0.6.3", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.5" @@ -1737,14 +1002,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http 1.3.1", - "hyper 1.6.0", + "http", + "hyper", "hyper-util", - "rustls 0.23.26", - "rustls-native-certs 0.8.1", + "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tower-service", ] @@ -1754,7 +1019,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.6.0", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -1769,7 +1034,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.6.0", + "hyper", "hyper-util", "native-tls", "tokio", @@ -1786,9 +1051,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "hyper 1.6.0", + "http", + "http-body", + "hyper", "libc", "pin-project-lite", "socket2", @@ -1797,6 +1062,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1970,9 +1259,9 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] @@ -2009,28 +1298,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" -[[package]] -name = "libloading" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" -dependencies = [ - "cfg-if", - "windows-targets 0.52.6", -] - [[package]] name = "libmimalloc-sys" version = "0.1.42" @@ -2050,12 +1323,6 @@ dependencies = [ "cc", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -2084,15 +1351,6 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.2", -] - [[package]] name = "matchers" version = "0.1.0" @@ -2226,8 +1484,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-compression", - "aws-config", - "aws-sdk-s3", + "bytes", "clap", "console-subscriber", "cxx", @@ -2235,14 +1492,18 @@ dependencies = [ "ed25519-dalek", "futures", "nix-compat", + "object_store", "pkg-config", "regex", "reqwest", "serde", "serde_json", "sha2", + "tempfile", "tokio", + "tokio-util", "tracing", + "ulid", "url", ] @@ -2265,21 +1526,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -2319,6 +1565,40 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.8.5", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -2369,23 +1649,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "outref" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" - -[[package]] -name = "p256" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" -dependencies = [ - "ecdsa", - "elliptic-curve", - "sha2", -] - [[package]] name = "parking_lot" version = "0.12.3" @@ -2447,24 +1710,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" -dependencies = [ - "der 0.6.1", - "spki 0.6.0", -] - [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.9", - "spki 0.7.3", + "der", + "spki", ] [[package]] @@ -2473,12 +1726,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2488,16 +1735,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -2548,6 +1785,70 @@ dependencies = [ "prost", ] +[[package]] +name = "quick-xml" +version = "0.37.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +dependencies = [ + "bytes", + "getrandom 0.3.2", + "rand 0.9.1", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.40" @@ -2570,8 +1871,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -2581,7 +1892,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -2593,6 +1914,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -2634,12 +1964,6 @@ dependencies = [ "regex-syntax 0.8.5", ] -[[package]] -name = "regex-lite" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" - [[package]] name = "regex-syntax" version = "0.6.29" @@ -2663,12 +1987,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.9", - "http 1.3.1", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.6.0", - "hyper-rustls 0.27.5", + "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "ipnet", @@ -2679,7 +2003,11 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.2.0", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2687,26 +2015,18 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-rustls", + "tokio-util", "tower 0.5.2", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "windows-registry", ] -[[package]] -name = "rfc6979" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" -dependencies = [ - "crypto-bigint 0.4.9", - "hmac", - "zeroize", -] - [[package]] name = "ring" version = "0.17.14" @@ -2729,9 +2049,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -2742,19 +2062,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.0.5" @@ -2764,48 +2071,24 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.9.4", + "linux-raw-sys", "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ - "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki", "subtle", "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -2818,15 +2101,6 @@ dependencies = [ "security-framework 3.2.0", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -2841,15 +2115,8 @@ name = "rustls-pki-types" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "web-time", ] [[package]] @@ -2858,7 +2125,6 @@ version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2876,6 +2142,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.27" @@ -2897,30 +2172,6 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "sec1" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" -dependencies = [ - "base16ct", - "der 0.6.1", - "generic-array", - "pkcs8 0.9.0", - "subtle", - "zeroize", -] - [[package]] name = "security-framework" version = "2.11.1" @@ -3007,17 +2258,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" version = "0.10.8" @@ -3053,23 +2293,13 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest", - "rand_core", -] - [[package]] name = "signature" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3097,16 +2327,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spki" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] - [[package]] name = "spki" version = "0.7.3" @@ -3114,7 +2334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.9", + "der", ] [[package]] @@ -3196,7 +2416,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix 1.0.5", + "rustix", "windows-sys 0.59.0", ] @@ -3239,36 +2459,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "time" -version = "0.3.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" - -[[package]] -name = "time-macros" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" -dependencies = [ - "num-conv", - "time-core", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -3279,6 +2469,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.44.2" @@ -3319,23 +2524,13 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.26", + "rustls", "tokio", ] @@ -3352,9 +2547,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -3391,11 +2586,11 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.9", - "http 1.3.1", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.6.0", + "hyper", "hyper-timeout", "hyper-util", "percent-encoding", @@ -3421,7 +2616,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util", @@ -3516,6 +2711,16 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand 0.9.1", + "web-time", +] + [[package]] name = "unicode-ident" version = "1.0.18" @@ -3546,12 +2751,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf16_iter" version = "1.0.5" @@ -3570,12 +2769,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" - [[package]] name = "valuable" version = "0.1.1" @@ -3595,10 +2788,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "vsimd" -version = "0.8.0" +name = "walkdir" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] [[package]] name = "want" @@ -3695,6 +2892,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.77" @@ -3706,15 +2916,13 @@ dependencies = [ ] [[package]] -name = "which" -version = "4.4.2" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", + "js-sys", + "wasm-bindgen", ] [[package]] @@ -3726,6 +2934,41 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings 0.4.0", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.1.1" @@ -3739,7 +2982,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", - "windows-strings", + "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -3761,6 +3004,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -3937,12 +3189,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" - [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index ee646cf..abaf3d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,6 @@ edition = "2024" [dependencies] anyhow = "1.0.97" async-compression = { version = "0.4.22", features = ["tokio", "zstd"] } -aws-config = { version = "1.6.1", features = ["behavior-version-latest"] } -aws-sdk-s3 = "1.82.0" clap = { version = "4.5.34", features = ["derive"] } ed25519-dalek = "2.1.1" futures = "0.3.31" @@ -22,6 +20,11 @@ tracing = "0.1.41" url = { version = "2.5.4", features = [ "serde" ]} cxx = "1.0" console-subscriber = "0.4.1" +tempfile = "3.19.1" +tokio-util = { version = "0.7.15", features = ["io"] } +bytes = "1.10.1" +object_store = { version = "0.12.0", features = ["aws"] } +ulid = "1.2.1" [build-dependencies] cxx-build = "1.0" diff --git a/flake.nix b/flake.nix index 46caa41..5796366 100644 --- a/flake.nix +++ b/flake.nix @@ -35,6 +35,7 @@ nix boost tokio-console + cargo-udeps ]; NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; RUST_LOG = "nixcp=debug"; diff --git a/src/main.rs b/src/main.rs index f4c0f4f..bca419b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ #![feature(let_chains)] #![feature(extend_one)] +#![feature(exit_status_error)] use std::path::PathBuf; @@ -11,6 +12,7 @@ use store::Store; mod bindings; mod cli; +mod make_nar; mod path_info; mod push; mod store; @@ -48,19 +50,15 @@ pub struct PushArgs { #[arg(long)] signing_key: String, - /// If unspecified, will get it form AWS_DEFAULT_REGION envar + /// If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 #[arg(long)] region: Option, - /// If unspecifed, will get it from AWS_ENDPOINT_URL envar + /// If unspecifed, will get it from AWS_ENDPOINT envar /// e.g. https://s3.example.com #[arg(long)] endpoint: Option, - /// AWS profile to use - #[arg(long)] - profile: Option, - #[arg(long)] skip_signature_check: bool, diff --git a/src/make_nar.rs b/src/make_nar.rs new file mode 100644 index 0000000..6e8d512 --- /dev/null +++ b/src/make_nar.rs @@ -0,0 +1,92 @@ +use anyhow::{Context, Result}; +use async_compression::{Level, tokio::bufread::ZstdEncoder}; +use nix_compat::{ + narinfo::{self, NarInfo}, + store_path::StorePath, +}; +use sha2::{Digest, Sha256}; +use std::mem::take; +use tempfile::NamedTempFile; +use tokio::{ + fs::File, + io::{AsyncRead, BufReader}, + process::Command, +}; +use tokio_util::io::InspectReader; + +use crate::path_info::PathInfo; + +pub struct MakeNar<'a> { + path_info: &'a PathInfo, + nar_file: NamedTempFile, + nar_hasher: Sha256, + /// hash of compressed nar file + file_hasher: Sha256, + nar_size: u64, + file_size: u64, +} + +impl<'a> MakeNar<'a> { + pub fn new(path_info: &'a PathInfo) -> Result { + Ok(Self { + path_info, + nar_file: NamedTempFile::new().context("crated tempfile for nar")?, + nar_hasher: Sha256::new(), + file_hasher: Sha256::new(), + nar_size: 0, + file_size: 0, + }) + } + + pub async fn make(&self) -> Result<()> { + Ok(Command::new("nix") + .arg("nar") + .arg("dump-path") + .arg(self.path_info.absolute_path()) + .kill_on_drop(true) + .stdout(self.nar_file.reopen()?) + .spawn()? + .wait() + .await? + .exit_ok()?) + } + + /// Returns a compressed nar reader which can be uploaded. File hash will be available when + /// everything is read + pub async fn compress_and_hash(&mut self) -> Result { + let nar_file = File::from_std(self.nar_file.reopen()?); + // reader that hashes as nar is read + let nar_reader = InspectReader::new(nar_file, |x| self.nar_hasher.update(x)); + + let encoder = ZstdEncoder::with_quality(BufReader::new(nar_reader), Level::Default); + // reader that updates file_hash as the compressed nar is read + Ok(InspectReader::new(encoder, |x| self.file_hasher.update(x))) + } + + /// Returns *unsigned* narinfo. `url` must be updated before uploading + pub fn get_narinfo(&mut self) -> Result { + let file_hash = take(&mut self.file_hasher).finalize().into(); + Ok(NarInfo { + flags: narinfo::Flags::empty(), + store_path: self.path_info.path.as_ref(), + nar_hash: take(&mut self.nar_hasher).finalize().into(), + nar_size: self.nar_size, + references: self + .path_info + .references + .iter() + .map(StorePath::as_ref) + .collect(), + signatures: Vec::new(), + ca: None, + system: None, + deriver: None, + compression: Some("zstd"), + file_hash: Some(file_hash), + file_size: Some(self.file_size), + url: "", + }) + // signature consists of: store_path, nar_hash, nar_size, and references + // nar_info.add_signature(self.signing_key); + } +} diff --git a/src/path_info.rs b/src/path_info.rs index e62b68a..f88e464 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,10 +1,10 @@ use std::collections::HashSet; use anyhow::{Context, Result}; -use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::nixbase32; use nix_compat::store_path::StorePath; +use object_store::{ObjectStore, aws::AmazonS3, path::Path as ObjectPath}; use regex::Regex; use std::path::Path; use tokio::process::Command; @@ -82,13 +82,13 @@ impl PathInfo { .filter_map(|signature| Some(signature.split_once(":")?.0)) .collect(); trace!("signers for {}: {:?}", self.path, signers); - return signers; + signers } pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { for upstream in upstreams { let upstream = upstream - .join(format!("{}.narinfo", self.digest()).as_str()) + .join(self.narinfo_path().as_ref()) .expect("adding .narinfo should make a valid url"); debug!("querying {}", upstream); let res_status = reqwest::Client::new() @@ -108,17 +108,12 @@ impl PathInfo { self.path.to_absolute_path() } - pub fn digest(&self) -> String { - nixbase32::encode(self.path.digest()) + pub fn narinfo_path(&self) -> ObjectPath { + ObjectPath::parse(format!("{}.narinfo", nixbase32::encode(self.path.digest()))) + .expect("must parse to a valid object_store path") } - pub async fn check_if_already_exists(&self, s3_client: &s3::Client, bucket: String) -> bool { - s3_client - .head_object() - .bucket(bucket) - .key(format!("{}.narinfo", self.digest())) - .send() - .await - .is_ok() + pub async fn check_if_already_exists(&self, s3: &AmazonS3) -> bool { + s3.head(&self.narinfo_path()).await.is_ok() } } diff --git a/src/push.rs b/src/push.rs index 8177864..c5a4229 100644 --- a/src/push.rs +++ b/src/push.rs @@ -9,23 +9,23 @@ use std::{ }; use anyhow::{Context, Result}; -use aws_config::Region; -use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::narinfo::{self, SigningKey}; -use tokio::sync::{RwLock, mpsc}; +use object_store::aws::{AmazonS3, AmazonS3Builder}; +use tokio::sync::{RwLock, Semaphore, mpsc}; use tracing::debug; use url::Url; use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; +const UPLOAD_CONCURRENCY: usize = 32; + pub struct Push { upstream_caches: Vec, store_paths: Arc>>, - s3_client: s3::Client, signing_key: SigningKey, - bucket: String, store: Arc, + s3: Arc, // paths that we skipped cause of a signature match signature_hit_count: AtomicUsize, // paths that we skipped cause we found it on an upstream @@ -51,25 +51,21 @@ impl Push { let key = fs::read_to_string(&cli.signing_key)?; let signing_key = narinfo::parse_keypair(key.as_str())?.0; - let mut s3_config = aws_config::from_env(); + let mut s3_builder = AmazonS3Builder::from_env().with_bucket_name(&cli.bucket); + if let Some(region) = &cli.region { - s3_config = s3_config.region(Region::new(region.clone())); + s3_builder = s3_builder.with_region(region); } if let Some(endpoint) = &cli.endpoint { - s3_config = s3_config.endpoint_url(endpoint); - } - if let Some(profile) = &cli.profile { - s3_config = s3_config.profile_name(profile); + s3_builder = s3_builder.with_endpoint(endpoint); } - let s3_client = s3::Client::new(&s3_config.load().await); Ok(Self { upstream_caches: upstreams, store_paths: Arc::new(RwLock::new(Vec::new())), - s3_client, signing_key, - bucket: cli.bucket.clone(), store: Arc::new(store), + s3: Arc::new(s3_builder.build()?), signature_hit_count: AtomicUsize::new(0), upstream_hit_count: AtomicUsize::new(0), already_exists_count: AtomicUsize::new(0), @@ -109,9 +105,10 @@ impl Push { } pub async fn run(&'static self) -> Result<()> { - let (tx, rx) = mpsc::channel(10); + let (tx, rx) = mpsc::channel(UPLOAD_CONCURRENCY); let filter = tokio::spawn(self.filter_from_upstream(tx)); let upload = tokio::spawn(self.upload(rx)); + filter.await?; upload.await??; Ok(()) @@ -135,10 +132,7 @@ impl Push { .check_upstream_hit(self.upstream_caches.as_slice()) .await { - if path - .check_if_already_exists(&self.s3_client, self.bucket.clone()) - .await - { + if path.check_if_already_exists(&self.s3).await { debug!("skip {} (already exists)", path.absolute_path()); self.already_exists_count.fetch_add(1, Ordering::Relaxed); } else { @@ -160,22 +154,23 @@ impl Push { } async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { - let mut uploads = Vec::with_capacity(10); + let mut uploads = Vec::new(); + let permits = Arc::new(Semaphore::new(UPLOAD_CONCURRENCY)); loop { + let permits = permits.clone(); + let permit = permits.acquire_owned().await.unwrap(); + if let Some(path_to_upload) = rx.recv().await { println!("uploading: {}", path_to_upload.absolute_path()); - let uploader = Uploader::new( - &self.signing_key, - path_to_upload, - &self.s3_client, - self.bucket.clone(), - )?; + let uploader = Uploader::new(&self.signing_key, path_to_upload)?; uploads.push(tokio::spawn({ + let s3 = self.s3.clone(); async move { - let res = uploader.upload().await; + let res = uploader.upload(s3).await; + drop(permit); self.upload_count.fetch_add(1, Ordering::Relaxed); res } diff --git a/src/store.rs b/src/store.rs index 4499243..763cb3d 100644 --- a/src/store.rs +++ b/src/store.rs @@ -28,13 +28,13 @@ impl Store { inner .store() .compute_fs_closure(path.to_string().as_bytes(), false, true, true)?; - Ok(cxx_vector + cxx_vector .iter() .map(|x| { StorePath::from_bytes(x.as_bytes()) .context("make StorePath from vector returned by compute_fs_closure") }) - .collect::>()?) + .collect::>() }) .await .unwrap() diff --git a/src/uploader.rs b/src/uploader.rs index eb955a2..c9c49f0 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -1,190 +1,75 @@ use anyhow::Result; -use async_compression::{Level, tokio::bufread::ZstdEncoder}; -use aws_sdk_s3::{ - self as s3, - types::{CompletedMultipartUpload, CompletedPart}, -}; -use futures::future::join_all; -use nix_compat::{ - narinfo::{self, NarInfo, SigningKey}, - nixbase32, - store_path::StorePath, -}; -use sha2::{Digest, Sha256}; -use tokio::{io::AsyncReadExt, process::Command}; -use tracing::debug; +use bytes::BytesMut; +use nix_compat::{narinfo::SigningKey, nixbase32}; +use object_store::{ObjectStore, aws::AmazonS3, buffered::BufWriter, path::Path}; +use std::sync::Arc; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tracing::{debug, trace}; +use ulid::Ulid; -use crate::path_info::PathInfo; +use crate::{make_nar::MakeNar, path_info::PathInfo}; -const MULTIPART_CUTOFF: usize = 1024 * 1024 * 5; +const CHUNK_SIZE: usize = 1024 * 1024 * 5; pub struct Uploader<'a> { signing_key: &'a SigningKey, path: PathInfo, - s3_client: &'a s3::Client, - bucket: String, - hash: Sha256, } impl<'a> Uploader<'a> { pub fn new( signing_key: &'a SigningKey, path: PathInfo, - s3_client: &'a s3::Client, - bucket: String, ) -> Result { - Ok(Self { - signing_key, - path, - s3_client, - bucket, - hash: Sha256::new(), - }) + Ok(Self { signing_key, path }) } - pub async fn upload(&self) -> Result<()> { - let nar = self.make_nar().await?; - let mut nar_info = self.narinfo_from_nar(&nar)?; - let nar = self.compress_nar(&nar).await; + pub async fn upload(&self, s3: Arc) -> Result<()> { + let mut nar = MakeNar::new(&self.path)?; + nar.make().await?; - // update fields that we know after compression - let mut hasher = Sha256::new(); - hasher.update(&nar); - let hash: [u8; 32] = hasher.finalize().into(); - let nar_url = self.nar_url(&hash); - nar_info.file_hash = Some(hash); - nar_info.file_size = Some(nar.len() as u64); - nar_info.url = nar_url.as_str(); - debug!("uploading nar with key: {nar_url}"); + // we don't know what the hash of the compressed file will be so upload to a + // temp location for now + let temp_path = Path::parse(Ulid::new().to_string())?; + let mut s3_writer = BufWriter::new(s3.clone(), temp_path.clone()); - if nar.len() < MULTIPART_CUTOFF { - let put_object = self - .s3_client - .put_object() - .bucket(&self.bucket) - .key(&nar_url) - .body(nar.into()) - .send() - .await?; - debug!("put object: {:#?}", put_object); - } else { - let multipart = self - .s3_client - .create_multipart_upload() - .bucket(&self.bucket) - .key(&nar_url) - .send() - .await?; - let upload_id = multipart.upload_id().unwrap(); - - let mut parts = Vec::with_capacity(nar.len() / MULTIPART_CUTOFF); - let chunks = nar.chunks(MULTIPART_CUTOFF); - for (i, chunk) in chunks.enumerate() { - parts.push(tokio::task::spawn( - self.s3_client - .upload_part() - .bucket(&self.bucket) - .key(&nar_url) - .upload_id(upload_id) - .part_number(i as i32 + 1) - .body(chunk.to_vec().into()) - .send(), - )); - } - - let completed_parts = join_all(parts) - .await - .into_iter() - .flatten() - .collect::, _>>()? - .into_iter() - .enumerate() - .map(|(i, part)| { - CompletedPart::builder() - .set_e_tag(part.e_tag().map(ToString::to_string)) - .set_part_number(Some(i as i32 + 1)) - .set_checksum_sha256(part.checksum_sha256().map(ToString::to_string)) - .build() - }) - .collect::>(); - - let completed_mp_upload = CompletedMultipartUpload::builder() - .set_parts(Some(completed_parts)) - .build(); - - let complete_mp_upload = self - .s3_client - .complete_multipart_upload() - .bucket(&self.bucket) - .key(&nar_url) - .upload_id(upload_id) - .multipart_upload(completed_mp_upload) - .send() - .await?; - - debug!("complete multipart upload: {:#?}", complete_mp_upload); + // compress and upload nar + let mut file_reader = nar.compress_and_hash().await?; + let mut buf = BytesMut::with_capacity(CHUNK_SIZE); + debug!("uploading to temp path: {}", temp_path); + while let n = file_reader.read_buf(&mut buf).await? + && n != 0 + { + s3_writer.write_all_buf(&mut buf).await?; } + drop(file_reader); - let narinfo_url = format!("{}.narinfo", self.path.digest()); - debug!("uploading narinfo with key {narinfo_url}"); - self.s3_client - .put_object() - .bucket(&self.bucket) - .key(narinfo_url) - .body(nar_info.to_string().as_bytes().to_vec().into()) - .send() - .await?; - debug!("done uploading narinfo"); + let mut nar_info = nar.get_narinfo()?; + nar_info.add_signature(self.signing_key); + trace!("narinfo: {:#}", nar_info); + + // now that we can calculate the file_hash move the nar to where it should be + let real_path = nar_url( + &nar_info + .file_hash + .expect("file hash must be known at this point"), + ); + debug!("moving {} to {}", temp_path, real_path); + // this is implemented as a copy-and-delete + s3.rename(&temp_path, &real_path).await?; + + // upload narinfo + let narinfo_path = self.path.narinfo_path(); + debug!("uploading narinfo: {}", narinfo_path); + s3.put(&narinfo_path, nar_info.to_string().into()).await?; Ok(()) } - - async fn make_nar(&self) -> Result> { - Ok(Command::new("nix") - .arg("nar") - .arg("dump-path") - .arg(self.path.absolute_path()) - .output() - .await? - .stdout) - } - - fn narinfo_from_nar(&self, nar: &[u8]) -> Result { - let mut hasher = Sha256::new(); - hasher.update(nar); - let nar_hash: [u8; 32] = hasher.finalize().into(); - let mut nar_info = NarInfo { - flags: narinfo::Flags::empty(), - store_path: self.path.path.as_ref(), - nar_hash, - nar_size: nar.len() as u64, - references: self.path.references.iter().map(StorePath::as_ref).collect(), - signatures: Vec::new(), - ca: None, - system: None, - deriver: None, - compression: Some("zstd"), - file_hash: None, - file_size: None, - url: "", - }; - // signature consists of: store_path, nar_hash, nar_size, and references - nar_info.add_signature(self.signing_key); - Ok(nar_info) - } - - fn nar_url(&self, compressed_nar_hash: &[u8]) -> String { - let compressed_nar_hash = nixbase32::encode(compressed_nar_hash); - format!("nar/{compressed_nar_hash}.nar.zst") - } - - async fn compress_nar(&self, nar: &[u8]) -> Vec { - let mut encoder = ZstdEncoder::with_quality(nar, Level::Default); - let mut compressed = Vec::with_capacity(nar.len()); - encoder - .read_to_end(&mut compressed) - .await - .expect("should compress just fine"); - compressed - } +} + +/// calculate url where the compressed nar should be uploaded +fn nar_url(file_hash: &[u8]) -> Path { + let compressed_nar_hash = nixbase32::encode(file_hash); + Path::parse(format!("nar/{compressed_nar_hash}.nar.zst")) + .expect("should parse to a valid object_store::path::Path") } From 846c465ea0988f2b5abef44d7443d536102acee1 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 13:11:54 -0400 Subject: [PATCH 05/44] fix flake --- flake.nix | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/flake.nix b/flake.nix index 5796366..1ee2f5d 100644 --- a/flake.nix +++ b/flake.nix @@ -23,34 +23,40 @@ toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); lib = pkgs.lib; + nativeBuildInputs = with pkgs; [ + pkg-config + ]; + buildInputs = with pkgs; [ + toolchain + openssl + nix + boost + ]; + env = { + # for cpp bindings to work + NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; + }; in { devShells.default = pkgs.mkShell { - nativeBuildInputs = with pkgs; [ - pkg-config - ]; - buildInputs = with pkgs; [ - toolchain - openssl - nix - boost + inherit buildInputs; + inherit nativeBuildInputs; + packages = with pkgs; [ tokio-console cargo-udeps ]; - NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; - RUST_LOG = "nixcp=debug"; - RUST_BACKGRACE = 1; + env = env // { + RUST_LOG = "nixcp=debug"; + RUST_BACKGRACE = 1; + }; }; packages.default = craneLib.buildPackage { + inherit nativeBuildInputs; + inherit buildInputs; + inherit env; src = craneLib.cleanCargoSource ./.; strictDeps = true; - nativeBuildInputs = with pkgs; [ - pkg-config - ]; - buildInputs = with pkgs; [ - openssl - ]; }; } ); From b8877f33a39ab2aaa6b5586b4b8a1500b1555c52 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 14:10:20 -0400 Subject: [PATCH 06/44] enable lto --- Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index abaf3d5..f548ea8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,10 @@ name = "nixcp" version = "0.1.0" edition = "2024" +[profile.release] +lto = true +codegen-units = 1 + [dependencies] anyhow = "1.0.97" async-compression = { version = "0.4.22", features = ["tokio", "zstd"] } From fc304df35ec2b3e0cb9dcede9d1d6bd9a5e3896c Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 14:21:18 -0400 Subject: [PATCH 07/44] refactor flake --- flake.nix | 52 ++++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/flake.nix b/flake.nix index 1ee2f5d..e43527f 100644 --- a/flake.nix +++ b/flake.nix @@ -23,41 +23,45 @@ toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); lib = pkgs.lib; - nativeBuildInputs = with pkgs; [ - pkg-config - ]; - buildInputs = with pkgs; [ - toolchain - openssl - nix - boost - ]; - env = { + src = craneLib.cleanCargoSource ./.; + + commonArgs = { + inherit src; + strictDeps = true; + nativeBuildInputs = with pkgs; [ + pkg-config + ]; + buildInputs = with pkgs; [ + toolchain + openssl + nix + boost + ]; # for cpp bindings to work NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; }; + + cargoArtifacts = craneLib.buildDepsOnly commonArgs; + nixcp = craneLib.buildPackage (commonArgs // { + inherit cargoArtifacts; + }); in { - devShells.default = pkgs.mkShell { - inherit buildInputs; - inherit nativeBuildInputs; + devShells.default = craneLib.devShell { + inputsFrom = [ nixcp ]; + + RUST_LOG = "nixcp=debug"; + RUST_BACKGRACE = 1; + # for cpp bindings to work + NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; + packages = with pkgs; [ tokio-console cargo-udeps ]; - env = env // { - RUST_LOG = "nixcp=debug"; - RUST_BACKGRACE = 1; - }; }; - packages.default = craneLib.buildPackage { - inherit nativeBuildInputs; - inherit buildInputs; - inherit env; - src = craneLib.cleanCargoSource ./.; - strictDeps = true; - }; + packages.default = nixcp; } ); } From 5a3e6089b4e373e74b4050778ca5b09ef42acc54 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 14:40:38 -0400 Subject: [PATCH 08/44] don't clean cpp files --- flake.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e43527f..01d135a 100644 --- a/flake.nix +++ b/flake.nix @@ -23,7 +23,16 @@ toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); lib = pkgs.lib; - src = craneLib.cleanCargoSource ./.; + + # don't clean cpp files + cppFilter = path: _type: builtins.match ".*(cpp|hpp)$" path != null; + cppOrCargo = path: type: + (cppFilter path type) || (craneLib.filterCargoSources path type); + src = lib.cleanSourceWith { + src = ./.; + filter = cppOrCargo; + name = "source"; + }; commonArgs = { inherit src; From d524222a862018da5b508a06985935db50d737b0 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 15:37:54 -0400 Subject: [PATCH 09/44] make tokio console optional and make it actually work --- Cargo.lock | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 4 ++++ src/main.rs | 29 ++++++++++++++++++++++++++--- 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbd7534..a3e049c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1503,6 +1503,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "tracing-subscriber", "ulid", "url", ] @@ -1526,6 +1527,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1649,6 +1660,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.3" @@ -2684,6 +2701,17 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.19" @@ -2691,12 +2719,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", + "nu-ansi-term", "once_cell", "regex", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", + "tracing-log", ] [[package]] @@ -2925,6 +2956,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.9" @@ -2934,6 +2981,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.61.0" diff --git a/Cargo.toml b/Cargo.toml index f548ea8..8dd350d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,9 @@ name = "nixcp" version = "0.1.0" edition = "2024" +[build] +rustflags = ["--cfg", "tokio_unstable"] + [profile.release] lto = true codegen-units = 1 @@ -29,6 +32,7 @@ tokio-util = { version = "0.7.15", features = ["io"] } bytes = "1.10.1" object_store = { version = "0.12.0", features = ["aws"] } ulid = "1.2.1" +tracing-subscriber = "0.3.19" [build-dependencies] cxx-build = "1.0" diff --git a/src/main.rs b/src/main.rs index bca419b..0fefdf5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,11 @@ #![feature(let_chains)] -#![feature(extend_one)] #![feature(exit_status_error)] use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Args, Parser, Subcommand}; +use tracing_subscriber::{EnvFilter, prelude::*}; use push::Push; use store::Store; @@ -26,6 +26,10 @@ mod uploader; struct Cli { #[command(subcommand)] command: Commands, + + /// Whether to enable tokio-console + #[arg(long)] + tokio_console: bool, } #[derive(Debug, Subcommand)] @@ -70,9 +74,8 @@ pub struct PushArgs { #[tokio::main] async fn main() -> Result<()> { - console_subscriber::init(); - let cli = Cli::parse(); + init_logging(cli.tokio_console); match &cli.command { Commands::Push(cli) => { @@ -87,3 +90,23 @@ async fn main() -> Result<()> { Ok(()) } + +fn init_logging(tokio_console: bool) { + let env_filter = EnvFilter::from_default_env(); + let fmt_layer = tracing_subscriber::fmt::layer().with_filter(env_filter); + + let console_layer = if tokio_console { + Some(console_subscriber::spawn()) + } else { + None + }; + + tracing_subscriber::registry() + .with(fmt_layer) + .with(console_layer) + .init(); + + if tokio_console { + println!("tokio-console is enabled"); + } +} From 7dec14fc1a80f6f7c0a6da83f581d5322c728228 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 15:46:23 -0400 Subject: [PATCH 10/44] set path to nar in narinfo --- src/make_nar.rs | 2 -- src/uploader.rs | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/make_nar.rs b/src/make_nar.rs index 6e8d512..e1ca57c 100644 --- a/src/make_nar.rs +++ b/src/make_nar.rs @@ -86,7 +86,5 @@ impl<'a> MakeNar<'a> { file_size: Some(self.file_size), url: "", }) - // signature consists of: store_path, nar_hash, nar_size, and references - // nar_info.add_signature(self.signing_key); } } diff --git a/src/uploader.rs b/src/uploader.rs index c9c49f0..0832a58 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -57,6 +57,8 @@ impl<'a> Uploader<'a> { debug!("moving {} to {}", temp_path, real_path); // this is implemented as a copy-and-delete s3.rename(&temp_path, &real_path).await?; + // set nar url in narinfo + nar_info.url = real_path.as_ref(); // upload narinfo let narinfo_path = self.path.narinfo_path(); From 0fae7ac3b08c73a73bf9a8cda9e29aacdf22b4c2 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 18:21:27 -0400 Subject: [PATCH 11/44] fix build with tokio unstable --- .cargo/config.toml | 2 ++ Cargo.toml | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..bff29e6 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] diff --git a/Cargo.toml b/Cargo.toml index 8dd350d..6ca052f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,9 +3,6 @@ name = "nixcp" version = "0.1.0" edition = "2024" -[build] -rustflags = ["--cfg", "tokio_unstable"] - [profile.release] lto = true codegen-units = 1 From 0fedae933465d2b1935af2ff0498118393203ed1 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 18:21:49 -0400 Subject: [PATCH 12/44] try to fix fd issues --- src/make_nar.rs | 36 +++++++++++++++++++++++++++--------- src/push.rs | 3 ++- src/uploader.rs | 18 +++++++++++------- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/src/make_nar.rs b/src/make_nar.rs index e1ca57c..19a112b 100644 --- a/src/make_nar.rs +++ b/src/make_nar.rs @@ -6,19 +6,20 @@ use nix_compat::{ }; use sha2::{Digest, Sha256}; use std::mem::take; -use tempfile::NamedTempFile; +use tempfile::{NamedTempFile, TempPath}; use tokio::{ fs::File, io::{AsyncRead, BufReader}, process::Command, }; use tokio_util::io::InspectReader; +use tracing::debug; use crate::path_info::PathInfo; pub struct MakeNar<'a> { path_info: &'a PathInfo, - nar_file: NamedTempFile, + nar_file: TempPath, nar_hasher: Sha256, /// hash of compressed nar file file_hasher: Sha256, @@ -30,7 +31,9 @@ impl<'a> MakeNar<'a> { pub fn new(path_info: &'a PathInfo) -> Result { Ok(Self { path_info, - nar_file: NamedTempFile::new().context("crated tempfile for nar")?, + nar_file: NamedTempFile::new() + .context("create tempfile for nar")? + .into_temp_path(), nar_hasher: Sha256::new(), file_hasher: Sha256::new(), nar_size: 0, @@ -39,12 +42,19 @@ impl<'a> MakeNar<'a> { } pub async fn make(&self) -> Result<()> { + let path = self.path_info.absolute_path(); + let out = std::fs::File::options() + .write(true) + .truncate(true) + .open(&self.nar_file)?; + + debug!("dumping nar for: {}", path); Ok(Command::new("nix") .arg("nar") - .arg("dump-path") + .arg("pack") .arg(self.path_info.absolute_path()) .kill_on_drop(true) - .stdout(self.nar_file.reopen()?) + .stdout(out) .spawn()? .wait() .await? @@ -54,22 +64,30 @@ impl<'a> MakeNar<'a> { /// Returns a compressed nar reader which can be uploaded. File hash will be available when /// everything is read pub async fn compress_and_hash(&mut self) -> Result { - let nar_file = File::from_std(self.nar_file.reopen()?); + let nar_file = File::open(&self.nar_file).await?; // reader that hashes as nar is read - let nar_reader = InspectReader::new(nar_file, |x| self.nar_hasher.update(x)); + let nar_reader = InspectReader::new(nar_file, |x| { + self.nar_size += x.len() as u64; + self.nar_hasher.update(x); + }); let encoder = ZstdEncoder::with_quality(BufReader::new(nar_reader), Level::Default); // reader that updates file_hash as the compressed nar is read - Ok(InspectReader::new(encoder, |x| self.file_hasher.update(x))) + Ok(InspectReader::new(encoder, |x| { + self.file_size += x.len() as u64; + self.file_hasher.update(x); + })) } /// Returns *unsigned* narinfo. `url` must be updated before uploading pub fn get_narinfo(&mut self) -> Result { let file_hash = take(&mut self.file_hasher).finalize().into(); + let nar_hash = take(&mut self.nar_hasher).finalize().into(); + Ok(NarInfo { flags: narinfo::Flags::empty(), store_path: self.path_info.path.as_ref(), - nar_hash: take(&mut self.nar_hasher).finalize().into(), + nar_hash, nar_size: self.nar_size, references: self .path_info diff --git a/src/push.rs b/src/push.rs index c5a4229..93a28da 100644 --- a/src/push.rs +++ b/src/push.rs @@ -18,7 +18,7 @@ use url::Url; use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; -const UPLOAD_CONCURRENCY: usize = 32; +const UPLOAD_CONCURRENCY: usize = 5; pub struct Push { upstream_caches: Vec, @@ -159,6 +159,7 @@ impl Push { loop { let permits = permits.clone(); + debug!("upload permits available: {}", permits.available_permits()); let permit = permits.acquire_owned().await.unwrap(); if let Some(path_to_upload) = rx.recv().await { diff --git a/src/uploader.rs b/src/uploader.rs index 0832a58..a2c2766 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -32,21 +32,22 @@ impl<'a> Uploader<'a> { // temp location for now let temp_path = Path::parse(Ulid::new().to_string())?; let mut s3_writer = BufWriter::new(s3.clone(), temp_path.clone()); + debug!("uploading to temp path: {}", temp_path); // compress and upload nar let mut file_reader = nar.compress_and_hash().await?; - let mut buf = BytesMut::with_capacity(CHUNK_SIZE); - debug!("uploading to temp path: {}", temp_path); - while let n = file_reader.read_buf(&mut buf).await? - && n != 0 - { - s3_writer.write_all_buf(&mut buf).await?; + loop { + let mut buf = BytesMut::with_capacity(CHUNK_SIZE); + let n = file_reader.read_buf(&mut buf).await?; + s3_writer.put(buf.freeze()).await?; + if n == 0 { + break; + } } drop(file_reader); let mut nar_info = nar.get_narinfo()?; nar_info.add_signature(self.signing_key); - trace!("narinfo: {:#}", nar_info); // now that we can calculate the file_hash move the nar to where it should be let real_path = nar_url( @@ -55,6 +56,8 @@ impl<'a> Uploader<'a> { .expect("file hash must be known at this point"), ); debug!("moving {} to {}", temp_path, real_path); + // the temp object must be done uploading + s3_writer.shutdown().await?; // this is implemented as a copy-and-delete s3.rename(&temp_path, &real_path).await?; // set nar url in narinfo @@ -63,6 +66,7 @@ impl<'a> Uploader<'a> { // upload narinfo let narinfo_path = self.path.narinfo_path(); debug!("uploading narinfo: {}", narinfo_path); + trace!("narinfo: {:#}", nar_info); s3.put(&narinfo_path, nar_info.to_string().into()).await?; Ok(()) From 85fefe9e7778423820860d1a267869f3b7295294 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 21:08:01 -0400 Subject: [PATCH 13/44] use cpp bindings to make nar --- src/bindings/mod.rs | 21 +++++++++++++++----- src/bindings/nix.cpp | 11 +++++++++++ src/bindings/nix.hpp | 1 + src/make_nar.rs | 47 ++++++++++---------------------------------- src/push.rs | 3 ++- src/store.rs | 24 ++++++++++++++++++++-- src/uploader.rs | 9 ++++----- 7 files changed, 66 insertions(+), 50 deletions(-) diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 61a32af..8084dff 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -23,6 +23,7 @@ use std::pin::Pin; use std::task::{Context, Poll}; use anyhow::Result; +use bytes::Bytes; use futures::stream::{Stream, StreamExt}; use tokio::io::{AsyncWrite, AsyncWriteExt}; @@ -125,7 +126,7 @@ impl AsyncWriteAdapter { writer.write_all(&v).await?; } Err(e) => { - return Err(e); + return Err(e.into()); } } } @@ -139,7 +140,7 @@ impl AsyncWriteAdapter { } impl Stream for AsyncWriteAdapter { - type Item = Result>; + type Item = std::io::Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.receiver.poll_recv(cx) { @@ -147,9 +148,12 @@ impl Stream for AsyncWriteAdapter { Poll::Ready(Some(message)) => { use AsyncWriteMessage::*; match message { - Data(v) => Poll::Ready(Some(Ok(v))), + Data(v) => Poll::Ready(Some(Ok(v.into()))), Error(exception) => { - let error = anyhow::Error::msg(format!("cxx error: {exception}")); + let error = std::io::Error::new( + io::ErrorKind::Other, + format!("cxx error: {exception}"), + ); Poll::Ready(Some(Err(error))) } Eof => { @@ -160,7 +164,7 @@ impl Stream for AsyncWriteAdapter { } Poll::Ready(None) => { if !self.eof { - Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe).into()))) + Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe)))) } else { Poll::Ready(None) } @@ -208,6 +212,13 @@ mod ffi { include_derivers: bool, ) -> Result>>; + /// Creates a NAR dump from a path. + fn nar_from_path( + self: Pin<&mut CNixStore>, + base_name: Vec, + sender: Box, + ) -> Result<()>; + /// Obtains a handle to the Nix store. fn open_nix_store() -> Result>; diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp index 3914de1..326e878 100644 --- a/src/bindings/nix.cpp +++ b/src/bindings/nix.cpp @@ -108,6 +108,17 @@ std::unique_ptr> CNixStore::compute_fs_closure(RBasePat return std::make_unique>(result); } +void CNixStore::nar_from_path(RVec base_name, RBox sender) { + RustSink sink(std::move(sender)); + + std::string_view sv((const char *)base_name.data(), base_name.size()); + nix::StorePath store_path(sv); + + // exceptions will be thrown into Rust + this->store->narFromPath(store_path, sink); + sink.eof(); +} + std::unique_ptr open_nix_store() { return std::make_unique(); } diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp index 9f4964a..5c79a33 100644 --- a/src/bindings/nix.hpp +++ b/src/bindings/nix.hpp @@ -79,6 +79,7 @@ public: bool flip_direction, bool include_outputs, bool include_derivers); + void nar_from_path(RVec base_name, RBox sender); }; std::unique_ptr open_nix_store(); diff --git a/src/make_nar.rs b/src/make_nar.rs index 19a112b..b5c1ab2 100644 --- a/src/make_nar.rs +++ b/src/make_nar.rs @@ -1,25 +1,20 @@ -use anyhow::{Context, Result}; +use anyhow::Result; use async_compression::{Level, tokio::bufread::ZstdEncoder}; use nix_compat::{ narinfo::{self, NarInfo}, store_path::StorePath, }; use sha2::{Digest, Sha256}; -use std::mem::take; -use tempfile::{NamedTempFile, TempPath}; -use tokio::{ - fs::File, - io::{AsyncRead, BufReader}, - process::Command, -}; +use std::{mem::take, sync::Arc}; +use tokio::io::{AsyncRead, BufReader}; use tokio_util::io::InspectReader; -use tracing::debug; use crate::path_info::PathInfo; +use crate::store::Store; pub struct MakeNar<'a> { path_info: &'a PathInfo, - nar_file: TempPath, + store: Arc, nar_hasher: Sha256, /// hash of compressed nar file file_hasher: Sha256, @@ -28,12 +23,10 @@ pub struct MakeNar<'a> { } impl<'a> MakeNar<'a> { - pub fn new(path_info: &'a PathInfo) -> Result { + pub fn new(path_info: &'a PathInfo, store: Arc) -> Result { Ok(Self { path_info, - nar_file: NamedTempFile::new() - .context("create tempfile for nar")? - .into_temp_path(), + store, nar_hasher: Sha256::new(), file_hasher: Sha256::new(), nar_size: 0, @@ -41,32 +34,12 @@ impl<'a> MakeNar<'a> { }) } - pub async fn make(&self) -> Result<()> { - let path = self.path_info.absolute_path(); - let out = std::fs::File::options() - .write(true) - .truncate(true) - .open(&self.nar_file)?; - - debug!("dumping nar for: {}", path); - Ok(Command::new("nix") - .arg("nar") - .arg("pack") - .arg(self.path_info.absolute_path()) - .kill_on_drop(true) - .stdout(out) - .spawn()? - .wait() - .await? - .exit_ok()?) - } - /// Returns a compressed nar reader which can be uploaded. File hash will be available when /// everything is read - pub async fn compress_and_hash(&mut self) -> Result { - let nar_file = File::open(&self.nar_file).await?; + pub fn compress_and_hash(&mut self) -> Result { + let nar_reader = self.store.nar_from_path(self.path_info.path.clone()); // reader that hashes as nar is read - let nar_reader = InspectReader::new(nar_file, |x| { + let nar_reader = InspectReader::new(nar_reader, |x| { self.nar_size += x.len() as u64; self.nar_hasher.update(x); }); diff --git a/src/push.rs b/src/push.rs index 93a28da..57a43d2 100644 --- a/src/push.rs +++ b/src/push.rs @@ -169,8 +169,9 @@ impl Push { uploads.push(tokio::spawn({ let s3 = self.s3.clone(); + let store = self.store.clone(); async move { - let res = uploader.upload(s3).await; + let res = uploader.upload(s3, store).await; drop(permit); self.upload_count.fetch_add(1, Ordering::Relaxed); res diff --git a/src/store.rs b/src/store.rs index 763cb3d..7589e94 100644 --- a/src/store.rs +++ b/src/store.rs @@ -2,9 +2,13 @@ use std::{ffi::OsStr, os::unix::ffi::OsStrExt, sync::Arc}; use anyhow::{Context, Result}; use nix_compat::store_path::StorePath; -use tokio::task; +use tokio::{io::AsyncRead, task}; +use tokio_util::io::StreamReader; -use crate::{bindings, path_info::PathInfo}; +use crate::{ + bindings::{self, AsyncWriteAdapter}, + path_info::PathInfo, +}; pub struct Store { inner: Arc, @@ -75,4 +79,20 @@ impl Store { .await .unwrap() } + + pub fn nar_from_path(&self, store_path: StorePath) -> impl AsyncRead { + let inner = self.inner.clone(); + let (adapter, mut sender) = AsyncWriteAdapter::new(); + let base_name = store_path.to_string().as_bytes().to_vec(); + + tokio::task::spawn_blocking(move || { + // Send all exceptions through the channel, and ignore errors + // during sending (the channel may have been closed). + if let Err(e) = inner.store().nar_from_path(base_name, sender.clone()) { + let _ = sender.rust_error(e); + } + }); + + StreamReader::new(adapter) + } } diff --git a/src/uploader.rs b/src/uploader.rs index a2c2766..c829a79 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -7,7 +7,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tracing::{debug, trace}; use ulid::Ulid; -use crate::{make_nar::MakeNar, path_info::PathInfo}; +use crate::{make_nar::MakeNar, path_info::PathInfo, store::Store}; const CHUNK_SIZE: usize = 1024 * 1024 * 5; @@ -24,9 +24,8 @@ impl<'a> Uploader<'a> { Ok(Self { signing_key, path }) } - pub async fn upload(&self, s3: Arc) -> Result<()> { - let mut nar = MakeNar::new(&self.path)?; - nar.make().await?; + pub async fn upload(&self, s3: Arc, store: Arc) -> Result<()> { + let mut nar = MakeNar::new(&self.path, store)?; // we don't know what the hash of the compressed file will be so upload to a // temp location for now @@ -35,7 +34,7 @@ impl<'a> Uploader<'a> { debug!("uploading to temp path: {}", temp_path); // compress and upload nar - let mut file_reader = nar.compress_and_hash().await?; + let mut file_reader = nar.compress_and_hash()?; loop { let mut buf = BytesMut::with_capacity(CHUNK_SIZE); let n = file_reader.read_buf(&mut buf).await?; From ca97aebd7ac84b2fbeb88a545c6968795cc186ce Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 23:07:56 -0400 Subject: [PATCH 14/44] limit directories even more --- src/push.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/push.rs b/src/push.rs index 57a43d2..9a9e156 100644 --- a/src/push.rs +++ b/src/push.rs @@ -18,8 +18,6 @@ use url::Url; use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; -const UPLOAD_CONCURRENCY: usize = 5; - pub struct Push { upstream_caches: Vec, store_paths: Arc>>, @@ -105,7 +103,7 @@ impl Push { } pub async fn run(&'static self) -> Result<()> { - let (tx, rx) = mpsc::channel(UPLOAD_CONCURRENCY); + let (tx, rx) = mpsc::channel(1); let filter = tokio::spawn(self.filter_from_upstream(tx)); let upload = tokio::spawn(self.upload(rx)); @@ -155,19 +153,25 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(UPLOAD_CONCURRENCY)); + let permits = Arc::new(Semaphore::new(10)); + let big_permits = Arc::new(Semaphore::new(2)); loop { let permits = permits.clone(); - debug!("upload permits available: {}", permits.available_permits()); - let permit = permits.acquire_owned().await.unwrap(); + let big_permits = big_permits.clone(); if let Some(path_to_upload) = rx.recv().await { - println!("uploading: {}", path_to_upload.absolute_path()); - - let uploader = Uploader::new(&self.signing_key, path_to_upload)?; + let mut permit = permits.acquire_owned().await.unwrap(); uploads.push(tokio::spawn({ + // directory may have many files and end up causing "too many open files" + if PathBuf::from(path_to_upload.absolute_path()).is_dir() { + // drop regular permit and take the big one + permit = big_permits.acquire_owned().await.unwrap(); + } + + println!("uploading: {}", path_to_upload.absolute_path()); + let uploader = Uploader::new(&self.signing_key, path_to_upload)?; let s3 = self.s3.clone(); let store = self.store.clone(); async move { From e5336d304d41d80cb9b07a4d9ffe3d9d480e6cf4 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 27 Apr 2025 01:23:45 -0400 Subject: [PATCH 15/44] improve concurrency control; use nar_size from cpathinfo --- src/bindings/mod.rs | 2 ++ src/bindings/nix.cpp | 4 ++++ src/bindings/nix.hpp | 1 + src/path_info.rs | 3 ++- src/push.rs | 29 ++++++++++++++++++++++------- src/store.rs | 2 ++ 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 8084dff..450bc98 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -228,6 +228,8 @@ mod ffi { /// Mid-level wrapper for the `nix::ValidPathInfo` struct. type CPathInfo; + /// Returns the size of the NAR. + fn nar_size(self: Pin<&mut CPathInfo>) -> u64; /// Returns the references of the store path. fn references(self: Pin<&mut CPathInfo>) -> UniquePtr>; diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp index 326e878..8bf2cb3 100644 --- a/src/bindings/nix.cpp +++ b/src/bindings/nix.cpp @@ -57,6 +57,10 @@ void RustSink::eof() { CPathInfo::CPathInfo(nix::ref pi) : pi(pi) {} +uint64_t CPathInfo::nar_size() { + return this->pi->narSize; +} + std::unique_ptr> CPathInfo::sigs() { std::vector result; for (auto&& elem : this->pi->sigs) { diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp index 5c79a33..87b3ebf 100644 --- a/src/bindings/nix.hpp +++ b/src/bindings/nix.hpp @@ -65,6 +65,7 @@ public: CPathInfo(nix::ref pi); std::unique_ptr> sigs(); std::unique_ptr> references(); + uint64_t nar_size(); }; class CNixStore { diff --git a/src/path_info.rs b/src/path_info.rs index f88e464..beea6b9 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -18,6 +18,7 @@ pub struct PathInfo { pub path: StorePath, pub signatures: Vec, pub references: Vec>, + pub nar_size: u64, } impl PathInfo { @@ -90,7 +91,7 @@ impl PathInfo { let upstream = upstream .join(self.narinfo_path().as_ref()) .expect("adding .narinfo should make a valid url"); - debug!("querying {}", upstream); + trace!("querying {}", upstream); let res_status = reqwest::Client::new() .head(upstream.as_str()) .send() diff --git a/src/push.rs b/src/push.rs index 9a9e156..8943355 100644 --- a/src/push.rs +++ b/src/push.rs @@ -114,18 +114,22 @@ impl Push { /// filter paths that are on upstream and send to `tx` async fn filter_from_upstream(&'static self, tx: mpsc::Sender) { - let mut handles = Vec::with_capacity(10); + let mut handles = Vec::new(); let store_paths = self.store_paths.read().await.clone(); + // limit number of inflight requests + let inflight_permits = Arc::new(Semaphore::new(32)); for path in store_paths.into_iter() { if path.check_upstream_signature(&self.upstream_caches) { debug!("skip {} (signature match)", path.absolute_path()); - self.signature_hit_count.fetch_add(1, Ordering::Release); + self.signature_hit_count.fetch_add(1, Ordering::Relaxed); continue; } handles.push({ let tx = tx.clone(); + let inflight_permits = inflight_permits.clone(); tokio::spawn(async move { + let _permit = inflight_permits.acquire().await.unwrap(); if !path .check_upstream_hit(self.upstream_caches.as_slice()) .await @@ -153,24 +157,35 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(10)); - let big_permits = Arc::new(Semaphore::new(2)); + let permits = Arc::new(Semaphore::new(16)); + let big_permits = Arc::new(Semaphore::new(5)); loop { let permits = permits.clone(); let big_permits = big_permits.clone(); if let Some(path_to_upload) = rx.recv().await { + debug!("upload permits available: {}", permits.available_permits()); let mut permit = permits.acquire_owned().await.unwrap(); uploads.push(tokio::spawn({ - // directory may have many files and end up causing "too many open files" - if PathBuf::from(path_to_upload.absolute_path()).is_dir() { + // a large directory may have many files and end up causing "too many open files" + if PathBuf::from(path_to_upload.absolute_path()).is_dir() + && path_to_upload.nar_size > 5 * 1024 * 1024 + { + debug!( + "upload big permits available: {}", + big_permits.available_permits() + ); // drop regular permit and take the big one permit = big_permits.acquire_owned().await.unwrap(); } - println!("uploading: {}", path_to_upload.absolute_path()); + println!( + "uploading: {} (size: {})", + path_to_upload.absolute_path(), + path_to_upload.nar_size + ); let uploader = Uploader::new(&self.signing_key, path_to_upload)?; let s3 = self.s3.clone(); let store = self.store.clone(); diff --git a/src/store.rs b/src/store.rs index 7589e94..9b925b2 100644 --- a/src/store.rs +++ b/src/store.rs @@ -69,11 +69,13 @@ impl Store { .map(|x| StorePath::from_bytes(x.as_bytes())) .collect::>() .context("get references from pathinfo")?; + let nar_size = c_path_info.pin_mut().nar_size(); Ok(PathInfo { path, signatures, references, + nar_size, }) }) .await From b49be95d09987702c5aba1a2af279014bc332302 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 27 Apr 2025 02:32:15 -0400 Subject: [PATCH 16/44] simplify here since the problem was somewhere else --- Cargo.lock | 16 ++++++++++++++++ Cargo.toml | 1 + src/push.rs | 22 ++++------------------ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3e049c..2a630e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -968,6 +968,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "humantime" version = "2.2.0" @@ -1304,6 +1313,12 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libm" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9627da5196e5d8ed0b0495e61e518847578da83483c37288316d9b2e03a7f72" + [[package]] name = "libmimalloc-sys" version = "0.1.42" @@ -1491,6 +1506,7 @@ dependencies = [ "cxx-build", "ed25519-dalek", "futures", + "humansize", "nix-compat", "object_store", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 6ca052f..272be3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ bytes = "1.10.1" object_store = { version = "0.12.0", features = ["aws"] } ulid = "1.2.1" tracing-subscriber = "0.3.19" +humansize = "2.1.3" [build-dependencies] cxx-build = "1.0" diff --git a/src/push.rs b/src/push.rs index 8943355..cd1373c 100644 --- a/src/push.rs +++ b/src/push.rs @@ -10,6 +10,7 @@ use std::{ use anyhow::{Context, Result}; use futures::future::join_all; +use humansize::{DECIMAL, format_size}; use nix_compat::narinfo::{self, SigningKey}; use object_store::aws::{AmazonS3, AmazonS3Builder}; use tokio::sync::{RwLock, Semaphore, mpsc}; @@ -157,34 +158,19 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(16)); - let big_permits = Arc::new(Semaphore::new(5)); + let permits = Arc::new(Semaphore::new(32)); loop { let permits = permits.clone(); - let big_permits = big_permits.clone(); if let Some(path_to_upload) = rx.recv().await { - debug!("upload permits available: {}", permits.available_permits()); - let mut permit = permits.acquire_owned().await.unwrap(); + let permit = permits.acquire_owned().await.unwrap(); uploads.push(tokio::spawn({ - // a large directory may have many files and end up causing "too many open files" - if PathBuf::from(path_to_upload.absolute_path()).is_dir() - && path_to_upload.nar_size > 5 * 1024 * 1024 - { - debug!( - "upload big permits available: {}", - big_permits.available_permits() - ); - // drop regular permit and take the big one - permit = big_permits.acquire_owned().await.unwrap(); - } - println!( "uploading: {} (size: {})", path_to_upload.absolute_path(), - path_to_upload.nar_size + format_size(path_to_upload.nar_size, DECIMAL) ); let uploader = Uploader::new(&self.signing_key, path_to_upload)?; let s3 = self.s3.clone(); From 01443d0d992cf1134a438076adccce9c96c7686e Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 02:46:11 -0400 Subject: [PATCH 17/44] use 10 permits but skip nars bigger than 15mb --- src/push.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/push.rs b/src/push.rs index cd1373c..c147053 100644 --- a/src/push.rs +++ b/src/push.rs @@ -158,15 +158,20 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(32)); + let permits = Arc::new(Semaphore::new(10)); loop { let permits = permits.clone(); if let Some(path_to_upload) = rx.recv().await { - let permit = permits.acquire_owned().await.unwrap(); - uploads.push(tokio::spawn({ + // large uploads will be concurrently uploaded with multipart anyway so don't spawn + // too much of them + let permit = if path_to_upload.nar_size > 15 * 1024 * 1024 { + Some(permits.acquire_owned().await.unwrap()) + } else { + None + }; println!( "uploading: {} (size: {})", path_to_upload.absolute_path(), From 9d2e9e38bd7582703698f108e17384b0d45b922e Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 03:01:24 -0400 Subject: [PATCH 18/44] split lib.rs and main.rs --- src/lib.rs | 65 ++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 74 +++-------------------------------------------------- 2 files changed, 69 insertions(+), 70 deletions(-) create mode 100644 src/lib.rs diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..0df8b02 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,65 @@ +use std::path::PathBuf; + +use clap::{Args, Parser, Subcommand}; + +mod bindings; +mod cli; +mod make_nar; +mod path_info; +pub mod push; +pub mod store; +mod uploader; + +#[derive(Parser, Debug)] +#[command(version)] +#[command(name = "nixcp")] +#[command(about = "Upload store paths to a s3 binary cache")] +#[command(long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, + + /// Whether to enable tokio-console + #[arg(long)] + pub tokio_console: bool, +} + +#[derive(Debug, Subcommand)] +pub enum Commands { + #[command(arg_required_else_help = true)] + Push(PushArgs), +} + +#[derive(Debug, Args)] +pub struct PushArgs { + /// The s3 bucket to upload to + #[arg(long, value_name = "bucket name")] + bucket: String, + + /// Upstream cache to check against. Can be specified multiple times. + /// cache.nixos.org is always included. + #[arg(long = "upstream", short, value_name = "nixcache.example.com")] + upstreams: Vec, + + /// Path to the file containing signing key + /// e.g. ~/cache-priv-key.pem + #[arg(long)] + signing_key: String, + + /// If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 + #[arg(long)] + region: Option, + + /// If unspecifed, will get it from AWS_ENDPOINT envar + /// e.g. https://s3.example.com + #[arg(long)] + endpoint: Option, + + #[arg(long)] + skip_signature_check: bool, + + /// Path to upload + /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 + #[arg(value_name = "PATH")] + pub paths: Vec, +} diff --git a/src/main.rs b/src/main.rs index 0fefdf5..1afe4b6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,76 +1,10 @@ -#![feature(let_chains)] -#![feature(exit_status_error)] - -use std::path::PathBuf; - use anyhow::{Context, Result}; -use clap::{Args, Parser, Subcommand}; +use clap::Parser; use tracing_subscriber::{EnvFilter, prelude::*}; -use push::Push; -use store::Store; - -mod bindings; -mod cli; -mod make_nar; -mod path_info; -mod push; -mod store; -mod uploader; - -#[derive(Parser, Debug)] -#[command(version)] -#[command(name = "nixcp")] -#[command(about = "Upload store paths to a s3 binary cache")] -#[command(long_about = None)] -struct Cli { - #[command(subcommand)] - command: Commands, - - /// Whether to enable tokio-console - #[arg(long)] - tokio_console: bool, -} - -#[derive(Debug, Subcommand)] -enum Commands { - #[command(arg_required_else_help = true)] - Push(PushArgs), -} - -#[derive(Debug, Args)] -pub struct PushArgs { - /// The s3 bucket to upload to - #[arg(long, value_name = "bucket name")] - bucket: String, - - /// Upstream cache to check against. Can be specified multiple times. - /// cache.nixos.org is always included. - #[arg(long = "upstream", short, value_name = "nixcache.example.com")] - upstreams: Vec, - - /// Path to the file containing signing key - /// e.g. ~/cache-priv-key.pem - #[arg(long)] - signing_key: String, - - /// If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 - #[arg(long)] - region: Option, - - /// If unspecifed, will get it from AWS_ENDPOINT envar - /// e.g. https://s3.example.com - #[arg(long)] - endpoint: Option, - - #[arg(long)] - skip_signature_check: bool, - - /// Path to upload - /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 - #[arg(value_name = "PATH")] - paths: Vec, -} +use nixcp::push::Push; +use nixcp::store::Store; +use nixcp::{Cli, Commands}; #[tokio::main] async fn main() -> Result<()> { From 7285c29e88012e58f5caeca97ba3a33a15f3bb80 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 11:11:46 -0400 Subject: [PATCH 19/44] use io::Error::other --- src/bindings/mod.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 450bc98..46b86dd 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -150,10 +150,7 @@ impl Stream for AsyncWriteAdapter { match message { Data(v) => Poll::Ready(Some(Ok(v.into()))), Error(exception) => { - let error = std::io::Error::new( - io::ErrorKind::Other, - format!("cxx error: {exception}"), - ); + let error = std::io::Error::other(format!("cxx error: {exception}")); Poll::Ready(Some(Err(error))) } Eof => { From 878e09649459062cafa43617cfd34baa1c39854f Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 13:30:49 -0400 Subject: [PATCH 20/44] add path_info tests --- src/lib.rs | 2 +- src/path_info.rs | 8 +++++++- tests/common/mod.rs | 16 ++++++++++++++++ tests/path_info.rs | 46 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 tests/common/mod.rs create mode 100644 tests/path_info.rs diff --git a/src/lib.rs b/src/lib.rs index 0df8b02..df3ba9e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,7 @@ use clap::{Args, Parser, Subcommand}; mod bindings; mod cli; mod make_nar; -mod path_info; +pub mod path_info; pub mod push; pub mod store; mod uploader; diff --git a/src/path_info.rs b/src/path_info.rs index beea6b9..5a70c39 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, anyhow}; use futures::future::join_all; use nix_compat::nixbase32; use nix_compat::store_path::StorePath; @@ -41,6 +41,12 @@ impl PathInfo { let derivation = String::from_utf8_lossy(derivation); debug!("derivation: {derivation}"); + if derivation.is_empty() { + return Err(anyhow!( + "nix path-info did not return a derivation for {path:#?}" + )); + } + let store_path = StorePath::from_absolute_path(derivation.trim().as_bytes()) .context("storepath from derivation")?; store diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..96ba300 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,16 @@ +use nixcp::store::Store; + +pub struct Context { + pub store: Store, +} + +impl Context { + fn new() -> Self { + let store = Store::connect().expect("connect to nix store"); + Self { store } + } +} + +pub fn context() -> Context { + Context::new() +} diff --git a/tests/path_info.rs b/tests/path_info.rs new file mode 100644 index 0000000..2653820 --- /dev/null +++ b/tests/path_info.rs @@ -0,0 +1,46 @@ +use nixcp::path_info::PathInfo; +use std::path::PathBuf; +use std::process::Command; + +mod common; + +const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; +const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; + +#[tokio::test] +async fn path_info_from_package() { + let ctx = common::context(); + let path = PathBuf::from(HELLO); + let path_info = PathInfo::from_path(&path, &ctx.store) + .await + .expect("get pathinfo from package"); + assert_eq!(path_info.path.to_string(), HELLO_DRV); +} + +#[tokio::test] +async fn path_info_from_path() { + // the path must be in the store + Command::new("nix") + .arg("build") + .arg("--no-link") + .arg(HELLO) + .status() + .unwrap(); + let ctx = common::context(); + let path = PathBuf::from("/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"); + let path_info = PathInfo::from_path(&path, &ctx.store) + .await + .expect("get pathinfo from package"); + assert_eq!(path_info.path.to_string(), HELLO_DRV); +} + +#[tokio::test] +async fn closure() { + let ctx = common::context(); + let path = PathBuf::from(HELLO); + let path_info = PathInfo::from_path(&path, &ctx.store) + .await + .expect("get pathinfo from package"); + let closure = path_info.get_closure(&ctx.store).await.unwrap(); + assert_eq!(closure.len(), 466); +} From 54d4c714afe54ea2be3fa12ef55982928f6be487 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 20:50:16 -0400 Subject: [PATCH 21/44] rename from_path to from_derivation --- src/path_info.rs | 24 ++++++++++++++---------- src/push.rs | 2 +- tests/path_info.rs | 6 +++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 5a70c39..0698533 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -22,19 +22,19 @@ pub struct PathInfo { } impl PathInfo { - pub async fn from_path(path: &Path, store: &Store) -> Result { - debug!("query path info for {:?}", path); + pub async fn from_derivation(drv: &Path, store: &Store) -> Result { + debug!("query path info for {:?}", drv); - let derivation = match path.extension() { - Some(ext) if ext == "drv" => path.as_os_str().as_encoded_bytes(), + let derivation = match drv.extension() { + Some(ext) if ext == "drv" => drv.as_os_str().as_encoded_bytes(), _ => { &Command::new("nix") .arg("path-info") .arg("--derivation") - .arg(path) + .arg(drv) .output() .await - .context(format!("run command: nix path-info --derivaiton {path:?}"))? + .context(format!("run command: nix path-info --derivaiton {drv:?}"))? .stdout } }; @@ -43,16 +43,20 @@ impl PathInfo { if derivation.is_empty() { return Err(anyhow!( - "nix path-info did not return a derivation for {path:#?}" + "nix path-info did not return a derivation for {drv:#?}" )); } - let store_path = StorePath::from_absolute_path(derivation.trim().as_bytes()) - .context("storepath from derivation")?; + Self::from_path(derivation.trim(), store).await + } + + pub async fn from_path(path: &str, store: &Store) -> Result { + let store_path = + StorePath::from_absolute_path(path.as_bytes()).context("storepath from path")?; store .query_path_info(store_path) .await - .context("query pathinfo for derivation") + .context("query pathinfo for path") } pub async fn get_closure(&self, store: &Store) -> Result> { diff --git a/src/push.rs b/src/push.rs index c147053..5f7a591 100644 --- a/src/push.rs +++ b/src/push.rs @@ -79,7 +79,7 @@ impl Push { let store = self.store.clone(); futs.push(tokio::spawn(async move { - let path_info = PathInfo::from_path(path.as_path(), &store) + let path_info = PathInfo::from_derivation(path.as_path(), &store) .await .context("get path info for path")?; debug!("path-info for {path:?}: {path_info:?}"); diff --git a/tests/path_info.rs b/tests/path_info.rs index 2653820..59b3dfd 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -11,7 +11,7 @@ const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; async fn path_info_from_package() { let ctx = common::context(); let path = PathBuf::from(HELLO); - let path_info = PathInfo::from_path(&path, &ctx.store) + let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); assert_eq!(path_info.path.to_string(), HELLO_DRV); @@ -28,7 +28,7 @@ async fn path_info_from_path() { .unwrap(); let ctx = common::context(); let path = PathBuf::from("/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"); - let path_info = PathInfo::from_path(&path, &ctx.store) + let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); assert_eq!(path_info.path.to_string(), HELLO_DRV); @@ -38,7 +38,7 @@ async fn path_info_from_path() { async fn closure() { let ctx = common::context(); let path = PathBuf::from(HELLO); - let path_info = PathInfo::from_path(&path, &ctx.store) + let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); let closure = path_info.get_closure(&ctx.store).await.unwrap(); From 09181ae7852fb399a9f575e62ab3a98ba0b4e93d Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 17:37:04 -0400 Subject: [PATCH 22/44] add nar tests --- src/lib.rs | 2 +- src/make_nar.rs | 4 ++-- tests/common/mod.rs | 12 ++++++++++-- tests/nar.rs | 26 ++++++++++++++++++++++++++ tests/path_info.rs | 7 +++---- 5 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 tests/nar.rs diff --git a/src/lib.rs b/src/lib.rs index df3ba9e..dfbab4f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ use clap::{Args, Parser, Subcommand}; mod bindings; mod cli; -mod make_nar; +pub mod make_nar; pub mod path_info; pub mod push; pub mod store; diff --git a/src/make_nar.rs b/src/make_nar.rs index b5c1ab2..97d6b1f 100644 --- a/src/make_nar.rs +++ b/src/make_nar.rs @@ -15,10 +15,10 @@ use crate::store::Store; pub struct MakeNar<'a> { path_info: &'a PathInfo, store: Arc, - nar_hasher: Sha256, + pub nar_hasher: Sha256, /// hash of compressed nar file file_hasher: Sha256, - nar_size: u64, + pub nar_size: u64, file_size: u64, } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 96ba300..aed468f 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,12 +1,20 @@ +#![allow(dead_code)] + +use std::sync::Arc; + use nixcp::store::Store; +pub const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; +pub const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; +pub const HELLO_PATH: &str = "/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"; + pub struct Context { - pub store: Store, + pub store: Arc, } impl Context { fn new() -> Self { - let store = Store::connect().expect("connect to nix store"); + let store = Arc::new(Store::connect().expect("connect to nix store")); Self { store } } } diff --git a/tests/nar.rs b/tests/nar.rs new file mode 100644 index 0000000..5ebadc5 --- /dev/null +++ b/tests/nar.rs @@ -0,0 +1,26 @@ +use crate::common::HELLO_PATH; +use nix_compat::nixbase32; +use nixcp::make_nar::MakeNar; +use nixcp::path_info::PathInfo; +use sha2::Digest; +use tokio::io::AsyncReadExt; + +mod common; + +#[tokio::test] +async fn nar_size_and_hash() { + let ctx = common::context(); + let path_info = PathInfo::from_path(HELLO_PATH, &ctx.store).await.unwrap(); + + let mut nar = MakeNar::new(&path_info, ctx.store).unwrap(); + let mut reader = nar.compress_and_hash().unwrap(); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + drop(reader); + + assert_eq!(nar.nar_size, 234680); + + let nar_hash = nar.nar_hasher.finalize(); + let real_nar_hash = "08za7nnjda8kpdsd73v3mhykjvp0rsmskwsr37winhmzgm6iw79w"; + assert_eq!(nixbase32::encode(nar_hash.as_slice()), real_nar_hash); +} diff --git a/tests/path_info.rs b/tests/path_info.rs index 59b3dfd..2d698d0 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -2,10 +2,9 @@ use nixcp::path_info::PathInfo; use std::path::PathBuf; use std::process::Command; -mod common; +use crate::common::{HELLO, HELLO_DRV, HELLO_PATH}; -const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; -const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; +mod common; #[tokio::test] async fn path_info_from_package() { @@ -27,7 +26,7 @@ async fn path_info_from_path() { .status() .unwrap(); let ctx = common::context(); - let path = PathBuf::from("/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"); + let path = PathBuf::from(HELLO_PATH); let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); From 76cbc85032965cea0cb9188b21900f3d3e683bd7 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 21:39:14 -0400 Subject: [PATCH 23/44] use hashset for closure --- src/path_info.rs | 3 ++- src/push.rs | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 0698533..1e1282d 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -13,7 +13,7 @@ use url::Url; use crate::store::Store; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct PathInfo { pub path: StorePath, pub signatures: Vec, @@ -59,6 +59,7 @@ impl PathInfo { .context("query pathinfo for path") } + // TODO: skip call to query_path_info and return Vec? pub async fn get_closure(&self, store: &Store) -> Result> { let futs = store .compute_fs_closure(self.path.clone()) diff --git a/src/push.rs b/src/push.rs index 5f7a591..bf25ea1 100644 --- a/src/push.rs +++ b/src/push.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashSet, fs, iter::once, path::PathBuf, @@ -21,7 +22,7 @@ use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; pub struct Push { upstream_caches: Vec, - store_paths: Arc>>, + store_paths: Arc>>, signing_key: SigningKey, store: Arc, s3: Arc, @@ -61,7 +62,7 @@ impl Push { Ok(Self { upstream_caches: upstreams, - store_paths: Arc::new(RwLock::new(Vec::new())), + store_paths: Arc::new(RwLock::new(HashSet::new())), signing_key, store: Arc::new(store), s3: Arc::new(s3_builder.build()?), From 05589641cfd59f093a6fd38d30bd34467713f4e4 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 21:39:14 -0400 Subject: [PATCH 24/44] move "nix build" to tests/common --- tests/common/mod.rs | 8 ++++++++ tests/path_info.rs | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index aed468f..3870a1d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] +use std::process::Command; use std::sync::Arc; use nixcp::store::Store; @@ -14,6 +15,13 @@ pub struct Context { impl Context { fn new() -> Self { + // hello must be in the store + Command::new("nix") + .arg("build") + .arg("--no-link") + .arg(HELLO) + .status() + .unwrap(); let store = Arc::new(Store::connect().expect("connect to nix store")); Self { store } } diff --git a/tests/path_info.rs b/tests/path_info.rs index 2d698d0..90d113d 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -1,6 +1,5 @@ use nixcp::path_info::PathInfo; use std::path::PathBuf; -use std::process::Command; use crate::common::{HELLO, HELLO_DRV, HELLO_PATH}; @@ -18,13 +17,6 @@ async fn path_info_from_package() { #[tokio::test] async fn path_info_from_path() { - // the path must be in the store - Command::new("nix") - .arg("build") - .arg("--no-link") - .arg(HELLO) - .status() - .unwrap(); let ctx = common::context(); let path = PathBuf::from(HELLO_PATH); let path_info = PathInfo::from_derivation(&path, &ctx.store) From a0794b03564c6f045c25cc96c93922a43afea367 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 01:15:10 -0400 Subject: [PATCH 25/44] fix closure test check; rm RUST_LOG envar in devshell --- flake.nix | 1 - tests/path_info.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 01d135a..fb86d40 100644 --- a/flake.nix +++ b/flake.nix @@ -59,7 +59,6 @@ devShells.default = craneLib.devShell { inputsFrom = [ nixcp ]; - RUST_LOG = "nixcp=debug"; RUST_BACKGRACE = 1; # for cpp bindings to work NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; diff --git a/tests/path_info.rs b/tests/path_info.rs index 90d113d..57738fd 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -33,5 +33,5 @@ async fn closure() { .await .expect("get pathinfo from package"); let closure = path_info.get_closure(&ctx.store).await.unwrap(); - assert_eq!(closure.len(), 466); + assert_eq!(closure.len(), 472); } From 0e97d1174503c33a40810dbec0922278867636fe Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 01:48:13 -0400 Subject: [PATCH 26/44] skip integration tests in nix package These tests need a connection to the nix store and we can't have that in the nix build environment. --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index fb86d40..2d1191f 100644 --- a/flake.nix +++ b/flake.nix @@ -48,6 +48,8 @@ ]; # for cpp bindings to work NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; + # skip integration tests (they need a connection to the nix store) + cargoTestExtraArgs = "--bins"; }; cargoArtifacts = craneLib.buildDepsOnly commonArgs; From 14d6e9d29eca6fea2ef32f99ec452b68da208c77 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 01:48:13 -0400 Subject: [PATCH 27/44] path_info: check for and resolve symlink --- Cargo.toml | 10 ++++++---- src/path_info.rs | 8 ++++++++ tests/path_info.rs | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 272be3b..c1de8ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,15 +16,14 @@ futures = "0.3.31" nix-compat = { git = "https://github.com/tvlfyi/tvix.git", version = "0.1.0" } regex = "1.11.1" reqwest = "0.12.15" -serde = { version = "1.0.219", features = [ "derive" ]} +serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" sha2 = "0.10.8" -tokio = { version = "1.44.1", features = [ "full", "tracing", "parking_lot" ]} +tokio = { version = "1.44.1", features = ["full", "tracing", "parking_lot"] } tracing = "0.1.41" -url = { version = "2.5.4", features = [ "serde" ]} +url = { version = "2.5.4", features = ["serde"] } cxx = "1.0" console-subscriber = "0.4.1" -tempfile = "3.19.1" tokio-util = { version = "0.7.15", features = ["io"] } bytes = "1.10.1" object_store = { version = "0.12.0", features = ["aws"] } @@ -35,3 +34,6 @@ humansize = "2.1.3" [build-dependencies] cxx-build = "1.0" pkg-config = "0.3.32" + +[dev-dependencies] +tempfile = "3.19.1" diff --git a/src/path_info.rs b/src/path_info.rs index 1e1282d..213fd1a 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -28,6 +28,14 @@ impl PathInfo { let derivation = match drv.extension() { Some(ext) if ext == "drv" => drv.as_os_str().as_encoded_bytes(), _ => { + let drv = { + // resolve symlink + if drv.is_symlink() { + &drv.canonicalize()? + } else { + drv + } + }; &Command::new("nix") .arg("path-info") .arg("--derivation") diff --git a/tests/path_info.rs b/tests/path_info.rs index 57738fd..0f9543b 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -1,6 +1,8 @@ use nixcp::path_info::PathInfo; use std::path::PathBuf; +use tempfile::TempDir; + use crate::common::{HELLO, HELLO_DRV, HELLO_PATH}; mod common; @@ -25,6 +27,23 @@ async fn path_info_from_path() { assert_eq!(path_info.path.to_string(), HELLO_DRV); } +#[tokio::test] +async fn path_info_symlink() { + let ctx = common::context(); + + let temp_path = TempDir::new().unwrap(); + let link_path = temp_path.path().join("result"); + + // symlink at ./result (like `nix build`) + std::os::unix::fs::symlink(HELLO_PATH, &link_path).unwrap(); + + // should resolve symlink + let path_info = PathInfo::from_derivation(&link_path, &ctx.store) + .await + .expect("get pathinfo from package"); + assert_eq!(path_info.path.to_string(), HELLO_DRV); +} + #[tokio::test] async fn closure() { let ctx = common::context(); From 9b0c6aece69147aeeaf300b1c8db8d7b131226db Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 01:48:13 -0400 Subject: [PATCH 28/44] add .editorconfig --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d17805f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,3 @@ +[*.nix] +indent_size = 2 +indent_stype = space From 3c40776981d6d1a82a537c0fc6c026ef043b3111 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 03:06:43 -0400 Subject: [PATCH 29/44] update readme --- README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ff71137..55fdaef 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,13 @@ The signing key is generated with: nix-store --generate-binary-cache-key nixcache.cy7.sh cache-priv-key.pem cache-pub-key.pem ``` -`AWS_ACCESS_KEY_ID` and `AWS_ENDPOINT_URL` environment variables should be set with your s3 credentials. +`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables should be set with your s3 credentials. ``` -Usage: nixcp [OPTIONS] --bucket --signing-key +Usage: nixcp push [OPTIONS] --bucket --signing-key [PATH]... -Commands: - push - help Print this message or the help of the given subcommand(s) +Arguments: + [PATH]... Path to upload e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 Options: --bucket @@ -28,15 +27,13 @@ Options: --signing-key Path to the file containing signing key e.g. ~/cache-priv-key.pem --region - If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default + If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 --endpoint - If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default e.g. https://s3.example.com - --profile - AWS profile to use + If unspecifed, will get it from AWS_ENDPOINT envar e.g. https://s3.example.com + --skip-signature-check + -h, --help Print help - -V, --version - Print version ``` ## Install with nix From d03058d125c2ef2bf056b99fff8e2990928a23de Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 03:18:51 -0400 Subject: [PATCH 30/44] add build workflow --- .github/workflows/build.yml | 61 +++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..8332f1c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,61 @@ +name: build +on: + workflow_dispatch: + push: + pull_request: + +env: + TERM: ansi + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT: https://s3.cy7.sh + +jobs: + build-packages: + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - ubuntu-24.04-arm + runs-on: ${{ matrix.os }} + + steps: + - name: setup binary cache key + run: echo -n "${{ secrets.NIX_CACHE_SECRET_KEY }}" | xxd -p -r > ${{ runner.temp }}/cache-priv-key.pem + + - name: Install Nix + uses: cachix/install-nix-action@v30 + with: + enable_kvm: true + extra_nix_config: | + show-trace = true + experimental-features = nix-command flakes + secret-key-files = ${{ runner.temp }}/cache-priv-key.pem + extra-substituters = https://nixcache.cy7.sh + extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= + + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - run: nix build -L . + + - name: cache + run: | + nix run \ + github:cything/nixcp/test-in-ci -- push \ + --bucket nixcache \ + --signing-key ${{ runner.temp }}/cache-priv-key.pem \ + result + + - name: prepare tarball to upload + run: nix run github:nixos/nixpkgs#gnutar hcvf result.tar result + + - name: upload result + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}.tar + path: result.tar + if-no-files-found: error From 688fcd8706d5f9e456c61822ec0d279f1fb6eac9 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 03:18:51 -0400 Subject: [PATCH 31/44] add test workflow --- .github/workflows/test.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..12ef747 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: test +on: + workflow_dispatch: + push: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Install Nix + uses: cachix/install-nix-action@v30 + with: + enable_kvm: true + extra_nix_config: | + show-trace = true + experimental-features = nix-command flakes + extra-substituters = https://nixcache.cy7.sh + extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= + + - uses: actions/checkout@v4 + + - name: Run tests + run: nix develop -c cargo test --verbose From 96ae0ca647474cb2012aa75108128f6fb2c3d668 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 03:18:51 -0400 Subject: [PATCH 32/44] disable closure test Size of the closure is not deterministic at all. I guess it's because if the package was sourced from a cache, some dependencies may or may not exist in the local nix store. --- tests/path_info.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/path_info.rs b/tests/path_info.rs index 0f9543b..2558703 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -44,6 +44,7 @@ async fn path_info_symlink() { assert_eq!(path_info.path.to_string(), HELLO_DRV); } +/* #[tokio::test] async fn closure() { let ctx = common::context(); @@ -54,3 +55,4 @@ async fn closure() { let closure = path_info.get_closure(&ctx.store).await.unwrap(); assert_eq!(closure.len(), 472); } +*/ From 6cfe67af0e8da502702b31f34a941753e64d9561 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 12:01:36 -0400 Subject: [PATCH 33/44] improve closure test We now compare against the output of `nix-store --query --requisites --include-outputs`. Our closure should include everything this command would output. --- tests/path_info.rs | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/tests/path_info.rs b/tests/path_info.rs index 2558703..f9c2ad2 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -1,5 +1,5 @@ use nixcp::path_info::PathInfo; -use std::path::PathBuf; +use std::{collections::HashSet, path::PathBuf, process::Command}; use tempfile::TempDir; @@ -44,15 +44,37 @@ async fn path_info_symlink() { assert_eq!(path_info.path.to_string(), HELLO_DRV); } -/* #[tokio::test] -async fn closure() { +async fn closure_includes_nix_store_requisites() { let ctx = common::context(); let path = PathBuf::from(HELLO); let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); - let closure = path_info.get_closure(&ctx.store).await.unwrap(); - assert_eq!(closure.len(), 472); + + // get what we think is the closure + let closure: HashSet = path_info + .get_closure(&ctx.store) + .await + .unwrap() + .iter() + .map(|x| x.path.to_absolute_path()) + .collect(); + + // get output of `nix-store --query --requisites --include-outputs` + let nix_store_out = Command::new("nix-store") + .arg("--query") + .arg("--requisites") + .arg("--include-outputs") + .arg(HELLO_PATH) + .output() + .unwrap() + .stdout; + let ref_closure = String::from_utf8_lossy(&nix_store_out); + let ref_closure = ref_closure.split_whitespace(); + + // check that we didn't miss anything nix-store would catch + for path in ref_closure { + assert!(closure.contains(path)); + } } -*/ From d9ca033a144d615f686d3ce55cdbce8a3149beeb Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 13:01:36 -0400 Subject: [PATCH 34/44] add additional case in closure test and run nix-store against the derivation --- tests/common/mod.rs | 20 +++++++++++++------- tests/path_info.rs | 29 +++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 3870a1d..4c2d932 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,8 +6,10 @@ use std::sync::Arc; use nixcp::store::Store; pub const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; -pub const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; +pub const HELLO_DRV: &str = "/nix/store/iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; pub const HELLO_PATH: &str = "/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"; +pub const NIXCP_PKG: &str = "github:cything/nixcp?ref=6cfe67af0e8da502702b31f34a941753e64d9561"; +pub const NIXCP_DRV: &str = "/nix/store/ldjvf9qjp980dyvka2hj99q4c0w6901x-nixcp-0.1.0.drv"; pub struct Context { pub store: Arc, @@ -16,12 +18,7 @@ pub struct Context { impl Context { fn new() -> Self { // hello must be in the store - Command::new("nix") - .arg("build") - .arg("--no-link") - .arg(HELLO) - .status() - .unwrap(); + ensure_exists(HELLO); let store = Arc::new(Store::connect().expect("connect to nix store")); Self { store } } @@ -30,3 +27,12 @@ impl Context { pub fn context() -> Context { Context::new() } + +pub fn ensure_exists(pkg: &str) { + Command::new("nix") + .arg("build") + .arg("--no-link") + .arg(pkg) + .status() + .unwrap(); +} diff --git a/tests/path_info.rs b/tests/path_info.rs index f9c2ad2..d71f9d6 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -3,7 +3,7 @@ use std::{collections::HashSet, path::PathBuf, process::Command}; use tempfile::TempDir; -use crate::common::{HELLO, HELLO_DRV, HELLO_PATH}; +use crate::common::{HELLO, HELLO_DRV, HELLO_PATH, NIXCP_DRV, NIXCP_PKG}; mod common; @@ -14,7 +14,7 @@ async fn path_info_from_package() { let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_string(), HELLO_DRV); + assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); } #[tokio::test] @@ -24,7 +24,7 @@ async fn path_info_from_path() { let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_string(), HELLO_DRV); + assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); } #[tokio::test] @@ -41,7 +41,7 @@ async fn path_info_symlink() { let path_info = PathInfo::from_derivation(&link_path, &ctx.store) .await .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_string(), HELLO_DRV); + assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); } #[tokio::test] @@ -53,7 +53,7 @@ async fn closure_includes_nix_store_requisites() { .expect("get pathinfo from package"); // get what we think is the closure - let closure: HashSet = path_info + let mut closure: HashSet = path_info .get_closure(&ctx.store) .await .unwrap() @@ -61,15 +61,32 @@ async fn closure_includes_nix_store_requisites() { .map(|x| x.path.to_absolute_path()) .collect(); + // for a somewhat more complicated case + common::ensure_exists(NIXCP_PKG); + let path = PathBuf::from(NIXCP_PKG); + let path_info = PathInfo::from_derivation(&path, &ctx.store) + .await + .expect("get pathinfo from package"); + closure.extend( + path_info + .get_closure(&ctx.store) + .await + .unwrap() + .iter() + .map(|x| x.path.to_absolute_path()), + ); + // get output of `nix-store --query --requisites --include-outputs` let nix_store_out = Command::new("nix-store") .arg("--query") .arg("--requisites") .arg("--include-outputs") - .arg(HELLO_PATH) + .arg(HELLO_DRV) + .arg(NIXCP_DRV) .output() .unwrap() .stdout; + assert!(!nix_store_out.is_empty()); let ref_closure = String::from_utf8_lossy(&nix_store_out); let ref_closure = ref_closure.split_whitespace(); From 76e6c6c537a1ec1d126ef1afd41b00cc29459969 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 11 May 2025 00:02:08 -0400 Subject: [PATCH 35/44] add license --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e886c24 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Cy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 68df59ad25563b812829651e90ae1865bbeb42b0 Mon Sep 17 00:00:00 2001 From: cy Date: Wed, 7 May 2025 17:14:25 -0400 Subject: [PATCH 36/44] use & instead of as_slice() --- src/push.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/push.rs b/src/push.rs index bf25ea1..7c23e39 100644 --- a/src/push.rs +++ b/src/push.rs @@ -132,10 +132,7 @@ impl Push { let inflight_permits = inflight_permits.clone(); tokio::spawn(async move { let _permit = inflight_permits.acquire().await.unwrap(); - if !path - .check_upstream_hit(self.upstream_caches.as_slice()) - .await - { + if !path.check_upstream_hit(&self.upstream_caches).await { if path.check_if_already_exists(&self.s3).await { debug!("skip {} (already exists)", path.absolute_path()); self.already_exists_count.fetch_add(1, Ordering::Relaxed); From ce0e70f95a5c45ad5082364bece63bb3883191dd Mon Sep 17 00:00:00 2001 From: cy Date: Wed, 7 May 2025 17:14:25 -0400 Subject: [PATCH 37/44] add option to disable cache.nixos.org --- README.md | 4 ++-- src/lib.rs | 3 ++- src/push.rs | 13 +++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 55fdaef..f9317c8 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ Options: If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 --endpoint If unspecifed, will get it from AWS_ENDPOINT envar e.g. https://s3.example.com - --skip-signature-check - + --no-default-upstream + Do not include cache.nixos.org as upstream -h, --help Print help ``` diff --git a/src/lib.rs b/src/lib.rs index dfbab4f..fa4a43d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -55,8 +55,9 @@ pub struct PushArgs { #[arg(long)] endpoint: Option, + /// Do not include cache.nixos.org as upstream #[arg(long)] - skip_signature_check: bool, + no_default_upstream: bool, /// Path to upload /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 diff --git a/src/push.rs b/src/push.rs index 7c23e39..9fc043d 100644 --- a/src/push.rs +++ b/src/push.rs @@ -1,7 +1,6 @@ use std::{ collections::HashSet, fs, - iter::once, path::PathBuf, sync::{ Arc, @@ -39,11 +38,13 @@ pub struct Push { impl Push { pub async fn new(cli: &PushArgs, store: Store) -> Result { let mut upstreams = Vec::with_capacity(cli.upstreams.len() + 1); - for upstream in cli - .upstreams - .iter() - .chain(once(&"https://cache.nixos.org".to_string())) - { + if !cli.no_default_upstream { + upstreams.push( + Url::parse("https://cache.nixos.org") + .expect("default upstream must be a valid url"), + ); + } + for upstream in &cli.upstreams { upstreams .push(Url::parse(upstream).context(format!("failed to parse {upstream} as url"))?); } From 112654f4480596d9a22014d0fa6b2065ef4c7ea6 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 11 May 2025 00:32:41 -0400 Subject: [PATCH 38/44] add flake checks and formatter, run some formatters --- flake.nix | 50 ++++++++++++++++++++++++++++++++++++--------- rust-toolchain.toml | 7 +------ src/cli.rs | 2 -- src/lib.rs | 1 - 4 files changed, 41 insertions(+), 19 deletions(-) delete mode 100644 src/cli.rs diff --git a/flake.nix b/flake.nix index 2d1191f..6e1b9d6 100644 --- a/flake.nix +++ b/flake.nix @@ -11,8 +11,15 @@ }; }; - outputs = inputs@{ nixpkgs, flake-utils, crane, ... }: - flake-utils.lib.eachDefaultSystem (system: + outputs = + inputs@{ + nixpkgs, + flake-utils, + crane, + ... + }: + flake-utils.lib.eachDefaultSystem ( + system: let pkgs = import nixpkgs { inherit system; @@ -21,13 +28,12 @@ ]; }; toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; - craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); + craneLib = (crane.mkLib pkgs).overrideToolchain (_: toolchain); lib = pkgs.lib; # don't clean cpp files cppFilter = path: _type: builtins.match ".*(cpp|hpp)$" path != null; - cppOrCargo = path: type: - (cppFilter path type) || (craneLib.filterCargoSources path type); + cppOrCargo = path: type: (cppFilter path type) || (craneLib.filterCargoSources path type); src = lib.cleanSourceWith { src = ./.; filter = cppOrCargo; @@ -48,16 +54,38 @@ ]; # for cpp bindings to work NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; - # skip integration tests (they need a connection to the nix store) - cargoTestExtraArgs = "--bins"; + # skip integration tests (they need a connection to the nix store) + cargoTestExtraArgs = "--bins"; }; cargoArtifacts = craneLib.buildDepsOnly commonArgs; - nixcp = craneLib.buildPackage (commonArgs // { - inherit cargoArtifacts; - }); + nixcp = craneLib.buildPackage ( + commonArgs + // { + inherit cargoArtifacts; + } + ); in { + checks = { + # clippy with all warnings denied + clippy = craneLib.cargoClippy ( + commonArgs + // { + inherit cargoArtifacts; + cargoClippyExtraArgs = "--all-targets -- --deny warnings"; + } + ); + + # check formatting + cargoFmt = craneLib.cargoFmt { + inherit src; + }; + tomlFmt = craneLib.taploFmt { + src = lib.sources.sourceFilesBySuffices src [ ".toml" ]; + }; + }; + devShells.default = craneLib.devShell { inputsFrom = [ nixcp ]; @@ -71,6 +99,8 @@ ]; }; + formatter = pkgs.nixfmt-rfc-style; + packages.default = nixcp; } ); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index eceaf24..c96aa24 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,9 +1,4 @@ [toolchain] channel = "nightly" profile = "minimal" -components = [ - "rust-src", - "rust-analyzer", - "rustfmt", - "clippy", -] \ No newline at end of file +components = ["rust-src", "rust-analyzer", "rustfmt", "clippy"] diff --git a/src/cli.rs b/src/cli.rs deleted file mode 100644 index 139597f..0000000 --- a/src/cli.rs +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/src/lib.rs b/src/lib.rs index fa4a43d..8b1fc18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,6 @@ use std::path::PathBuf; use clap::{Args, Parser, Subcommand}; mod bindings; -mod cli; pub mod make_nar; pub mod path_info; pub mod push; From 2b52792959ab518b1ef097c71d97b1412138c3f3 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 11 May 2025 01:01:01 -0400 Subject: [PATCH 39/44] add flake check workflow --- .github/workflows/check.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/check.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..896aad1 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,25 @@ +name: check +on: + workflow_dispatch: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Install Nix + uses: cachix/install-nix-action@v30 + with: + enable_kvm: true + extra_nix_config: | + show-trace = true + experimental-features = nix-command flakes + extra-substituters = https://nixcache.cy7.sh + extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= + + - uses: actions/checkout@v4 + + - name: Run checks + run: nix flake check -L From a9957162128440ca41917054a413b8041230523b Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 11 May 2025 01:04:15 -0400 Subject: [PATCH 40/44] pin workflows by sha --- .github/workflows/build.yml | 7 +++---- .github/workflows/check.yml | 6 ++++-- .github/workflows/test.yml | 6 ++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8332f1c..090eec8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: run: echo -n "${{ secrets.NIX_CACHE_SECRET_KEY }}" | xxd -p -r > ${{ runner.temp }}/cache-priv-key.pem - name: Install Nix - uses: cachix/install-nix-action@v30 + uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 with: enable_kvm: true extra_nix_config: | @@ -35,8 +35,7 @@ jobs: extra-substituters = https://nixcache.cy7.sh extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - name: Sync repository - uses: actions/checkout@v4 + - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 with: persist-credentials: false @@ -54,7 +53,7 @@ jobs: run: nix run github:nixos/nixpkgs#gnutar hcvf result.tar result - name: upload result - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@6027e3dd177782cd8ab9af838c04fd81a07f1d47 with: name: ${{ matrix.os }}.tar path: result.tar diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 896aad1..0b9ac66 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Install Nix - uses: cachix/install-nix-action@v30 + uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 with: enable_kvm: true extra_nix_config: | @@ -19,7 +19,9 @@ jobs: extra-substituters = https://nixcache.cy7.sh extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - uses: actions/checkout@v4 + - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 + with: + persist-credentials: false - name: Run checks run: nix flake check -L diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 12ef747..cc7fabc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Install Nix - uses: cachix/install-nix-action@v30 + uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 with: enable_kvm: true extra_nix_config: | @@ -22,7 +22,9 @@ jobs: extra-substituters = https://nixcache.cy7.sh extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - uses: actions/checkout@v4 + - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 + with: + persist-credentials: false - name: Run tests run: nix develop -c cargo test --verbose From ab1fcc820760b93dd87e74a4a5652c726d7d6566 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 11 May 2025 01:19:53 -0400 Subject: [PATCH 41/44] run builds on mac --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 090eec8..a9864f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,9 @@ jobs: os: - ubuntu-latest - ubuntu-24.04-arm + - macos-latest # arm64 + - macos-13 # x86 + runs-on: ${{ matrix.os }} steps: From 8ba2c6cc9bf7004eb812dafc971a321e108eef84 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 11 May 2025 01:22:03 -0400 Subject: [PATCH 42/44] cache devshell in ci --- .github/workflows/build.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a9864f3..b9be0e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,17 @@ jobs: with: persist-credentials: false - - run: nix build -L . + - name: cache devshell + run: | + nix build .#devShells.$(nix eval --impure --raw --expr 'builtins.currentSystem').default + nix run \ + github:cything/nixcp/test-in-ci -- push \ + --bucket nixcache \ + --signing-key ${{ runner.temp }}/cache-priv-key.pem \ + result + + - name: build + run: nix build -L . - name: cache run: | From 139dcf2fe73c51b73158a6dbda132dcf5c2728d3 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 11 May 2025 01:22:51 -0400 Subject: [PATCH 43/44] use main branch in ci --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b9be0e9..a4560bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: run: | nix build .#devShells.$(nix eval --impure --raw --expr 'builtins.currentSystem').default nix run \ - github:cything/nixcp/test-in-ci -- push \ + github:cything/nixcp -- push \ --bucket nixcache \ --signing-key ${{ runner.temp }}/cache-priv-key.pem \ result @@ -57,7 +57,7 @@ jobs: - name: cache run: | nix run \ - github:cything/nixcp/test-in-ci -- push \ + github:cything/nixcp -- push \ --bucket nixcache \ --signing-key ${{ runner.temp }}/cache-priv-key.pem \ result From 885a49701c8f0de204467fcbe0089b67a5cf1c09 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 11 May 2025 01:34:12 -0400 Subject: [PATCH 44/44] add cargo-audit to devshell --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 6e1b9d6..16b57e6 100644 --- a/flake.nix +++ b/flake.nix @@ -96,6 +96,7 @@ packages = with pkgs; [ tokio-console cargo-udeps + cargo-audit ]; };