From c956d6741a802b10f0dff1620716da05ac881e8a Mon Sep 17 00:00:00 2001 From: cy Date: Tue, 15 Apr 2025 23:49:34 -0400 Subject: [PATCH 01/32] bring code from attic --- src/bindings/mod.rs | 270 +++++++++++++++++++++++++++++++++++++++++++ src/bindings/nix.cpp | 173 +++++++++++++++++++++++++++ src/bindings/nix.hpp | 96 +++++++++++++++ 3 files changed, 539 insertions(+) create mode 100644 src/bindings/mod.rs create mode 100644 src/bindings/nix.cpp create mode 100644 src/bindings/nix.hpp diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs new file mode 100644 index 0000000..521df2e --- /dev/null +++ b/src/bindings/mod.rs @@ -0,0 +1,270 @@ +/* +Copyright 2022 Zhaofeng Li and the Attic contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//! `libnixstore` Bindings + +use std::cell::UnsafeCell; +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use futures::stream::{Stream, StreamExt}; +use tokio::io::{AsyncWrite, AsyncWriteExt}; + +use crate::{AtticError, AtticResult}; + +// The C++ implementation takes care of concurrency +#[repr(transparent)] +pub struct FfiNixStore(UnsafeCell>); + +unsafe impl Send for FfiNixStore {} +unsafe impl Sync for FfiNixStore {} + +impl FfiNixStore { + pub fn store(&self) -> Pin<&mut ffi::CNixStore> { + unsafe { + let ptr = self.0.get().as_mut().unwrap(); + ptr.pin_mut() + } + } +} + +/// Obtain a handle to the Nix store. +pub unsafe fn open_nix_store() -> AtticResult { + match ffi::open_nix_store() { + Ok(ptr) => { + let cell = UnsafeCell::new(ptr); + Ok(FfiNixStore(cell)) + } + Err(e) => Err(e.into()), + } +} + +// TODO: Benchmark different implementations +// (tokio, crossbeam, flume) +mod mpsc { + // Tokio + pub use tokio::sync::mpsc::{ + UnboundedReceiver, UnboundedSender, error::SendError, unbounded_channel, + }; +} + +/// Async write request. +#[derive(Debug)] +enum AsyncWriteMessage { + Data(Vec), + Error(String), + Eof, +} + +/// Async write request sender. +#[derive(Clone)] +pub struct AsyncWriteSender { + sender: mpsc::UnboundedSender, +} + +impl AsyncWriteSender { + fn send(&mut self, data: &[u8]) -> Result<(), mpsc::SendError> { + let message = AsyncWriteMessage::Data(Vec::from(data)); + self.sender.send(message) + } + + fn eof(&mut self) -> Result<(), mpsc::SendError> { + let message = AsyncWriteMessage::Eof; + self.sender.send(message) + } + + pub(crate) fn rust_error( + &mut self, + error: impl std::error::Error, + ) -> Result<(), impl std::error::Error> { + let message = AsyncWriteMessage::Error(error.to_string()); + self.sender.send(message) + } +} + +/// A wrapper of the `AsyncWrite` trait for the synchronous Nix C++ land. +pub struct AsyncWriteAdapter { + receiver: mpsc::UnboundedReceiver, + eof: bool, +} + +impl AsyncWriteAdapter { + pub fn new() -> (Self, Box) { + let (sender, receiver) = mpsc::unbounded_channel(); + + let r = Self { + receiver, + eof: false, + }; + let sender = Box::new(AsyncWriteSender { sender }); + + (r, sender) + } + + /// Write everything the sender sends to us. + pub async fn write_all(mut self, mut writer: Box) -> AtticResult<()> { + let writer = writer.as_mut(); + + while let Some(data) = self.next().await { + match data { + Ok(v) => { + writer.write_all(&v).await?; + } + Err(e) => { + return Err(e); + } + } + } + + if !self.eof { + Err(io::Error::from(io::ErrorKind::BrokenPipe).into()) + } else { + Ok(()) + } + } +} + +impl Stream for AsyncWriteAdapter { + type Item = AtticResult>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.receiver.poll_recv(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Some(message)) => { + use AsyncWriteMessage::*; + match message { + Data(v) => Poll::Ready(Some(Ok(v))), + Error(exception) => { + let error = AtticError::CxxError { exception }; + Poll::Ready(Some(Err(error))) + } + Eof => { + self.eof = true; + Poll::Ready(None) + } + } + } + Poll::Ready(None) => { + if !self.eof { + Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe).into()))) + } else { + Poll::Ready(None) + } + } + } + } +} + +#[cxx::bridge] +/// Generated by `cxx.rs`. +/// +/// Mid-level wrapper of `libnixstore` implemented in C++. +mod ffi { + extern "Rust" { + type AsyncWriteSender; + fn send(self: &mut AsyncWriteSender, data: &[u8]) -> Result<()>; + fn eof(self: &mut AsyncWriteSender) -> Result<()>; + } + + unsafe extern "C++" { + include!("attic/src/nix_store/bindings/nix.hpp"); + + // ========= + // CNixStore + // ========= + + /// Mid-level wrapper for the Unix Domain Socket Nix Store. + type CNixStore; + + /// Returns the path of the Nix store itself. + fn store_dir(self: Pin<&mut CNixStore>) -> String; + + /* + /// Verifies that a path is indeed in the Nix store, then return the base store path. + /// + /// Use parse_store_path instead. + fn to_store_path(self: Pin<&mut CNixStore>, path: &str) -> Result; + */ + + /// Queries information about a valid path. + fn query_path_info( + self: Pin<&mut CNixStore>, + store_path: &[u8], + ) -> Result>; + + /// Computes the closure of a valid path. + /// + /// If `flip_directions` is true, the set of paths that can reach `store_path` is + /// returned. + fn compute_fs_closure( + self: Pin<&mut CNixStore>, + store_path: &[u8], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result>>; + + /// Computes the closure of a list of valid paths. + /// + /// This is the multi-path variant of `compute_fs_closure`. + /// If `flip_directions` is true, the set of paths that can reach `store_path` is + /// returned. + /// + /// It's easier and more efficient to just pass a vector of slices + /// instead of wrangling with concrete "extern rust" / "extern C++" + /// types. + fn compute_fs_closure_multi( + self: Pin<&mut CNixStore>, + base_names: &[&[u8]], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result>>; + + /// Creates a NAR dump from a path. + fn nar_from_path( + self: Pin<&mut CNixStore>, + base_name: Vec, + sender: Box, + ) -> Result<()>; + + /// Obtains a handle to the Nix store. + fn open_nix_store() -> Result>; + + // ========= + // CPathInfo + // ========= + + /// Mid-level wrapper for the `nix::ValidPathInfo` struct. + type CPathInfo; + + /// Returns the SHA-256 hash of the store path. + fn nar_sha256_hash(self: Pin<&mut CPathInfo>) -> &[u8]; + + /// Returns the size of the NAR. + fn nar_size(self: Pin<&mut CPathInfo>) -> u64; + + /// Returns the references of the store path. + fn references(self: Pin<&mut CPathInfo>) -> UniquePtr>; + + /// Returns the possibly invalid signatures attached to the store path. + fn sigs(self: Pin<&mut CPathInfo>) -> UniquePtr>; + + /// Returns the CA field of the store path. + fn ca(self: Pin<&mut CPathInfo>) -> String; + } +} diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp new file mode 100644 index 0000000..167244c --- /dev/null +++ b/src/bindings/nix.cpp @@ -0,0 +1,173 @@ +/* +Copyright 2022 Zhaofeng Li and the Attic contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// C++ side of the libnixstore glue. +// +// We implement a mid-level wrapper of the Nix Store interface, +// which is then wrapped again in the Rust side to enable full +// async-await operation. +// +// Here we stick with the naming conventions of Rust and handle +// Rust types directly where possible, so that the interfaces are +// satisfying to use from the Rust side via cxx.rs. + +#include "attic/src/nix_store/bindings/nix.hpp" + +static std::mutex g_init_nix_mutex; +static bool g_init_nix_done = false; + +static nix::StorePath store_path_from_rust(RBasePathSlice base_name) { + std::string_view sv((const char *)base_name.data(), base_name.size()); + return nix::StorePath(sv); +} + +static bool hash_is_sha256(const nix::Hash &hash) { +#ifdef ATTIC_NIX_2_20 + return hash.algo == nix::HashAlgorithm::SHA256; +#else + return hash.type == nix::htSHA256; +#endif +} + +// ======== +// RustSink +// ======== + +RustSink::RustSink(RBox sender) : sender(std::move(sender)) {} + +void RustSink::operator () (std::string_view data) { + RBasePathSlice s((const unsigned char *)data.data(), data.size()); + + this->sender->send(s); +} + +void RustSink::eof() { + this->sender->eof(); +} + + +// ========= +// CPathInfo +// ========= + +CPathInfo::CPathInfo(nix::ref pi) : pi(pi) {} + +RHashSlice CPathInfo::nar_sha256_hash() { + auto &hash = this->pi->narHash; + + if (!hash_is_sha256(hash)) { + throw nix::Error("Only SHA-256 hashes are supported at the moment"); + } + + return RHashSlice(hash.hash, hash.hashSize); +} + +uint64_t CPathInfo::nar_size() { + return this->pi->narSize; +} + +std::unique_ptr> CPathInfo::sigs() { + std::vector result; + for (auto&& elem : this->pi->sigs) { + result.push_back(std::string(elem)); + } + return std::make_unique>(result); +} + +std::unique_ptr> CPathInfo::references() { + std::vector result; + for (auto&& elem : this->pi->references) { + result.push_back(std::string(elem.to_string())); + } + return std::make_unique>(result); +} + +RString CPathInfo::ca() { + if (this->pi->ca) { + return RString(nix::renderContentAddress(this->pi->ca)); + } else { + return RString(""); + } +} + +// ========= +// CNixStore +// ========= + +CNixStore::CNixStore() { + std::map params; + std::lock_guard lock(g_init_nix_mutex); + + if (!g_init_nix_done) { + nix::initNix(); + g_init_nix_done = true; + } + + this->store = nix::openStore(nix::settings.storeUri.get(), params); +} + +RString CNixStore::store_dir() { + return RString(this->store->storeDir); +} + +std::unique_ptr CNixStore::query_path_info(RBasePathSlice base_name) { + auto store_path = store_path_from_rust(base_name); + + auto r = this->store->queryPathInfo(store_path); + return std::make_unique(r); +} + +std::unique_ptr> CNixStore::compute_fs_closure(RBasePathSlice base_name, bool flip_direction, bool include_outputs, bool include_derivers) { + std::set out; + + this->store->computeFSClosure(store_path_from_rust(base_name), out, flip_direction, include_outputs, include_derivers); + + std::vector result; + for (auto&& elem : out) { + result.push_back(std::string(elem.to_string())); + } + return std::make_unique>(result); +} + +std::unique_ptr> CNixStore::compute_fs_closure_multi(RSlice base_names, bool flip_direction, bool include_outputs, bool include_derivers) { + std::set path_set, out; + for (auto&& base_name : base_names) { + path_set.insert(store_path_from_rust(base_name)); + } + + this->store->computeFSClosure(path_set, out, flip_direction, include_outputs, include_derivers); + + std::vector result; + for (auto&& elem : out) { + result.push_back(std::string(elem.to_string())); + } + return std::make_unique>(result); +} + +void CNixStore::nar_from_path(RVec base_name, RBox sender) { + RustSink sink(std::move(sender)); + + std::string_view sv((const char *)base_name.data(), base_name.size()); + nix::StorePath store_path(sv); + + // exceptions will be thrown into Rust + this->store->narFromPath(store_path, sink); + sink.eof(); +} + +std::unique_ptr open_nix_store() { + return std::make_unique(); +} diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp new file mode 100644 index 0000000..bbab4c5 --- /dev/null +++ b/src/bindings/nix.hpp @@ -0,0 +1,96 @@ +/* +Copyright 2022 Zhaofeng Li and the Attic contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// C++ side of the libnixstore glue. +// +// We implement a mid-level wrapper of the Nix Store interface, +// which is then wrapped again in the Rust side to enable full +// async-await operation. +// +// Here we stick with the naming conventions of Rust and handle +// Rust types directly where possible, so that the interfaces are +// satisfying to use from the Rust side via cxx.rs. + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template using RVec = rust::Vec; +template using RBox = rust::Box; +template using RSlice = rust::Slice; +using RString = rust::String; +using RStr = rust::Str; +using RBasePathSlice = RSlice; +using RHashSlice = RSlice; + +struct AsyncWriteSender; + +struct RustSink : nix::Sink +{ + RBox sender; +public: + RustSink(RBox sender); + void operator () (std::string_view data) override; + void eof(); +}; + +// Opaque wrapper for nix::ValidPathInfo +class CPathInfo { + nix::ref pi; +public: + CPathInfo(nix::ref pi); + RHashSlice nar_sha256_hash(); + uint64_t nar_size(); + std::unique_ptr> sigs(); + std::unique_ptr> references(); + RString ca(); +}; + +class CNixStore { + std::shared_ptr store; +public: + CNixStore(); + + RString store_dir(); + std::unique_ptr query_path_info(RBasePathSlice base_name); + std::unique_ptr> compute_fs_closure( + RBasePathSlice base_name, + bool flip_direction, + bool include_outputs, + bool include_derivers); + std::unique_ptr> compute_fs_closure_multi( + RSlice base_names, + bool flip_direction, + bool include_outputs, + bool include_derivers); + void nar_from_path(RVec base_name, RBox sender); +}; + +std::unique_ptr open_nix_store(); + +// Relies on our definitions +#include "attic/src/nix_store/bindings/mod.rs.h" From 8ac9253ea3dff87a2a1cbe28619c5536ca0d662e Mon Sep 17 00:00:00 2001 From: cy Date: Wed, 16 Apr 2025 03:46:56 -0400 Subject: [PATCH 02/32] change stuff to our way --- src/bindings/mod.rs | 17 ++++++++++------- src/bindings/nix.cpp | 8 ++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 521df2e..f636f4c 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -15,17 +15,17 @@ limitations under the License. */ //! `libnixstore` Bindings +#![allow(dead_code)] use std::cell::UnsafeCell; use std::io; use std::pin::Pin; use std::task::{Context, Poll}; +use anyhow::Result; use futures::stream::{Stream, StreamExt}; use tokio::io::{AsyncWrite, AsyncWriteExt}; -use crate::{AtticError, AtticResult}; - // The C++ implementation takes care of concurrency #[repr(transparent)] pub struct FfiNixStore(UnsafeCell>); @@ -43,7 +43,7 @@ impl FfiNixStore { } /// Obtain a handle to the Nix store. -pub unsafe fn open_nix_store() -> AtticResult { +pub unsafe fn open_nix_store() -> Result { match ffi::open_nix_store() { Ok(ptr) => { let cell = UnsafeCell::new(ptr); @@ -116,7 +116,7 @@ impl AsyncWriteAdapter { } /// Write everything the sender sends to us. - pub async fn write_all(mut self, mut writer: Box) -> AtticResult<()> { + pub async fn write_all(mut self, mut writer: Box) -> Result<()> { let writer = writer.as_mut(); while let Some(data) = self.next().await { @@ -139,7 +139,7 @@ impl AsyncWriteAdapter { } impl Stream for AsyncWriteAdapter { - type Item = AtticResult>; + type Item = Result>; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.receiver.poll_recv(cx) { @@ -149,7 +149,7 @@ impl Stream for AsyncWriteAdapter { match message { Data(v) => Poll::Ready(Some(Ok(v))), Error(exception) => { - let error = AtticError::CxxError { exception }; + let error = anyhow::Error::msg(format!("cxx error: {exception}")); Poll::Ready(Some(Err(error))) } Eof => { @@ -181,7 +181,7 @@ mod ffi { } unsafe extern "C++" { - include!("attic/src/nix_store/bindings/nix.hpp"); + include!("nix.hpp"); // ========= // CNixStore @@ -266,5 +266,8 @@ mod ffi { /// Returns the CA field of the store path. fn ca(self: Pin<&mut CPathInfo>) -> String; + + /// Returns the derivation that built this path + fn deriver(self: Pin<&mut CPathInfo>) -> String; } } diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp index 167244c..7783b4b 100644 --- a/src/bindings/nix.cpp +++ b/src/bindings/nix.cpp @@ -103,6 +103,14 @@ RString CPathInfo::ca() { } } +RString CPathInfo::deriver() { + if (this->pi->deriver) { + return RString((this->pi->deriver).to_string()); + } else { + return RString(""); + } +} + // ========= // CNixStore // ========= From a771785352a72ae6e3a18bbe4b5f46e1bda7196f Mon Sep 17 00:00:00 2001 From: cy Date: Wed, 16 Apr 2025 03:47:42 -0400 Subject: [PATCH 03/32] use libstore cxx bindings --- src/main.rs | 20 ++++-- src/path_info.rs | 167 ++++++++--------------------------------------- src/push.rs | 46 +++++++++---- src/store.rs | 80 +++++++++++++++++++++++ 4 files changed, 152 insertions(+), 161 deletions(-) create mode 100644 src/store.rs diff --git a/src/main.rs b/src/main.rs index 57d3340..5a7e91f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,20 @@ #![feature(let_chains)] #![feature(extend_one)] +use std::path::PathBuf; + use anyhow::{Context, Result}; use clap::{Args, Parser, Subcommand}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; use push::Push; +use store::Store; +mod bindings; mod cli; mod path_info; mod push; +mod store; mod uploader; #[derive(Parser, Debug)] @@ -60,10 +65,10 @@ pub struct PushArgs { #[arg(long)] skip_signature_check: bool, - /// Package or store path to upload - /// e.g. nixpkgs#hello or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 - #[arg(value_name = "package or store path")] - package: String, + /// Path to upload + /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 + #[arg(value_name = "PATH")] + paths: Vec, } #[tokio::main] @@ -76,10 +81,11 @@ async fn main() -> Result<()> { match &cli.command { Commands::Push(cli) => { - let push = Box::leak(Box::new(Push::new(cli).await?)); - push.paths_from_package(&cli.package) + let store = Store::connect()?; + let push = Box::leak(Box::new(Push::new(cli, store).await?)); + push.add_paths(cli.paths.clone()) .await - .context("nixcp get paths from package")?; + .context("add paths to push")?; push.run().await.context("nixcp run")?; } } diff --git a/src/path_info.rs b/src/path_info.rs index 6dcbb53..3e69ad9 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,85 +1,40 @@ use std::collections::HashSet; -use anyhow::{Context, Error, Result}; +use anyhow::{Context, Result}; use aws_sdk_s3 as s3; +use futures::future::join_all; use nix_compat::nixbase32; use nix_compat::store_path::StorePath; use regex::Regex; -use serde::Deserialize; -use tokio::process::Command; -use tracing::{debug, error, trace}; +use std::path::Path; +use tracing::{debug, trace}; use url::Url; -// nix path-info --derivation --json -#[derive(Debug, Clone, Deserialize)] +use crate::store::Store; + +#[derive(Debug, Clone)] pub struct PathInfo { pub deriver: Option>, pub path: StorePath, - signatures: Option>, + pub signatures: Vec, pub references: Vec>, } impl PathInfo { - // get PathInfo for a package or a store path - // we deserialize this as an array of `PathInfo` below - pub async fn from_path(path: &str) -> Result { - debug!("query nix path-info for {path}"); - // use lix cause nix would return a json map instead of an array - // json output is not stable and could break in future - // TODO figure out a better way - let nix_cmd = Command::new("nix") - .arg("run") - .arg("--experimental-features") - .arg("nix-command flakes") - .arg("github:nixos/nixpkgs/nixos-unstable#lix") - .arg("--") - .arg("path-info") - .arg("--json") - .arg(path) - .output() - .await - .context("`nix path-info` failed for {package}")?; - - trace!( - "nix path-info output: {}", - String::from_utf8_lossy(&nix_cmd.stdout) - ); - - // nix path-info returns an array with one element - match serde_json::from_slice::>(&nix_cmd.stdout) - .context("parse path info from stdout") - { - Ok(path_info) => path_info - .into_iter() - .next() - .ok_or_else(|| Error::msg("nix path-info returned empty")), - Err(e) => { - error!( - "Failed to parse data from `nix path-info`. The path may not exist on your system." - ); - Err(e) - } - } + pub async fn from_path(path: &Path, store: &Store) -> Result { + debug!("query path info for {:?}", path); + let canon = path.canonicalize().context("canonicalize path")?; + let store_path = StorePath::from_absolute_path(canon.into_os_string().as_encoded_bytes())?; + store.query_path_info(store_path).await } - pub async fn get_closure(&self) -> Result> { - debug!("query nix-store for {}", self.absolute_path()); - let nix_store_cmd = Command::new("nix-store") - .arg("--query") - .arg("--requisites") - .arg("--include-outputs") - .arg(self.absolute_path()) - .output() - .await - .expect("nix-store cmd failed"); - - let nix_store_paths = String::from_utf8(nix_store_cmd.stdout)?; - let nix_store_paths: Vec<&str> = nix_store_paths.lines().collect(); - let mut closure = Vec::with_capacity(nix_store_paths.len()); - for path in nix_store_paths { - closure.push(Self::from_path(path).await?); - } - Ok(closure) + pub async fn get_closure(&self, store: &Store) -> Result> { + let futs = store + .compute_fs_closure(self.path.clone()) + .await? + .into_iter() + .map(|x| store.query_path_info(x)); + join_all(futs).await.into_iter().collect() } /// checks if the path is signed by any upstream. if it is, we assume a cache hit. @@ -101,15 +56,13 @@ impl PathInfo { } fn signees(&self) -> Vec<&str> { - if let Some(signatures) = self.signatures.as_ref() { - let signees: Vec<_> = signatures - .iter() - .filter_map(|signature| Some(signature.split_once(":")?.0)) - .collect(); - trace!("signees for {}: {:?}", self.path, signees); - return signees; - } - Vec::new() + let signers: Vec<_> = self + .signatures + .iter() + .filter_map(|signature| Some(signature.split_once(":")?.0)) + .collect(); + trace!("signers for {}: {:?}", self.path, signers); + return signers; } pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { @@ -149,69 +102,3 @@ impl PathInfo { .is_ok() } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn get_signees_from_path_info() { - let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#; - let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); - - path_info.signatures = Some(vec![ - "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), - "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), - ]); - let signees = path_info.signees(); - assert_eq!(signees, vec!["cache.nixos.org-1", "nixcache.cy7.sh"]); - } - - #[test] - fn match_upstream_cache_from_signature() { - let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#; - let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); - - path_info.signatures = Some(vec![ - "cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(), - "nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(), - "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=".to_string(), - ]); - assert!( - path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]) - ); - assert!( - path_info.check_upstream_signature(&[Url::parse("https://nixcache.cy7.sh").unwrap()]) - ); - assert!( - path_info.check_upstream_signature(&[ - Url::parse("https://nix-community.cachix.org").unwrap() - ]) - ); - assert!( - !path_info - .check_upstream_signature(&[Url::parse("https://fake-cache.cachix.org").unwrap()]), - ); - } - - #[test] - fn path_info_without_signature() { - let path_info_json = r#"{"ca":"fixed:r:sha256:1q10p04pgx9sk6xbvrkn4nvh0ys2lzplgcni5368f4z3cr8ikbmz","narHash":"sha256-v64ZUWbjE4fMKNGyR++nQnsAtyV25r26mTr1dwm4IOA=","narSize":5520,"path":"/nix/store/gj6hz9mj23v01yvq1nn5f655jrcky1qq-nixos-option.nix","references":[],"registrationTime":1744740942,"valid":true}"#; - let path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize"); - - assert!( - !path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]) - ); - } - - /* - #[test] - fn path_info_deserialize_nix_map() { - let path_info_json = r#"{"/nix/store/8vm1jxsc0jphd65vb7r6g5ysgqw0yh9f-home-manager-generation":{"ca":null,"deriver":"/nix/store/h8z25s6arcrns5nmrq1yhgbamywjivpn-home-manager-generation.drv","narHash":"sha256-o4qwqyJ5UVm9cyC/nBNcNYVnIM14Pewgw7fou+wUVSY=","narSize":13608,"references":["/nix/store/40yifhx34v4g4llrdn3v2ag8w02j10fv-gnugrep-3.11","/nix/store/4d0ix5djms3n2jnjdc58l916cwack1rp-empty-directory","/nix/store/56zmgla8443qfrkrh2ch0vz0mh8ywrw1-home-manager-files","/nix/store/58br4vk3q5akf4g8lx0pqzfhn47k3j8d-bash-5.2p37","/nix/store/80l1sb3vcmrkcdd7ihlizkcnv19rq9fj-ncurses-6.5","/nix/store/8vm1jxsc0jphd65vb7r6g5ysgqw0yh9f-home-manager-generation","/nix/store/92as847i10kl6s19fi910ddyk9l83835-check-link-targets.sh","/nix/store/9c90iz95yynyh3vsc67zndch6j01vgz3-home-manager-path","/nix/store/b2cfj7yk3wfg1jdwjzim7306hvsc5gnl-systemd-257.3","/nix/store/bm5fi6wj0w4r2wjll2448k307bzfcjwx-cleanup","/nix/store/c244fsb3a7i5837lzn94m4bmav9i5p9b-link","/nix/store/cvlbhhrvzfkjl2hrrzhq3vr5gzan1r60-bash-interactive-5.2p37","/nix/store/gpxsdrrd4x93fs75395vr2dfys1ki9mq-jq-1.7.1-bin","/nix/store/jlf743lqxbvad6dbgndsgqfg20m2np5i-sd-switch-0.5.3","/nix/store/mhmgm739aagj4x7hr6ag2wjmxhmpy8mf-gettext-0.22.5","/nix/store/w9db12j05yv5hl31s6jndd9cfm1g1gw4-hm-modules-messages","/nix/store/wj1c3gsiajabnq50ifxqnlv60i5rhqj7-diffutils-3.10","/nix/store/xhql0ilzbiqwnmz4z8y0phk611wynxf2-gnused-4.9","/nix/store/xq5f95pp297afc2xjgrmhmf9w631qp7m-findutils-4.10.0","/nix/store/yh6qg1nsi5h2xblcr67030pz58fsaxx3-coreutils-9.6","/nix/store/zhrjg6wxrxmdlpn6iapzpp2z2vylpvw5-home-manager.sh"],"registrationTime":1744742989,"signatures":["nixcache.cy7.sh:Vq4X95kSzum7BwrBhjmmM2yVipfBI3AE3jgZ3b3RoYrP4/ghotbDdlwCvwK3qx4BQdEOLSgrC1tDwiMNb6oRBw=="],"ultimate":false}}"#; - serde_json::from_str::>(path_info_json).expect("must serialize"); - - let path_info_json = r#"{"/nix/store/3a2ahdaprw6df0lml1pj9jhbi038dsjh-nixos-system-chunk-25.05.20250412.2631b0b":{"ca":null,"deriver":"/nix/store/12ssi931481jlkizgfk1c1jnawvwjbhh-nixos-system-chunk-25.05.20250412.2631b0b.drv","narHash":"sha256-CHhBIzMD4v/FKqKgGroq0UC1k3GrK5lcNwQPMpv2xLc=","narSize":20704,"references":["/nix/store/0yjiyixxsr137iw93hnaacdsssy1li9h-switch-to-configuration-0.1.0","/nix/store/14rby7cpwrzjsjym44cl5h6nj6qpn1gs-etc","/nix/store/3a2ahdaprw6df0lml1pj9jhbi038dsjh-nixos-system-chunk-25.05.20250412.2631b0b","/nix/store/3wjljpj30fvv2cdb60apr4126pa5bm87-shadow-4.17.2","/nix/store/40yifhx34v4g4llrdn3v2ag8w02j10fv-gnugrep-3.11","/nix/store/58br4vk3q5akf4g8lx0pqzfhn47k3j8d-bash-5.2p37","/nix/store/5dyh8l59kfvf89zjkbmjfnx7fix93n4f-net-tools-2.10","/nix/store/aq9wdsz12bg9252790l9awiry2bml4ls-sops-install-secrets-0.0.1","/nix/store/b00kq6fjhgisdrykg621vml8505nnmb3-users-groups.json","/nix/store/b2cfj7yk3wfg1jdwjzim7306hvsc5gnl-systemd-257.3","/nix/store/bfr68wi6k8icb3j9fy3fzchva56djfhd-mounts.sh","/nix/store/cjnihsds5hhnji9r85hglph07q9y9hgc-system-path","/nix/store/cvlbhhrvzfkjl2hrrzhq3vr5gzan1r60-bash-interactive-5.2p37","/nix/store/f9jll96j74f5ykvs062718b98lfjbn9g-util-linux-2.40.4-bin","/nix/store/h7zih134d3n5yk8pnhv1fa38n6qkyrn2-pre-switch-checks","/nix/store/idn5n51246piyxcr3v6gxnj5a5l9mzpn-linux-6.14.2","/nix/store/ipn5793y61x2904xqnkgbjnp91svjjzx-perl-5.40.0-env","/nix/store/j1rikvl25pz0b5ham1ijq0nbg1q2fqfy-initrd-linux-6.14.2","/nix/store/jgawnqyh6piwcl79gxpmq5czx9rfr9xh-glibc-locales-2.40-66","/nix/store/jqgmcv8j4gj59218hcbiyn8z951rycdj-install-grub.sh","/nix/store/kpmybhxy3gz6k1znbdirwsp3c6wvsgg9-manifest.json","/nix/store/lgainx4gl6q7mhiwmls81d3n51p5jz7z-linux-6.14.2-modules","/nix/store/mhxn5kwnri3z9hdzi3x0980id65p0icn-lib.sh","/nix/store/n8n0faszqlnf3mdg0fj6abnknrhjsw5j-perl-5.40.0-env","/nix/store/nq61v7a601gjndijq5nndprkzpwz4q9g-glibc-2.40-66-bin","/nix/store/nx27idxpvi3fk3p7admvhipny73nr25n-kmod-31","/nix/store/pggww1d2pg24fcg5v36xn63n53vanyyi-gnupg-2.4.7","/nix/store/rg5rf512szdxmnj9qal3wfdnpfsx38qi-setup-etc.pl","/nix/store/vvlfaafnz3pdhw7lx5kc5gb9pl4zhz5l-local-cmds","/nix/store/w142vx7ij1fz6qwhp5dprkf59cizvv1v-update-users-groups.pl","/nix/store/xq5f95pp297afc2xjgrmhmf9w631qp7m-findutils-4.10.0","/nix/store/yh6qg1nsi5h2xblcr67030pz58fsaxx3-coreutils-9.6","/nix/store/zlsmh0ccgvncg30qb4y0mp5pahnk1wnw-append-initrd-secrets","/nix/store/zs07icpv5ykf8m36xcv717hh26bp09fa-firmware","/nix/store/zy2n4id5gcxcbx2x8jbblkmcpdlpsypk-getent-glibc-2.40-66"],"registrationTime":1744743136,"signatures":["nixcache.cy7.sh:dZ1XiKQNe0fRX48gBj03PIABYJGV6BPwb72YpMqEBONZMF+JrkVKhRCF0ur/4Bf5prHxg6Qfg1ytP/4csRC9DQ=="],"ultimate":false}}"#; - serde_json::from_str::>(path_info_json).expect("must serialize"); - } - */ -} diff --git a/src/push.rs b/src/push.rs index 719d3a8..df8304d 100644 --- a/src/push.rs +++ b/src/push.rs @@ -1,6 +1,7 @@ use std::{ fs, iter::once, + path::PathBuf, sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -13,10 +14,10 @@ use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::narinfo::{self, SigningKey}; use tokio::sync::{RwLock, mpsc}; -use tracing::{debug, info, trace}; +use tracing::{debug, trace}; use url::Url; -use crate::{PushArgs, path_info::PathInfo, uploader::Uploader}; +use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; pub struct Push { upstream_caches: Vec, @@ -24,6 +25,7 @@ pub struct Push { s3_client: s3::Client, signing_key: SigningKey, bucket: String, + store: Arc, // paths that we skipped cause of a signature match signature_hit_count: AtomicUsize, // paths that we skipped cause we found it on an upstream @@ -35,7 +37,7 @@ pub struct Push { } impl Push { - pub async fn new(cli: &PushArgs) -> Result { + pub async fn new(cli: &PushArgs, store: Store) -> Result { let mut upstreams = Vec::with_capacity(cli.upstreams.len() + 1); for upstream in cli .upstreams @@ -67,6 +69,7 @@ impl Push { s3_client, signing_key, bucket: cli.bucket.clone(), + store: Arc::new(store), signature_hit_count: AtomicUsize::new(0), upstream_hit_count: AtomicUsize::new(0), already_exists_count: AtomicUsize::new(0), @@ -74,18 +77,33 @@ impl Push { }) } - pub async fn paths_from_package(&mut self, package: &str) -> Result<()> { - let path_info = PathInfo::from_path(package) + pub async fn add_paths(&'static self, paths: Vec) -> Result<()> { + let mut futs = Vec::with_capacity(paths.len()); + for path in paths { + let store_paths = self.store_paths.clone(); + let store = self.store.clone(); + + futs.push(tokio::spawn(async move { + let path_info = PathInfo::from_path(path.as_path(), &store) + .await + .context("get path info for path")?; + debug!("path-info for {path:?}: {path_info:?}"); + + store_paths.write().await.extend( + path_info + .get_closure(&store) + .await + .context("closure from path info")?, + ); + Ok(()) + })); + } + join_all(futs) .await - .context("get path info for package")?; - debug!("path-info for {package}: {:?}", path_info); - self.store_paths.write().await.extend( - path_info - .get_closure() - .await - .context("closure from path info")?, - ); - info!("found {} store paths", self.store_paths.read().await.len()); + .into_iter() + .flatten() + .collect::>>()?; + println!("found {} store paths", self.store_paths.read().await.len()); Ok(()) } diff --git a/src/store.rs b/src/store.rs new file mode 100644 index 0000000..e988535 --- /dev/null +++ b/src/store.rs @@ -0,0 +1,80 @@ +use std::{ffi::OsStr, os::unix::ffi::OsStrExt, sync::Arc}; + +use anyhow::{Context, Result}; +use nix_compat::store_path::StorePath; +use tokio::task; + +use crate::{bindings, path_info::PathInfo}; + +pub struct Store { + inner: Arc, +} + +impl Store { + pub fn connect() -> Result { + let inner = unsafe { bindings::open_nix_store()? }; + Ok(Self { + inner: Arc::new(inner), + }) + } + + pub async fn compute_fs_closure( + &self, + path: StorePath, + ) -> Result>> { + let inner = self.inner.clone(); + task::spawn_blocking(move || { + let cxx_vector = + inner + .store() + .compute_fs_closure(path.to_string().as_bytes(), false, true, true)?; + Ok(cxx_vector + .iter() + .map(|x| { + StorePath::from_bytes(x.as_bytes()) + .context("make StorePath from vector returned by compute_fs_closure") + }) + .collect::>()?) + }) + .await + .unwrap() + } + + pub async fn query_path_info(&self, path: StorePath) -> Result { + let inner = self.inner.clone(); + + task::spawn_blocking(move || { + let mut c_path_info = inner.store().query_path_info(path.to_string().as_bytes())?; + + let deriver = c_path_info.pin_mut().deriver(); + let signatures = c_path_info + .pin_mut() + .sigs() + .into_iter() + .map(|x| { + let osstr = OsStr::from_bytes(x.as_bytes()); + osstr.to_str().unwrap().to_string() + }) + .collect(); + let references = c_path_info + .pin_mut() + .references() + .into_iter() + .map(|x| StorePath::from_bytes(x.as_bytes())) + .collect::>()?; + + Ok(PathInfo { + path, + deriver: if deriver.is_empty() { + None + } else { + Some(StorePath::from_bytes(deriver.as_bytes())?) + }, + signatures, + references, + }) + }) + .await + .unwrap() + } +} From 84bbe5dcb49ae8e39cebed5386006d749823ea94 Mon Sep 17 00:00:00 2001 From: cy Date: Wed, 16 Apr 2025 12:42:44 -0400 Subject: [PATCH 04/32] fix build --- Cargo.lock | 112 +++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 5 ++ build.rs | 21 ++++++++ flake.nix | 6 ++- src/bindings/mod.rs | 48 +------------------ src/bindings/nix.cpp | 70 +-------------------------- src/bindings/nix.hpp | 11 +---- src/path_info.rs | 1 - src/store.rs | 6 --- src/uploader.rs | 2 +- 10 files changed, 147 insertions(+), 135 deletions(-) create mode 100644 build.rs diff --git a/Cargo.lock b/Cargo.lock index aee2282..4d5600c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -759,6 +759,17 @@ dependencies = [ "cc", ] +[[package]] +name = "codespan-reporting" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -907,6 +918,65 @@ dependencies = [ "syn", ] +[[package]] +name = "cxx" +version = "1.0.157" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6354e975ea4ec28033ec3a36fa9baa1a02e3eb22ad740eeb4929370d4f5ba8" +dependencies = [ + "cc", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.157" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4400e26ea4b99417e4263b1ce2d8452404d750ba0809a7bd043072593d430d" +dependencies = [ + "cc", + "codespan-reporting", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.157" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31860c98f69fc14da5742c5deaf78983e846c7b27804ca8c8319e32eef421bde" +dependencies = [ + "clap", + "codespan-reporting", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.157" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0402a66013f3b8d3d9f2d7c9994656cc81e671054822b0728d7454d9231892f" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.157" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c0b38f32d68f3324a981645ee39b2d686af36d03c98a386df3716108c9feae" +dependencies = [ + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -1772,6 +1842,15 @@ dependencies = [ "libc", ] +[[package]] +name = "link-cplusplus" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6f6da007f968f9def0d65a05b187e2960183de70c160204ecfccf0ee330212" +dependencies = [ + "cc", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1945,9 +2024,12 @@ dependencies = [ "aws-config", "aws-sdk-s3", "clap", + "cxx", + "cxx-build", "ed25519-dalek", "futures", "nix-compat", + "pkg-config", "regex", "reqwest", "serde", @@ -2538,6 +2620,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52" + [[package]] name = "sct" version = "0.7.1" @@ -2841,6 +2929,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "2.0.12" @@ -3106,6 +3203,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "untrusted" version = "0.9.0" @@ -3311,6 +3414,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index f7bc3f0..8adf37e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,8 @@ tokio = { version = "1.44.1", features = [ "full" ]} tracing = "0.1.41" tracing-subscriber = { version = "0.3.19", features = ["env-filter"]} url = { version = "2.5.4", features = [ "serde" ]} +cxx = "1.0" + +[build-dependencies] +cxx-build = "1.0" +pkg-config = "0.3.32" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..2bbe451 --- /dev/null +++ b/build.rs @@ -0,0 +1,21 @@ +fn main() { + cxx_build::bridge("src/bindings/mod.rs") + .file("src/bindings/nix.cpp") + .flag("-std=c++2a") + .flag("-O2") + .flag("-include") + .flag("nix/config.h") + .flag("-I") + .flag(concat!(env!("NIX_INCLUDE_PATH"), "/nix")) + .compile("nixbinding"); + println!("cargo:rerun-if-changed=src/bindings"); + + pkg_config::Config::new() + .atleast_version("2.4") + .probe("nix-store") + .unwrap(); + pkg_config::Config::new() + .atleast_version("2.4") + .probe("nix-main") + .unwrap(); +} diff --git a/flake.nix b/flake.nix index c263f49..e9165c2 100644 --- a/flake.nix +++ b/flake.nix @@ -22,6 +22,7 @@ }; toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); + lib = pkgs.lib; in { devShells.default = pkgs.mkShell { @@ -29,9 +30,12 @@ pkg-config ]; buildInputs = with pkgs; [ - openssl toolchain + openssl + nix + boost ]; + NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; }; packages.default = craneLib.buildPackage { diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index f636f4c..61a32af 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -181,7 +181,7 @@ mod ffi { } unsafe extern "C++" { - include!("nix.hpp"); + include!("nixcp/src/bindings/nix.hpp"); // ========= // CNixStore @@ -190,16 +190,6 @@ mod ffi { /// Mid-level wrapper for the Unix Domain Socket Nix Store. type CNixStore; - /// Returns the path of the Nix store itself. - fn store_dir(self: Pin<&mut CNixStore>) -> String; - - /* - /// Verifies that a path is indeed in the Nix store, then return the base store path. - /// - /// Use parse_store_path instead. - fn to_store_path(self: Pin<&mut CNixStore>, path: &str) -> Result; - */ - /// Queries information about a valid path. fn query_path_info( self: Pin<&mut CNixStore>, @@ -218,30 +208,6 @@ mod ffi { include_derivers: bool, ) -> Result>>; - /// Computes the closure of a list of valid paths. - /// - /// This is the multi-path variant of `compute_fs_closure`. - /// If `flip_directions` is true, the set of paths that can reach `store_path` is - /// returned. - /// - /// It's easier and more efficient to just pass a vector of slices - /// instead of wrangling with concrete "extern rust" / "extern C++" - /// types. - fn compute_fs_closure_multi( - self: Pin<&mut CNixStore>, - base_names: &[&[u8]], - flip_direction: bool, - include_outputs: bool, - include_derivers: bool, - ) -> Result>>; - - /// Creates a NAR dump from a path. - fn nar_from_path( - self: Pin<&mut CNixStore>, - base_name: Vec, - sender: Box, - ) -> Result<()>; - /// Obtains a handle to the Nix store. fn open_nix_store() -> Result>; @@ -252,22 +218,10 @@ mod ffi { /// Mid-level wrapper for the `nix::ValidPathInfo` struct. type CPathInfo; - /// Returns the SHA-256 hash of the store path. - fn nar_sha256_hash(self: Pin<&mut CPathInfo>) -> &[u8]; - - /// Returns the size of the NAR. - fn nar_size(self: Pin<&mut CPathInfo>) -> u64; - /// Returns the references of the store path. fn references(self: Pin<&mut CPathInfo>) -> UniquePtr>; /// Returns the possibly invalid signatures attached to the store path. fn sigs(self: Pin<&mut CPathInfo>) -> UniquePtr>; - - /// Returns the CA field of the store path. - fn ca(self: Pin<&mut CPathInfo>) -> String; - - /// Returns the derivation that built this path - fn deriver(self: Pin<&mut CPathInfo>) -> String; } } diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp index 7783b4b..3914de1 100644 --- a/src/bindings/nix.cpp +++ b/src/bindings/nix.cpp @@ -24,7 +24,7 @@ limitations under the License. // Rust types directly where possible, so that the interfaces are // satisfying to use from the Rust side via cxx.rs. -#include "attic/src/nix_store/bindings/nix.hpp" +#include "nixcp/src/bindings/nix.hpp" static std::mutex g_init_nix_mutex; static bool g_init_nix_done = false; @@ -34,14 +34,6 @@ static nix::StorePath store_path_from_rust(RBasePathSlice base_name) { return nix::StorePath(sv); } -static bool hash_is_sha256(const nix::Hash &hash) { -#ifdef ATTIC_NIX_2_20 - return hash.algo == nix::HashAlgorithm::SHA256; -#else - return hash.type == nix::htSHA256; -#endif -} - // ======== // RustSink // ======== @@ -65,20 +57,6 @@ void RustSink::eof() { CPathInfo::CPathInfo(nix::ref pi) : pi(pi) {} -RHashSlice CPathInfo::nar_sha256_hash() { - auto &hash = this->pi->narHash; - - if (!hash_is_sha256(hash)) { - throw nix::Error("Only SHA-256 hashes are supported at the moment"); - } - - return RHashSlice(hash.hash, hash.hashSize); -} - -uint64_t CPathInfo::nar_size() { - return this->pi->narSize; -} - std::unique_ptr> CPathInfo::sigs() { std::vector result; for (auto&& elem : this->pi->sigs) { @@ -95,22 +73,6 @@ std::unique_ptr> CPathInfo::references() { return std::make_unique>(result); } -RString CPathInfo::ca() { - if (this->pi->ca) { - return RString(nix::renderContentAddress(this->pi->ca)); - } else { - return RString(""); - } -} - -RString CPathInfo::deriver() { - if (this->pi->deriver) { - return RString((this->pi->deriver).to_string()); - } else { - return RString(""); - } -} - // ========= // CNixStore // ========= @@ -127,10 +89,6 @@ CNixStore::CNixStore() { this->store = nix::openStore(nix::settings.storeUri.get(), params); } -RString CNixStore::store_dir() { - return RString(this->store->storeDir); -} - std::unique_ptr CNixStore::query_path_info(RBasePathSlice base_name) { auto store_path = store_path_from_rust(base_name); @@ -150,32 +108,6 @@ std::unique_ptr> CNixStore::compute_fs_closure(RBasePat return std::make_unique>(result); } -std::unique_ptr> CNixStore::compute_fs_closure_multi(RSlice base_names, bool flip_direction, bool include_outputs, bool include_derivers) { - std::set path_set, out; - for (auto&& base_name : base_names) { - path_set.insert(store_path_from_rust(base_name)); - } - - this->store->computeFSClosure(path_set, out, flip_direction, include_outputs, include_derivers); - - std::vector result; - for (auto&& elem : out) { - result.push_back(std::string(elem.to_string())); - } - return std::make_unique>(result); -} - -void CNixStore::nar_from_path(RVec base_name, RBox sender) { - RustSink sink(std::move(sender)); - - std::string_view sv((const char *)base_name.data(), base_name.size()); - nix::StorePath store_path(sv); - - // exceptions will be thrown into Rust - this->store->narFromPath(store_path, sink); - sink.eof(); -} - std::unique_ptr open_nix_store() { return std::make_unique(); } diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp index bbab4c5..9f4964a 100644 --- a/src/bindings/nix.hpp +++ b/src/bindings/nix.hpp @@ -63,11 +63,8 @@ class CPathInfo { nix::ref pi; public: CPathInfo(nix::ref pi); - RHashSlice nar_sha256_hash(); - uint64_t nar_size(); std::unique_ptr> sigs(); std::unique_ptr> references(); - RString ca(); }; class CNixStore { @@ -82,15 +79,9 @@ public: bool flip_direction, bool include_outputs, bool include_derivers); - std::unique_ptr> compute_fs_closure_multi( - RSlice base_names, - bool flip_direction, - bool include_outputs, - bool include_derivers); - void nar_from_path(RVec base_name, RBox sender); }; std::unique_ptr open_nix_store(); // Relies on our definitions -#include "attic/src/nix_store/bindings/mod.rs.h" +#include "nixcp/src/bindings/mod.rs.h" diff --git a/src/path_info.rs b/src/path_info.rs index 3e69ad9..3d54b17 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -14,7 +14,6 @@ use crate::store::Store; #[derive(Debug, Clone)] pub struct PathInfo { - pub deriver: Option>, pub path: StorePath, pub signatures: Vec, pub references: Vec>, diff --git a/src/store.rs b/src/store.rs index e988535..54f185c 100644 --- a/src/store.rs +++ b/src/store.rs @@ -46,7 +46,6 @@ impl Store { task::spawn_blocking(move || { let mut c_path_info = inner.store().query_path_info(path.to_string().as_bytes())?; - let deriver = c_path_info.pin_mut().deriver(); let signatures = c_path_info .pin_mut() .sigs() @@ -65,11 +64,6 @@ impl Store { Ok(PathInfo { path, - deriver: if deriver.is_empty() { - None - } else { - Some(StorePath::from_bytes(deriver.as_bytes())?) - }, signatures, references, }) diff --git a/src/uploader.rs b/src/uploader.rs index b0520ac..95e03df 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -159,7 +159,7 @@ impl<'a> Uploader<'a> { signatures: Vec::new(), ca: None, system: None, - deriver: self.path.deriver.as_ref().map(|x| x.as_ref()), + deriver: None, compression: Some("zstd"), file_hash: None, file_size: None, From 6806b968924089ee2a8b4c62d4aed15da3f10d91 Mon Sep 17 00:00:00 2001 From: cy Date: Wed, 16 Apr 2025 15:49:01 -0400 Subject: [PATCH 05/32] limit uploads with semaphore --- src/push.rs | 16 +++++++++++----- src/uploader.rs | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/push.rs b/src/push.rs index df8304d..dca2a9f 100644 --- a/src/push.rs +++ b/src/push.rs @@ -13,7 +13,7 @@ use aws_config::Region; use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::narinfo::{self, SigningKey}; -use tokio::sync::{RwLock, mpsc}; +use tokio::sync::{RwLock, Semaphore, mpsc}; use tracing::{debug, trace}; use url::Url; @@ -161,6 +161,7 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::with_capacity(10); + let permits = Arc::new(Semaphore::new(10)); loop { if let Some(path_to_upload) = rx.recv().await { @@ -174,10 +175,15 @@ impl Push { self.bucket.clone(), )?; - uploads.push(tokio::spawn(async move { - let res = uploader.upload().await; - self.upload_count.fetch_add(1, Ordering::Relaxed); - res + uploads.push(tokio::spawn({ + let permits = permits.clone(); + + async move { + let _permit = permits.acquire().await; + let res = uploader.upload().await; + self.upload_count.fetch_add(1, Ordering::Relaxed); + res + } })); } else { join_all(uploads) diff --git a/src/uploader.rs b/src/uploader.rs index 95e03df..cacae2b 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -132,6 +132,7 @@ impl<'a> Uploader<'a> { .body(nar_info.to_string().as_bytes().to_vec().into()) .send() .await?; + debug!("done uploading narinfo"); Ok(()) } From b1e59d0a6c302805475d2f4c5644bdf8a24c6334 Mon Sep 17 00:00:00 2001 From: cy Date: Fri, 18 Apr 2025 00:50:11 -0400 Subject: [PATCH 06/32] use nix path-info cmd for derivation; console_subscriber --- Cargo.lock | 437 ++++++++++++++++++++++++++++++++++++++++------- Cargo.toml | 4 +- flake.nix | 3 + src/main.rs | 5 +- src/path_info.rs | 27 ++- src/push.rs | 17 +- src/store.rs | 8 +- src/uploader.rs | 2 + 8 files changed, 421 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d5600c..9fa7a11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,39 @@ dependencies = [ "zstd-safe", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -429,7 +462,7 @@ dependencies = [ "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tower", + "tower 0.5.2", "tracing", ] @@ -553,6 +586,53 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -657,6 +737,12 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.10.1" @@ -776,6 +862,45 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "console-api" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -859,6 +984,21 @@ dependencies = [ "crc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-bigint" version = "0.4.9" @@ -1160,6 +1300,16 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "flate2" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1359,7 +1509,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", @@ -1378,13 +1528,19 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.2" @@ -1396,6 +1552,19 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom 7.1.3", + "num-traits", +] + [[package]] name = "heck" version = "0.5.0" @@ -1494,6 +1663,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" + [[package]] name = "hyper" version = "0.14.32" @@ -1531,6 +1706,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1572,6 +1748,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.6.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -1747,6 +1936,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.9.0" @@ -1754,7 +1953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1891,7 +2090,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1903,6 +2102,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -2024,6 +2229,7 @@ dependencies = [ "aws-config", "aws-sdk-s3", "clap", + "console-subscriber", "cxx", "cxx-build", "ed25519-dalek", @@ -2037,7 +2243,6 @@ dependencies = [ "sha2", "tokio", "tracing", - "tracing-subscriber", "url", ] @@ -2060,16 +2265,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num-conv" version = "0.1.0" @@ -2180,12 +2375,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "p256" version = "0.11.1" @@ -2226,6 +2415,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -2270,6 +2479,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "prettyplease" version = "0.2.32" @@ -2298,6 +2516,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.40" @@ -2313,6 +2563,27 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -2416,7 +2687,7 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tower", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -3023,6 +3294,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.52.0", ] @@ -3067,6 +3339,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.14" @@ -3092,11 +3375,61 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap", + "indexmap 2.9.0", "toml_datetime", "winnow", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.9", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.2" @@ -3156,17 +3489,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.19" @@ -3174,15 +3496,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", - "nu-ansi-term", "once_cell", "regex", "sharded-slab", - "smallvec", "thread_local", "tracing", "tracing-core", - "tracing-log", ] [[package]] @@ -3398,22 +3717,6 @@ dependencies = [ "rustix 0.38.44", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.9" @@ -3423,12 +3726,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-link" version = "0.1.1" @@ -3670,6 +3967,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" diff --git a/Cargo.toml b/Cargo.toml index 8adf37e..ee646cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,11 +17,11 @@ reqwest = "0.12.15" serde = { version = "1.0.219", features = [ "derive" ]} serde_json = "1.0.140" sha2 = "0.10.8" -tokio = { version = "1.44.1", features = [ "full" ]} +tokio = { version = "1.44.1", features = [ "full", "tracing", "parking_lot" ]} tracing = "0.1.41" -tracing-subscriber = { version = "0.3.19", features = ["env-filter"]} url = { version = "2.5.4", features = [ "serde" ]} cxx = "1.0" +console-subscriber = "0.4.1" [build-dependencies] cxx-build = "1.0" diff --git a/flake.nix b/flake.nix index e9165c2..46caa41 100644 --- a/flake.nix +++ b/flake.nix @@ -34,8 +34,11 @@ openssl nix boost + tokio-console ]; NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; + RUST_LOG = "nixcp=debug"; + RUST_BACKGRACE = 1; }; packages.default = craneLib.buildPackage { diff --git a/src/main.rs b/src/main.rs index 5a7e91f..f4c0f4f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,6 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Args, Parser, Subcommand}; -use tracing_subscriber::{EnvFilter, FmtSubscriber}; use push::Push; use store::Store; @@ -73,9 +72,7 @@ pub struct PushArgs { #[tokio::main] async fn main() -> Result<()> { - let filter = EnvFilter::from_default_env(); - let subscriber = FmtSubscriber::builder().with_env_filter(filter).finish(); - tracing::subscriber::set_global_default(subscriber)?; + console_subscriber::init(); let cli = Cli::parse(); diff --git a/src/path_info.rs b/src/path_info.rs index 3d54b17..e62b68a 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -7,6 +7,7 @@ use nix_compat::nixbase32; use nix_compat::store_path::StorePath; use regex::Regex; use std::path::Path; +use tokio::process::Command; use tracing::{debug, trace}; use url::Url; @@ -22,9 +23,29 @@ pub struct PathInfo { impl PathInfo { pub async fn from_path(path: &Path, store: &Store) -> Result { debug!("query path info for {:?}", path); - let canon = path.canonicalize().context("canonicalize path")?; - let store_path = StorePath::from_absolute_path(canon.into_os_string().as_encoded_bytes())?; - store.query_path_info(store_path).await + + let derivation = match path.extension() { + Some(ext) if ext == "drv" => path.as_os_str().as_encoded_bytes(), + _ => { + &Command::new("nix") + .arg("path-info") + .arg("--derivation") + .arg(path) + .output() + .await + .context(format!("run command: nix path-info --derivaiton {path:?}"))? + .stdout + } + }; + let derivation = String::from_utf8_lossy(derivation); + debug!("derivation: {derivation}"); + + let store_path = StorePath::from_absolute_path(derivation.trim().as_bytes()) + .context("storepath from derivation")?; + store + .query_path_info(store_path) + .await + .context("query pathinfo for derivation") } pub async fn get_closure(&self, store: &Store) -> Result> { diff --git a/src/push.rs b/src/push.rs index dca2a9f..8177864 100644 --- a/src/push.rs +++ b/src/push.rs @@ -13,8 +13,8 @@ use aws_config::Region; use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::narinfo::{self, SigningKey}; -use tokio::sync::{RwLock, Semaphore, mpsc}; -use tracing::{debug, trace}; +use tokio::sync::{RwLock, mpsc}; +use tracing::debug; use url::Url; use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; @@ -124,7 +124,7 @@ impl Push { for path in store_paths.into_iter() { if path.check_upstream_signature(&self.upstream_caches) { - trace!("skip {} (signature match)", path.absolute_path()); + debug!("skip {} (signature match)", path.absolute_path()); self.signature_hit_count.fetch_add(1, Ordering::Release); continue; } @@ -139,13 +139,13 @@ impl Push { .check_if_already_exists(&self.s3_client, self.bucket.clone()) .await { - trace!("skip {} (already exists)", path.absolute_path()); + debug!("skip {} (already exists)", path.absolute_path()); self.already_exists_count.fetch_add(1, Ordering::Relaxed); } else { tx.send(path).await.unwrap(); } } else { - trace!("skip {} (upstream hit)", path.absolute_path()); + debug!("skip {} (upstream hit)", path.absolute_path()); self.upstream_hit_count.fetch_add(1, Ordering::Relaxed); } }) @@ -161,13 +161,11 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::with_capacity(10); - let permits = Arc::new(Semaphore::new(10)); loop { if let Some(path_to_upload) = rx.recv().await { - let absolute_path = path_to_upload.absolute_path(); + println!("uploading: {}", path_to_upload.absolute_path()); - println!("uploading: {}", absolute_path); let uploader = Uploader::new( &self.signing_key, path_to_upload, @@ -176,10 +174,7 @@ impl Push { )?; uploads.push(tokio::spawn({ - let permits = permits.clone(); - async move { - let _permit = permits.acquire().await; let res = uploader.upload().await; self.upload_count.fetch_add(1, Ordering::Relaxed); res diff --git a/src/store.rs b/src/store.rs index 54f185c..4499243 100644 --- a/src/store.rs +++ b/src/store.rs @@ -44,7 +44,10 @@ impl Store { let inner = self.inner.clone(); task::spawn_blocking(move || { - let mut c_path_info = inner.store().query_path_info(path.to_string().as_bytes())?; + let mut c_path_info = inner + .store() + .query_path_info(path.to_string().as_bytes()) + .context("query cpp for path info")?; let signatures = c_path_info .pin_mut() @@ -60,7 +63,8 @@ impl Store { .references() .into_iter() .map(|x| StorePath::from_bytes(x.as_bytes())) - .collect::>()?; + .collect::>() + .context("get references from pathinfo")?; Ok(PathInfo { path, diff --git a/src/uploader.rs b/src/uploader.rs index cacae2b..eb955a2 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -23,6 +23,7 @@ pub struct Uploader<'a> { path: PathInfo, s3_client: &'a s3::Client, bucket: String, + hash: Sha256, } impl<'a> Uploader<'a> { @@ -37,6 +38,7 @@ impl<'a> Uploader<'a> { path, s3_client, bucket, + hash: Sha256::new(), }) } From 81ce855dae7cf803111f8543e12e61cde09a2898 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 12:37:07 -0400 Subject: [PATCH 07/32] refactor and bunch more improvements; use object_store for s3 --- Cargo.lock | 1454 +++++++++++----------------------------------- Cargo.toml | 7 +- flake.nix | 1 + src/main.rs | 10 +- src/make_nar.rs | 92 +++ src/path_info.rs | 21 +- src/push.rs | 49 +- src/store.rs | 4 +- src/uploader.rs | 217 ++----- 9 files changed, 535 insertions(+), 1320 deletions(-) create mode 100644 src/make_nar.rs diff --git a/Cargo.lock b/Cargo.lock index 9fa7a11..bbd7534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,10 +27,19 @@ dependencies = [ ] [[package]] -name = "allocator-api2" -version = "0.2.21" +name = "android-tzdata" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] [[package]] name = "anstream" @@ -147,445 +156,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "aws-config" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "hex", - "http 1.3.1", - "ring", - "time", - "tokio", - "tracing", - "url", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "zeroize", -] - -[[package]] -name = "aws-lc-rs" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" -dependencies = [ - "bindgen", - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "aws-runtime" -version = "1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-s3" -version = "1.82.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6eab2900764411ab01c8e91a76fd11a63b4e12bc3da97d9e14a0ce1343d86d3" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "fastrand", - "hex", - "hmac", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "lru", - "once_cell", - "percent-encoding", - "regex-lite", - "sha2", - "tracing", - "url", -] - -[[package]] -name = "aws-sdk-sso" -version = "1.64.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-ssooidc" -version = "1.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "1.65.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "fastrand", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" -dependencies = [ - "aws-credential-types", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "crypto-bigint 0.5.5", - "form_urlencoded", - "hex", - "hmac", - "http 0.2.12", - "http 1.3.1", - "once_cell", - "p256", - "percent-encoding", - "ring", - "sha2", - "subtle", - "time", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-async" -version = "1.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.63.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65d21e1ba6f2cdec92044f904356a19f5ad86961acf015741106cdfafd747c0" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc32c", - "crc32fast", - "crc64fast-nvme", - "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", - "pin-project-lite", - "sha1", - "sha2", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.60.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http-client" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "h2 0.4.9", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper 1.6.0", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.5", - "hyper-util", - "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.26", - "rustls-native-certs 0.8.1", - "rustls-pki-types", - "tokio", - "tower 0.5.2", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-observability" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0" -dependencies = [ - "aws-smithy-runtime-api", - "once_cell", -] - -[[package]] -name = "aws-smithy-query" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-runtime" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-client", - "aws-smithy-observability", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "fastrand", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "http-body 1.0.1", - "once_cell", - "pin-project-lite", - "pin-utils", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "1.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "bytes", - "http 0.2.12", - "http 1.3.1", - "pin-project-lite", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-types" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" -dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "http-body 1.0.1", - "http-body-util", - "itoa", - "num-integer", - "pin-project-lite", - "pin-utils", - "ryu", - "serde", - "time", - "tokio", - "tokio-util", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.60.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "1.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "rustc_version", - "tracing", -] - [[package]] name = "axum" version = "0.7.9" @@ -596,8 +166,8 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "itoa", "matchit", @@ -622,8 +192,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", @@ -648,12 +218,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "base16ct" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" - [[package]] name = "base64" version = "0.21.7" @@ -666,45 +230,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] - [[package]] name = "base64ct" version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn", - "which", -] - [[package]] name = "bitflags" version = "2.9.0" @@ -749,16 +280,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "cc" version = "1.2.19" @@ -770,15 +291,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom 7.1.3", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -786,14 +298,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "clang-sys" -version = "1.8.1" +name = "cfg_aliases" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ - "glob", - "libc", - "libloading", + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-link", ] [[package]] @@ -836,15 +356,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" -[[package]] -name = "cmake" -version = "0.1.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" -dependencies = [ - "cc", -] - [[package]] name = "codespan-reporting" version = "0.12.0" @@ -942,30 +453,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.4.2" @@ -975,15 +462,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crc64fast-nvme" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" -dependencies = [ - "crc", -] - [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -999,28 +477,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array", - "rand_core", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "rand_core", - "subtle", -] - [[package]] name = "crypto-common" version = "0.1.6" @@ -1123,16 +579,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid", - "zeroize", -] - [[package]] name = "der" version = "0.7.9" @@ -1143,15 +589,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "deranged" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" -dependencies = [ - "powerfmt", -] - [[package]] name = "digest" version = "0.10.7" @@ -1160,7 +597,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -1174,32 +610,14 @@ dependencies = [ "syn", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "ecdsa" -version = "0.14.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" -dependencies = [ - "der 0.6.1", - "elliptic-curve", - "rfc6979", - "signature 1.6.4", -] - [[package]] name = "ed25519" version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", + "pkcs8", + "signature", ] [[package]] @@ -1222,26 +640,6 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" -[[package]] -name = "elliptic-curve" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" -dependencies = [ - "base16ct", - "crypto-bigint 0.4.9", - "der 0.6.1", - "digest", - "ff", - "generic-array", - "group", - "pkcs8 0.9.0", - "rand_core", - "sec1", - "subtle", - "zeroize", -] - [[package]] name = "encoding_rs" version = "0.8.35" @@ -1284,16 +682,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "ff" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" -dependencies = [ - "rand_core", - "subtle", -] - [[package]] name = "fiat-crypto" version = "0.2.9" @@ -1346,12 +734,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.31" @@ -1458,8 +840,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1469,9 +853,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1486,36 +872,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" -[[package]] -name = "group" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" -dependencies = [ - "ff", - "rand_core", - "subtle", -] - -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.9.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.9" @@ -1527,7 +883,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.3.1", + "http", "indexmap 2.9.0", "slab", "tokio", @@ -1546,11 +902,6 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] [[package]] name = "hdrhistogram" @@ -1571,41 +922,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.3.1" @@ -1617,17 +933,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -1635,7 +940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http", ] [[package]] @@ -1646,8 +951,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -1669,30 +974,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.6.0" @@ -1702,9 +983,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.9", - "http 1.3.1", - "http-body 1.0.1", + "h2", + "http", + "http-body", "httparse", "httpdate", "itoa", @@ -1714,22 +995,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "rustls-native-certs 0.6.3", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.5" @@ -1737,14 +1002,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http 1.3.1", - "hyper 1.6.0", + "http", + "hyper", "hyper-util", - "rustls 0.23.26", - "rustls-native-certs 0.8.1", + "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tower-service", ] @@ -1754,7 +1019,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.6.0", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -1769,7 +1034,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.6.0", + "hyper", "hyper-util", "native-tls", "tokio", @@ -1786,9 +1051,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "hyper 1.6.0", + "http", + "http-body", + "hyper", "libc", "pin-project-lite", "socket2", @@ -1797,6 +1062,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1970,9 +1259,9 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] @@ -2009,28 +1298,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" -[[package]] -name = "libloading" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" -dependencies = [ - "cfg-if", - "windows-targets 0.52.6", -] - [[package]] name = "libmimalloc-sys" version = "0.1.42" @@ -2050,12 +1323,6 @@ dependencies = [ "cc", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -2084,15 +1351,6 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.2", -] - [[package]] name = "matchers" version = "0.1.0" @@ -2226,8 +1484,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-compression", - "aws-config", - "aws-sdk-s3", + "bytes", "clap", "console-subscriber", "cxx", @@ -2235,14 +1492,18 @@ dependencies = [ "ed25519-dalek", "futures", "nix-compat", + "object_store", "pkg-config", "regex", "reqwest", "serde", "serde_json", "sha2", + "tempfile", "tokio", + "tokio-util", "tracing", + "ulid", "url", ] @@ -2265,21 +1526,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -2319,6 +1565,40 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.8.5", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -2369,23 +1649,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "outref" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" - -[[package]] -name = "p256" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" -dependencies = [ - "ecdsa", - "elliptic-curve", - "sha2", -] - [[package]] name = "parking_lot" version = "0.12.3" @@ -2447,24 +1710,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" -dependencies = [ - "der 0.6.1", - "spki 0.6.0", -] - [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.9", - "spki 0.7.3", + "der", + "spki", ] [[package]] @@ -2473,12 +1726,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2488,16 +1735,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -2548,6 +1785,70 @@ dependencies = [ "prost", ] +[[package]] +name = "quick-xml" +version = "0.37.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +dependencies = [ + "bytes", + "getrandom 0.3.2", + "rand 0.9.1", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.40" @@ -2570,8 +1871,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -2581,7 +1892,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -2593,6 +1914,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -2634,12 +1964,6 @@ dependencies = [ "regex-syntax 0.8.5", ] -[[package]] -name = "regex-lite" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" - [[package]] name = "regex-syntax" version = "0.6.29" @@ -2663,12 +1987,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.9", - "http 1.3.1", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.6.0", - "hyper-rustls 0.27.5", + "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "ipnet", @@ -2679,7 +2003,11 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.2.0", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2687,26 +2015,18 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-rustls", + "tokio-util", "tower 0.5.2", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "windows-registry", ] -[[package]] -name = "rfc6979" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" -dependencies = [ - "crypto-bigint 0.4.9", - "hmac", - "zeroize", -] - [[package]] name = "ring" version = "0.17.14" @@ -2729,9 +2049,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -2742,19 +2062,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.0.5" @@ -2764,48 +2071,24 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.9.4", + "linux-raw-sys", "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ - "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki", "subtle", "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -2818,15 +2101,6 @@ dependencies = [ "security-framework 3.2.0", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -2841,15 +2115,8 @@ name = "rustls-pki-types" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "web-time", ] [[package]] @@ -2858,7 +2125,6 @@ version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2876,6 +2142,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.27" @@ -2897,30 +2172,6 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "sec1" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" -dependencies = [ - "base16ct", - "der 0.6.1", - "generic-array", - "pkcs8 0.9.0", - "subtle", - "zeroize", -] - [[package]] name = "security-framework" version = "2.11.1" @@ -3007,17 +2258,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" version = "0.10.8" @@ -3053,23 +2293,13 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest", - "rand_core", -] - [[package]] name = "signature" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3097,16 +2327,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spki" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] - [[package]] name = "spki" version = "0.7.3" @@ -3114,7 +2334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.9", + "der", ] [[package]] @@ -3196,7 +2416,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix 1.0.5", + "rustix", "windows-sys 0.59.0", ] @@ -3239,36 +2459,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "time" -version = "0.3.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" - -[[package]] -name = "time-macros" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" -dependencies = [ - "num-conv", - "time-core", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -3279,6 +2469,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.44.2" @@ -3319,23 +2524,13 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.26", + "rustls", "tokio", ] @@ -3352,9 +2547,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -3391,11 +2586,11 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.9", - "http 1.3.1", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.6.0", + "hyper", "hyper-timeout", "hyper-util", "percent-encoding", @@ -3421,7 +2616,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util", @@ -3516,6 +2711,16 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand 0.9.1", + "web-time", +] + [[package]] name = "unicode-ident" version = "1.0.18" @@ -3546,12 +2751,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf16_iter" version = "1.0.5" @@ -3570,12 +2769,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" - [[package]] name = "valuable" version = "0.1.1" @@ -3595,10 +2788,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "vsimd" -version = "0.8.0" +name = "walkdir" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] [[package]] name = "want" @@ -3695,6 +2892,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.77" @@ -3706,15 +2916,13 @@ dependencies = [ ] [[package]] -name = "which" -version = "4.4.2" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", + "js-sys", + "wasm-bindgen", ] [[package]] @@ -3726,6 +2934,41 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "windows-core" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings 0.4.0", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.1.1" @@ -3739,7 +2982,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", - "windows-strings", + "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -3761,6 +3004,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -3937,12 +3189,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" - [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index ee646cf..abaf3d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,6 @@ edition = "2024" [dependencies] anyhow = "1.0.97" async-compression = { version = "0.4.22", features = ["tokio", "zstd"] } -aws-config = { version = "1.6.1", features = ["behavior-version-latest"] } -aws-sdk-s3 = "1.82.0" clap = { version = "4.5.34", features = ["derive"] } ed25519-dalek = "2.1.1" futures = "0.3.31" @@ -22,6 +20,11 @@ tracing = "0.1.41" url = { version = "2.5.4", features = [ "serde" ]} cxx = "1.0" console-subscriber = "0.4.1" +tempfile = "3.19.1" +tokio-util = { version = "0.7.15", features = ["io"] } +bytes = "1.10.1" +object_store = { version = "0.12.0", features = ["aws"] } +ulid = "1.2.1" [build-dependencies] cxx-build = "1.0" diff --git a/flake.nix b/flake.nix index 46caa41..5796366 100644 --- a/flake.nix +++ b/flake.nix @@ -35,6 +35,7 @@ nix boost tokio-console + cargo-udeps ]; NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; RUST_LOG = "nixcp=debug"; diff --git a/src/main.rs b/src/main.rs index f4c0f4f..bca419b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ #![feature(let_chains)] #![feature(extend_one)] +#![feature(exit_status_error)] use std::path::PathBuf; @@ -11,6 +12,7 @@ use store::Store; mod bindings; mod cli; +mod make_nar; mod path_info; mod push; mod store; @@ -48,19 +50,15 @@ pub struct PushArgs { #[arg(long)] signing_key: String, - /// If unspecified, will get it form AWS_DEFAULT_REGION envar + /// If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 #[arg(long)] region: Option, - /// If unspecifed, will get it from AWS_ENDPOINT_URL envar + /// If unspecifed, will get it from AWS_ENDPOINT envar /// e.g. https://s3.example.com #[arg(long)] endpoint: Option, - /// AWS profile to use - #[arg(long)] - profile: Option, - #[arg(long)] skip_signature_check: bool, diff --git a/src/make_nar.rs b/src/make_nar.rs new file mode 100644 index 0000000..6e8d512 --- /dev/null +++ b/src/make_nar.rs @@ -0,0 +1,92 @@ +use anyhow::{Context, Result}; +use async_compression::{Level, tokio::bufread::ZstdEncoder}; +use nix_compat::{ + narinfo::{self, NarInfo}, + store_path::StorePath, +}; +use sha2::{Digest, Sha256}; +use std::mem::take; +use tempfile::NamedTempFile; +use tokio::{ + fs::File, + io::{AsyncRead, BufReader}, + process::Command, +}; +use tokio_util::io::InspectReader; + +use crate::path_info::PathInfo; + +pub struct MakeNar<'a> { + path_info: &'a PathInfo, + nar_file: NamedTempFile, + nar_hasher: Sha256, + /// hash of compressed nar file + file_hasher: Sha256, + nar_size: u64, + file_size: u64, +} + +impl<'a> MakeNar<'a> { + pub fn new(path_info: &'a PathInfo) -> Result { + Ok(Self { + path_info, + nar_file: NamedTempFile::new().context("crated tempfile for nar")?, + nar_hasher: Sha256::new(), + file_hasher: Sha256::new(), + nar_size: 0, + file_size: 0, + }) + } + + pub async fn make(&self) -> Result<()> { + Ok(Command::new("nix") + .arg("nar") + .arg("dump-path") + .arg(self.path_info.absolute_path()) + .kill_on_drop(true) + .stdout(self.nar_file.reopen()?) + .spawn()? + .wait() + .await? + .exit_ok()?) + } + + /// Returns a compressed nar reader which can be uploaded. File hash will be available when + /// everything is read + pub async fn compress_and_hash(&mut self) -> Result { + let nar_file = File::from_std(self.nar_file.reopen()?); + // reader that hashes as nar is read + let nar_reader = InspectReader::new(nar_file, |x| self.nar_hasher.update(x)); + + let encoder = ZstdEncoder::with_quality(BufReader::new(nar_reader), Level::Default); + // reader that updates file_hash as the compressed nar is read + Ok(InspectReader::new(encoder, |x| self.file_hasher.update(x))) + } + + /// Returns *unsigned* narinfo. `url` must be updated before uploading + pub fn get_narinfo(&mut self) -> Result { + let file_hash = take(&mut self.file_hasher).finalize().into(); + Ok(NarInfo { + flags: narinfo::Flags::empty(), + store_path: self.path_info.path.as_ref(), + nar_hash: take(&mut self.nar_hasher).finalize().into(), + nar_size: self.nar_size, + references: self + .path_info + .references + .iter() + .map(StorePath::as_ref) + .collect(), + signatures: Vec::new(), + ca: None, + system: None, + deriver: None, + compression: Some("zstd"), + file_hash: Some(file_hash), + file_size: Some(self.file_size), + url: "", + }) + // signature consists of: store_path, nar_hash, nar_size, and references + // nar_info.add_signature(self.signing_key); + } +} diff --git a/src/path_info.rs b/src/path_info.rs index e62b68a..f88e464 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,10 +1,10 @@ use std::collections::HashSet; use anyhow::{Context, Result}; -use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::nixbase32; use nix_compat::store_path::StorePath; +use object_store::{ObjectStore, aws::AmazonS3, path::Path as ObjectPath}; use regex::Regex; use std::path::Path; use tokio::process::Command; @@ -82,13 +82,13 @@ impl PathInfo { .filter_map(|signature| Some(signature.split_once(":")?.0)) .collect(); trace!("signers for {}: {:?}", self.path, signers); - return signers; + signers } pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { for upstream in upstreams { let upstream = upstream - .join(format!("{}.narinfo", self.digest()).as_str()) + .join(self.narinfo_path().as_ref()) .expect("adding .narinfo should make a valid url"); debug!("querying {}", upstream); let res_status = reqwest::Client::new() @@ -108,17 +108,12 @@ impl PathInfo { self.path.to_absolute_path() } - pub fn digest(&self) -> String { - nixbase32::encode(self.path.digest()) + pub fn narinfo_path(&self) -> ObjectPath { + ObjectPath::parse(format!("{}.narinfo", nixbase32::encode(self.path.digest()))) + .expect("must parse to a valid object_store path") } - pub async fn check_if_already_exists(&self, s3_client: &s3::Client, bucket: String) -> bool { - s3_client - .head_object() - .bucket(bucket) - .key(format!("{}.narinfo", self.digest())) - .send() - .await - .is_ok() + pub async fn check_if_already_exists(&self, s3: &AmazonS3) -> bool { + s3.head(&self.narinfo_path()).await.is_ok() } } diff --git a/src/push.rs b/src/push.rs index 8177864..c5a4229 100644 --- a/src/push.rs +++ b/src/push.rs @@ -9,23 +9,23 @@ use std::{ }; use anyhow::{Context, Result}; -use aws_config::Region; -use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::narinfo::{self, SigningKey}; -use tokio::sync::{RwLock, mpsc}; +use object_store::aws::{AmazonS3, AmazonS3Builder}; +use tokio::sync::{RwLock, Semaphore, mpsc}; use tracing::debug; use url::Url; use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; +const UPLOAD_CONCURRENCY: usize = 32; + pub struct Push { upstream_caches: Vec, store_paths: Arc>>, - s3_client: s3::Client, signing_key: SigningKey, - bucket: String, store: Arc, + s3: Arc, // paths that we skipped cause of a signature match signature_hit_count: AtomicUsize, // paths that we skipped cause we found it on an upstream @@ -51,25 +51,21 @@ impl Push { let key = fs::read_to_string(&cli.signing_key)?; let signing_key = narinfo::parse_keypair(key.as_str())?.0; - let mut s3_config = aws_config::from_env(); + let mut s3_builder = AmazonS3Builder::from_env().with_bucket_name(&cli.bucket); + if let Some(region) = &cli.region { - s3_config = s3_config.region(Region::new(region.clone())); + s3_builder = s3_builder.with_region(region); } if let Some(endpoint) = &cli.endpoint { - s3_config = s3_config.endpoint_url(endpoint); - } - if let Some(profile) = &cli.profile { - s3_config = s3_config.profile_name(profile); + s3_builder = s3_builder.with_endpoint(endpoint); } - let s3_client = s3::Client::new(&s3_config.load().await); Ok(Self { upstream_caches: upstreams, store_paths: Arc::new(RwLock::new(Vec::new())), - s3_client, signing_key, - bucket: cli.bucket.clone(), store: Arc::new(store), + s3: Arc::new(s3_builder.build()?), signature_hit_count: AtomicUsize::new(0), upstream_hit_count: AtomicUsize::new(0), already_exists_count: AtomicUsize::new(0), @@ -109,9 +105,10 @@ impl Push { } pub async fn run(&'static self) -> Result<()> { - let (tx, rx) = mpsc::channel(10); + let (tx, rx) = mpsc::channel(UPLOAD_CONCURRENCY); let filter = tokio::spawn(self.filter_from_upstream(tx)); let upload = tokio::spawn(self.upload(rx)); + filter.await?; upload.await??; Ok(()) @@ -135,10 +132,7 @@ impl Push { .check_upstream_hit(self.upstream_caches.as_slice()) .await { - if path - .check_if_already_exists(&self.s3_client, self.bucket.clone()) - .await - { + if path.check_if_already_exists(&self.s3).await { debug!("skip {} (already exists)", path.absolute_path()); self.already_exists_count.fetch_add(1, Ordering::Relaxed); } else { @@ -160,22 +154,23 @@ impl Push { } async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { - let mut uploads = Vec::with_capacity(10); + let mut uploads = Vec::new(); + let permits = Arc::new(Semaphore::new(UPLOAD_CONCURRENCY)); loop { + let permits = permits.clone(); + let permit = permits.acquire_owned().await.unwrap(); + if let Some(path_to_upload) = rx.recv().await { println!("uploading: {}", path_to_upload.absolute_path()); - let uploader = Uploader::new( - &self.signing_key, - path_to_upload, - &self.s3_client, - self.bucket.clone(), - )?; + let uploader = Uploader::new(&self.signing_key, path_to_upload)?; uploads.push(tokio::spawn({ + let s3 = self.s3.clone(); async move { - let res = uploader.upload().await; + let res = uploader.upload(s3).await; + drop(permit); self.upload_count.fetch_add(1, Ordering::Relaxed); res } diff --git a/src/store.rs b/src/store.rs index 4499243..763cb3d 100644 --- a/src/store.rs +++ b/src/store.rs @@ -28,13 +28,13 @@ impl Store { inner .store() .compute_fs_closure(path.to_string().as_bytes(), false, true, true)?; - Ok(cxx_vector + cxx_vector .iter() .map(|x| { StorePath::from_bytes(x.as_bytes()) .context("make StorePath from vector returned by compute_fs_closure") }) - .collect::>()?) + .collect::>() }) .await .unwrap() diff --git a/src/uploader.rs b/src/uploader.rs index eb955a2..c9c49f0 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -1,190 +1,75 @@ use anyhow::Result; -use async_compression::{Level, tokio::bufread::ZstdEncoder}; -use aws_sdk_s3::{ - self as s3, - types::{CompletedMultipartUpload, CompletedPart}, -}; -use futures::future::join_all; -use nix_compat::{ - narinfo::{self, NarInfo, SigningKey}, - nixbase32, - store_path::StorePath, -}; -use sha2::{Digest, Sha256}; -use tokio::{io::AsyncReadExt, process::Command}; -use tracing::debug; +use bytes::BytesMut; +use nix_compat::{narinfo::SigningKey, nixbase32}; +use object_store::{ObjectStore, aws::AmazonS3, buffered::BufWriter, path::Path}; +use std::sync::Arc; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tracing::{debug, trace}; +use ulid::Ulid; -use crate::path_info::PathInfo; +use crate::{make_nar::MakeNar, path_info::PathInfo}; -const MULTIPART_CUTOFF: usize = 1024 * 1024 * 5; +const CHUNK_SIZE: usize = 1024 * 1024 * 5; pub struct Uploader<'a> { signing_key: &'a SigningKey, path: PathInfo, - s3_client: &'a s3::Client, - bucket: String, - hash: Sha256, } impl<'a> Uploader<'a> { pub fn new( signing_key: &'a SigningKey, path: PathInfo, - s3_client: &'a s3::Client, - bucket: String, ) -> Result { - Ok(Self { - signing_key, - path, - s3_client, - bucket, - hash: Sha256::new(), - }) + Ok(Self { signing_key, path }) } - pub async fn upload(&self) -> Result<()> { - let nar = self.make_nar().await?; - let mut nar_info = self.narinfo_from_nar(&nar)?; - let nar = self.compress_nar(&nar).await; + pub async fn upload(&self, s3: Arc) -> Result<()> { + let mut nar = MakeNar::new(&self.path)?; + nar.make().await?; - // update fields that we know after compression - let mut hasher = Sha256::new(); - hasher.update(&nar); - let hash: [u8; 32] = hasher.finalize().into(); - let nar_url = self.nar_url(&hash); - nar_info.file_hash = Some(hash); - nar_info.file_size = Some(nar.len() as u64); - nar_info.url = nar_url.as_str(); - debug!("uploading nar with key: {nar_url}"); + // we don't know what the hash of the compressed file will be so upload to a + // temp location for now + let temp_path = Path::parse(Ulid::new().to_string())?; + let mut s3_writer = BufWriter::new(s3.clone(), temp_path.clone()); - if nar.len() < MULTIPART_CUTOFF { - let put_object = self - .s3_client - .put_object() - .bucket(&self.bucket) - .key(&nar_url) - .body(nar.into()) - .send() - .await?; - debug!("put object: {:#?}", put_object); - } else { - let multipart = self - .s3_client - .create_multipart_upload() - .bucket(&self.bucket) - .key(&nar_url) - .send() - .await?; - let upload_id = multipart.upload_id().unwrap(); - - let mut parts = Vec::with_capacity(nar.len() / MULTIPART_CUTOFF); - let chunks = nar.chunks(MULTIPART_CUTOFF); - for (i, chunk) in chunks.enumerate() { - parts.push(tokio::task::spawn( - self.s3_client - .upload_part() - .bucket(&self.bucket) - .key(&nar_url) - .upload_id(upload_id) - .part_number(i as i32 + 1) - .body(chunk.to_vec().into()) - .send(), - )); - } - - let completed_parts = join_all(parts) - .await - .into_iter() - .flatten() - .collect::, _>>()? - .into_iter() - .enumerate() - .map(|(i, part)| { - CompletedPart::builder() - .set_e_tag(part.e_tag().map(ToString::to_string)) - .set_part_number(Some(i as i32 + 1)) - .set_checksum_sha256(part.checksum_sha256().map(ToString::to_string)) - .build() - }) - .collect::>(); - - let completed_mp_upload = CompletedMultipartUpload::builder() - .set_parts(Some(completed_parts)) - .build(); - - let complete_mp_upload = self - .s3_client - .complete_multipart_upload() - .bucket(&self.bucket) - .key(&nar_url) - .upload_id(upload_id) - .multipart_upload(completed_mp_upload) - .send() - .await?; - - debug!("complete multipart upload: {:#?}", complete_mp_upload); + // compress and upload nar + let mut file_reader = nar.compress_and_hash().await?; + let mut buf = BytesMut::with_capacity(CHUNK_SIZE); + debug!("uploading to temp path: {}", temp_path); + while let n = file_reader.read_buf(&mut buf).await? + && n != 0 + { + s3_writer.write_all_buf(&mut buf).await?; } + drop(file_reader); - let narinfo_url = format!("{}.narinfo", self.path.digest()); - debug!("uploading narinfo with key {narinfo_url}"); - self.s3_client - .put_object() - .bucket(&self.bucket) - .key(narinfo_url) - .body(nar_info.to_string().as_bytes().to_vec().into()) - .send() - .await?; - debug!("done uploading narinfo"); + let mut nar_info = nar.get_narinfo()?; + nar_info.add_signature(self.signing_key); + trace!("narinfo: {:#}", nar_info); + + // now that we can calculate the file_hash move the nar to where it should be + let real_path = nar_url( + &nar_info + .file_hash + .expect("file hash must be known at this point"), + ); + debug!("moving {} to {}", temp_path, real_path); + // this is implemented as a copy-and-delete + s3.rename(&temp_path, &real_path).await?; + + // upload narinfo + let narinfo_path = self.path.narinfo_path(); + debug!("uploading narinfo: {}", narinfo_path); + s3.put(&narinfo_path, nar_info.to_string().into()).await?; Ok(()) } - - async fn make_nar(&self) -> Result> { - Ok(Command::new("nix") - .arg("nar") - .arg("dump-path") - .arg(self.path.absolute_path()) - .output() - .await? - .stdout) - } - - fn narinfo_from_nar(&self, nar: &[u8]) -> Result { - let mut hasher = Sha256::new(); - hasher.update(nar); - let nar_hash: [u8; 32] = hasher.finalize().into(); - let mut nar_info = NarInfo { - flags: narinfo::Flags::empty(), - store_path: self.path.path.as_ref(), - nar_hash, - nar_size: nar.len() as u64, - references: self.path.references.iter().map(StorePath::as_ref).collect(), - signatures: Vec::new(), - ca: None, - system: None, - deriver: None, - compression: Some("zstd"), - file_hash: None, - file_size: None, - url: "", - }; - // signature consists of: store_path, nar_hash, nar_size, and references - nar_info.add_signature(self.signing_key); - Ok(nar_info) - } - - fn nar_url(&self, compressed_nar_hash: &[u8]) -> String { - let compressed_nar_hash = nixbase32::encode(compressed_nar_hash); - format!("nar/{compressed_nar_hash}.nar.zst") - } - - async fn compress_nar(&self, nar: &[u8]) -> Vec { - let mut encoder = ZstdEncoder::with_quality(nar, Level::Default); - let mut compressed = Vec::with_capacity(nar.len()); - encoder - .read_to_end(&mut compressed) - .await - .expect("should compress just fine"); - compressed - } +} + +/// calculate url where the compressed nar should be uploaded +fn nar_url(file_hash: &[u8]) -> Path { + let compressed_nar_hash = nixbase32::encode(file_hash); + Path::parse(format!("nar/{compressed_nar_hash}.nar.zst")) + .expect("should parse to a valid object_store::path::Path") } From 846c465ea0988f2b5abef44d7443d536102acee1 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 13:11:54 -0400 Subject: [PATCH 08/32] fix flake --- flake.nix | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/flake.nix b/flake.nix index 5796366..1ee2f5d 100644 --- a/flake.nix +++ b/flake.nix @@ -23,34 +23,40 @@ toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); lib = pkgs.lib; + nativeBuildInputs = with pkgs; [ + pkg-config + ]; + buildInputs = with pkgs; [ + toolchain + openssl + nix + boost + ]; + env = { + # for cpp bindings to work + NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; + }; in { devShells.default = pkgs.mkShell { - nativeBuildInputs = with pkgs; [ - pkg-config - ]; - buildInputs = with pkgs; [ - toolchain - openssl - nix - boost + inherit buildInputs; + inherit nativeBuildInputs; + packages = with pkgs; [ tokio-console cargo-udeps ]; - NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; - RUST_LOG = "nixcp=debug"; - RUST_BACKGRACE = 1; + env = env // { + RUST_LOG = "nixcp=debug"; + RUST_BACKGRACE = 1; + }; }; packages.default = craneLib.buildPackage { + inherit nativeBuildInputs; + inherit buildInputs; + inherit env; src = craneLib.cleanCargoSource ./.; strictDeps = true; - nativeBuildInputs = with pkgs; [ - pkg-config - ]; - buildInputs = with pkgs; [ - openssl - ]; }; } ); From b8877f33a39ab2aaa6b5586b4b8a1500b1555c52 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 14:10:20 -0400 Subject: [PATCH 09/32] enable lto --- Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index abaf3d5..f548ea8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,10 @@ name = "nixcp" version = "0.1.0" edition = "2024" +[profile.release] +lto = true +codegen-units = 1 + [dependencies] anyhow = "1.0.97" async-compression = { version = "0.4.22", features = ["tokio", "zstd"] } From fc304df35ec2b3e0cb9dcede9d1d6bd9a5e3896c Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 14:21:18 -0400 Subject: [PATCH 10/32] refactor flake --- flake.nix | 52 ++++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/flake.nix b/flake.nix index 1ee2f5d..e43527f 100644 --- a/flake.nix +++ b/flake.nix @@ -23,41 +23,45 @@ toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); lib = pkgs.lib; - nativeBuildInputs = with pkgs; [ - pkg-config - ]; - buildInputs = with pkgs; [ - toolchain - openssl - nix - boost - ]; - env = { + src = craneLib.cleanCargoSource ./.; + + commonArgs = { + inherit src; + strictDeps = true; + nativeBuildInputs = with pkgs; [ + pkg-config + ]; + buildInputs = with pkgs; [ + toolchain + openssl + nix + boost + ]; # for cpp bindings to work NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; }; + + cargoArtifacts = craneLib.buildDepsOnly commonArgs; + nixcp = craneLib.buildPackage (commonArgs // { + inherit cargoArtifacts; + }); in { - devShells.default = pkgs.mkShell { - inherit buildInputs; - inherit nativeBuildInputs; + devShells.default = craneLib.devShell { + inputsFrom = [ nixcp ]; + + RUST_LOG = "nixcp=debug"; + RUST_BACKGRACE = 1; + # for cpp bindings to work + NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; + packages = with pkgs; [ tokio-console cargo-udeps ]; - env = env // { - RUST_LOG = "nixcp=debug"; - RUST_BACKGRACE = 1; - }; }; - packages.default = craneLib.buildPackage { - inherit nativeBuildInputs; - inherit buildInputs; - inherit env; - src = craneLib.cleanCargoSource ./.; - strictDeps = true; - }; + packages.default = nixcp; } ); } From 5a3e6089b4e373e74b4050778ca5b09ef42acc54 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 14:40:38 -0400 Subject: [PATCH 11/32] don't clean cpp files --- flake.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e43527f..01d135a 100644 --- a/flake.nix +++ b/flake.nix @@ -23,7 +23,16 @@ toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); lib = pkgs.lib; - src = craneLib.cleanCargoSource ./.; + + # don't clean cpp files + cppFilter = path: _type: builtins.match ".*(cpp|hpp)$" path != null; + cppOrCargo = path: type: + (cppFilter path type) || (craneLib.filterCargoSources path type); + src = lib.cleanSourceWith { + src = ./.; + filter = cppOrCargo; + name = "source"; + }; commonArgs = { inherit src; From d524222a862018da5b508a06985935db50d737b0 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 15:37:54 -0400 Subject: [PATCH 12/32] make tokio console optional and make it actually work --- Cargo.lock | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 4 ++++ src/main.rs | 29 ++++++++++++++++++++++++++--- 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbd7534..a3e049c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1503,6 +1503,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "tracing-subscriber", "ulid", "url", ] @@ -1526,6 +1527,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1649,6 +1660,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.3" @@ -2684,6 +2701,17 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.19" @@ -2691,12 +2719,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", + "nu-ansi-term", "once_cell", "regex", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", + "tracing-log", ] [[package]] @@ -2925,6 +2956,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.9" @@ -2934,6 +2981,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.61.0" diff --git a/Cargo.toml b/Cargo.toml index f548ea8..8dd350d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,9 @@ name = "nixcp" version = "0.1.0" edition = "2024" +[build] +rustflags = ["--cfg", "tokio_unstable"] + [profile.release] lto = true codegen-units = 1 @@ -29,6 +32,7 @@ tokio-util = { version = "0.7.15", features = ["io"] } bytes = "1.10.1" object_store = { version = "0.12.0", features = ["aws"] } ulid = "1.2.1" +tracing-subscriber = "0.3.19" [build-dependencies] cxx-build = "1.0" diff --git a/src/main.rs b/src/main.rs index bca419b..0fefdf5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,11 @@ #![feature(let_chains)] -#![feature(extend_one)] #![feature(exit_status_error)] use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Args, Parser, Subcommand}; +use tracing_subscriber::{EnvFilter, prelude::*}; use push::Push; use store::Store; @@ -26,6 +26,10 @@ mod uploader; struct Cli { #[command(subcommand)] command: Commands, + + /// Whether to enable tokio-console + #[arg(long)] + tokio_console: bool, } #[derive(Debug, Subcommand)] @@ -70,9 +74,8 @@ pub struct PushArgs { #[tokio::main] async fn main() -> Result<()> { - console_subscriber::init(); - let cli = Cli::parse(); + init_logging(cli.tokio_console); match &cli.command { Commands::Push(cli) => { @@ -87,3 +90,23 @@ async fn main() -> Result<()> { Ok(()) } + +fn init_logging(tokio_console: bool) { + let env_filter = EnvFilter::from_default_env(); + let fmt_layer = tracing_subscriber::fmt::layer().with_filter(env_filter); + + let console_layer = if tokio_console { + Some(console_subscriber::spawn()) + } else { + None + }; + + tracing_subscriber::registry() + .with(fmt_layer) + .with(console_layer) + .init(); + + if tokio_console { + println!("tokio-console is enabled"); + } +} From 7dec14fc1a80f6f7c0a6da83f581d5322c728228 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 15:46:23 -0400 Subject: [PATCH 13/32] set path to nar in narinfo --- src/make_nar.rs | 2 -- src/uploader.rs | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/make_nar.rs b/src/make_nar.rs index 6e8d512..e1ca57c 100644 --- a/src/make_nar.rs +++ b/src/make_nar.rs @@ -86,7 +86,5 @@ impl<'a> MakeNar<'a> { file_size: Some(self.file_size), url: "", }) - // signature consists of: store_path, nar_hash, nar_size, and references - // nar_info.add_signature(self.signing_key); } } diff --git a/src/uploader.rs b/src/uploader.rs index c9c49f0..0832a58 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -57,6 +57,8 @@ impl<'a> Uploader<'a> { debug!("moving {} to {}", temp_path, real_path); // this is implemented as a copy-and-delete s3.rename(&temp_path, &real_path).await?; + // set nar url in narinfo + nar_info.url = real_path.as_ref(); // upload narinfo let narinfo_path = self.path.narinfo_path(); From 0fae7ac3b08c73a73bf9a8cda9e29aacdf22b4c2 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 18:21:27 -0400 Subject: [PATCH 14/32] fix build with tokio unstable --- .cargo/config.toml | 2 ++ Cargo.toml | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..bff29e6 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] diff --git a/Cargo.toml b/Cargo.toml index 8dd350d..6ca052f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,9 +3,6 @@ name = "nixcp" version = "0.1.0" edition = "2024" -[build] -rustflags = ["--cfg", "tokio_unstable"] - [profile.release] lto = true codegen-units = 1 From 0fedae933465d2b1935af2ff0498118393203ed1 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 18:21:49 -0400 Subject: [PATCH 15/32] try to fix fd issues --- src/make_nar.rs | 36 +++++++++++++++++++++++++++--------- src/push.rs | 3 ++- src/uploader.rs | 18 +++++++++++------- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/src/make_nar.rs b/src/make_nar.rs index e1ca57c..19a112b 100644 --- a/src/make_nar.rs +++ b/src/make_nar.rs @@ -6,19 +6,20 @@ use nix_compat::{ }; use sha2::{Digest, Sha256}; use std::mem::take; -use tempfile::NamedTempFile; +use tempfile::{NamedTempFile, TempPath}; use tokio::{ fs::File, io::{AsyncRead, BufReader}, process::Command, }; use tokio_util::io::InspectReader; +use tracing::debug; use crate::path_info::PathInfo; pub struct MakeNar<'a> { path_info: &'a PathInfo, - nar_file: NamedTempFile, + nar_file: TempPath, nar_hasher: Sha256, /// hash of compressed nar file file_hasher: Sha256, @@ -30,7 +31,9 @@ impl<'a> MakeNar<'a> { pub fn new(path_info: &'a PathInfo) -> Result { Ok(Self { path_info, - nar_file: NamedTempFile::new().context("crated tempfile for nar")?, + nar_file: NamedTempFile::new() + .context("create tempfile for nar")? + .into_temp_path(), nar_hasher: Sha256::new(), file_hasher: Sha256::new(), nar_size: 0, @@ -39,12 +42,19 @@ impl<'a> MakeNar<'a> { } pub async fn make(&self) -> Result<()> { + let path = self.path_info.absolute_path(); + let out = std::fs::File::options() + .write(true) + .truncate(true) + .open(&self.nar_file)?; + + debug!("dumping nar for: {}", path); Ok(Command::new("nix") .arg("nar") - .arg("dump-path") + .arg("pack") .arg(self.path_info.absolute_path()) .kill_on_drop(true) - .stdout(self.nar_file.reopen()?) + .stdout(out) .spawn()? .wait() .await? @@ -54,22 +64,30 @@ impl<'a> MakeNar<'a> { /// Returns a compressed nar reader which can be uploaded. File hash will be available when /// everything is read pub async fn compress_and_hash(&mut self) -> Result { - let nar_file = File::from_std(self.nar_file.reopen()?); + let nar_file = File::open(&self.nar_file).await?; // reader that hashes as nar is read - let nar_reader = InspectReader::new(nar_file, |x| self.nar_hasher.update(x)); + let nar_reader = InspectReader::new(nar_file, |x| { + self.nar_size += x.len() as u64; + self.nar_hasher.update(x); + }); let encoder = ZstdEncoder::with_quality(BufReader::new(nar_reader), Level::Default); // reader that updates file_hash as the compressed nar is read - Ok(InspectReader::new(encoder, |x| self.file_hasher.update(x))) + Ok(InspectReader::new(encoder, |x| { + self.file_size += x.len() as u64; + self.file_hasher.update(x); + })) } /// Returns *unsigned* narinfo. `url` must be updated before uploading pub fn get_narinfo(&mut self) -> Result { let file_hash = take(&mut self.file_hasher).finalize().into(); + let nar_hash = take(&mut self.nar_hasher).finalize().into(); + Ok(NarInfo { flags: narinfo::Flags::empty(), store_path: self.path_info.path.as_ref(), - nar_hash: take(&mut self.nar_hasher).finalize().into(), + nar_hash, nar_size: self.nar_size, references: self .path_info diff --git a/src/push.rs b/src/push.rs index c5a4229..93a28da 100644 --- a/src/push.rs +++ b/src/push.rs @@ -18,7 +18,7 @@ use url::Url; use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; -const UPLOAD_CONCURRENCY: usize = 32; +const UPLOAD_CONCURRENCY: usize = 5; pub struct Push { upstream_caches: Vec, @@ -159,6 +159,7 @@ impl Push { loop { let permits = permits.clone(); + debug!("upload permits available: {}", permits.available_permits()); let permit = permits.acquire_owned().await.unwrap(); if let Some(path_to_upload) = rx.recv().await { diff --git a/src/uploader.rs b/src/uploader.rs index 0832a58..a2c2766 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -32,21 +32,22 @@ impl<'a> Uploader<'a> { // temp location for now let temp_path = Path::parse(Ulid::new().to_string())?; let mut s3_writer = BufWriter::new(s3.clone(), temp_path.clone()); + debug!("uploading to temp path: {}", temp_path); // compress and upload nar let mut file_reader = nar.compress_and_hash().await?; - let mut buf = BytesMut::with_capacity(CHUNK_SIZE); - debug!("uploading to temp path: {}", temp_path); - while let n = file_reader.read_buf(&mut buf).await? - && n != 0 - { - s3_writer.write_all_buf(&mut buf).await?; + loop { + let mut buf = BytesMut::with_capacity(CHUNK_SIZE); + let n = file_reader.read_buf(&mut buf).await?; + s3_writer.put(buf.freeze()).await?; + if n == 0 { + break; + } } drop(file_reader); let mut nar_info = nar.get_narinfo()?; nar_info.add_signature(self.signing_key); - trace!("narinfo: {:#}", nar_info); // now that we can calculate the file_hash move the nar to where it should be let real_path = nar_url( @@ -55,6 +56,8 @@ impl<'a> Uploader<'a> { .expect("file hash must be known at this point"), ); debug!("moving {} to {}", temp_path, real_path); + // the temp object must be done uploading + s3_writer.shutdown().await?; // this is implemented as a copy-and-delete s3.rename(&temp_path, &real_path).await?; // set nar url in narinfo @@ -63,6 +66,7 @@ impl<'a> Uploader<'a> { // upload narinfo let narinfo_path = self.path.narinfo_path(); debug!("uploading narinfo: {}", narinfo_path); + trace!("narinfo: {:#}", nar_info); s3.put(&narinfo_path, nar_info.to_string().into()).await?; Ok(()) From 85fefe9e7778423820860d1a267869f3b7295294 Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 21:08:01 -0400 Subject: [PATCH 16/32] use cpp bindings to make nar --- src/bindings/mod.rs | 21 +++++++++++++++----- src/bindings/nix.cpp | 11 +++++++++++ src/bindings/nix.hpp | 1 + src/make_nar.rs | 47 ++++++++++---------------------------------- src/push.rs | 3 ++- src/store.rs | 24 ++++++++++++++++++++-- src/uploader.rs | 9 ++++----- 7 files changed, 66 insertions(+), 50 deletions(-) diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 61a32af..8084dff 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -23,6 +23,7 @@ use std::pin::Pin; use std::task::{Context, Poll}; use anyhow::Result; +use bytes::Bytes; use futures::stream::{Stream, StreamExt}; use tokio::io::{AsyncWrite, AsyncWriteExt}; @@ -125,7 +126,7 @@ impl AsyncWriteAdapter { writer.write_all(&v).await?; } Err(e) => { - return Err(e); + return Err(e.into()); } } } @@ -139,7 +140,7 @@ impl AsyncWriteAdapter { } impl Stream for AsyncWriteAdapter { - type Item = Result>; + type Item = std::io::Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.receiver.poll_recv(cx) { @@ -147,9 +148,12 @@ impl Stream for AsyncWriteAdapter { Poll::Ready(Some(message)) => { use AsyncWriteMessage::*; match message { - Data(v) => Poll::Ready(Some(Ok(v))), + Data(v) => Poll::Ready(Some(Ok(v.into()))), Error(exception) => { - let error = anyhow::Error::msg(format!("cxx error: {exception}")); + let error = std::io::Error::new( + io::ErrorKind::Other, + format!("cxx error: {exception}"), + ); Poll::Ready(Some(Err(error))) } Eof => { @@ -160,7 +164,7 @@ impl Stream for AsyncWriteAdapter { } Poll::Ready(None) => { if !self.eof { - Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe).into()))) + Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe)))) } else { Poll::Ready(None) } @@ -208,6 +212,13 @@ mod ffi { include_derivers: bool, ) -> Result>>; + /// Creates a NAR dump from a path. + fn nar_from_path( + self: Pin<&mut CNixStore>, + base_name: Vec, + sender: Box, + ) -> Result<()>; + /// Obtains a handle to the Nix store. fn open_nix_store() -> Result>; diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp index 3914de1..326e878 100644 --- a/src/bindings/nix.cpp +++ b/src/bindings/nix.cpp @@ -108,6 +108,17 @@ std::unique_ptr> CNixStore::compute_fs_closure(RBasePat return std::make_unique>(result); } +void CNixStore::nar_from_path(RVec base_name, RBox sender) { + RustSink sink(std::move(sender)); + + std::string_view sv((const char *)base_name.data(), base_name.size()); + nix::StorePath store_path(sv); + + // exceptions will be thrown into Rust + this->store->narFromPath(store_path, sink); + sink.eof(); +} + std::unique_ptr open_nix_store() { return std::make_unique(); } diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp index 9f4964a..5c79a33 100644 --- a/src/bindings/nix.hpp +++ b/src/bindings/nix.hpp @@ -79,6 +79,7 @@ public: bool flip_direction, bool include_outputs, bool include_derivers); + void nar_from_path(RVec base_name, RBox sender); }; std::unique_ptr open_nix_store(); diff --git a/src/make_nar.rs b/src/make_nar.rs index 19a112b..b5c1ab2 100644 --- a/src/make_nar.rs +++ b/src/make_nar.rs @@ -1,25 +1,20 @@ -use anyhow::{Context, Result}; +use anyhow::Result; use async_compression::{Level, tokio::bufread::ZstdEncoder}; use nix_compat::{ narinfo::{self, NarInfo}, store_path::StorePath, }; use sha2::{Digest, Sha256}; -use std::mem::take; -use tempfile::{NamedTempFile, TempPath}; -use tokio::{ - fs::File, - io::{AsyncRead, BufReader}, - process::Command, -}; +use std::{mem::take, sync::Arc}; +use tokio::io::{AsyncRead, BufReader}; use tokio_util::io::InspectReader; -use tracing::debug; use crate::path_info::PathInfo; +use crate::store::Store; pub struct MakeNar<'a> { path_info: &'a PathInfo, - nar_file: TempPath, + store: Arc, nar_hasher: Sha256, /// hash of compressed nar file file_hasher: Sha256, @@ -28,12 +23,10 @@ pub struct MakeNar<'a> { } impl<'a> MakeNar<'a> { - pub fn new(path_info: &'a PathInfo) -> Result { + pub fn new(path_info: &'a PathInfo, store: Arc) -> Result { Ok(Self { path_info, - nar_file: NamedTempFile::new() - .context("create tempfile for nar")? - .into_temp_path(), + store, nar_hasher: Sha256::new(), file_hasher: Sha256::new(), nar_size: 0, @@ -41,32 +34,12 @@ impl<'a> MakeNar<'a> { }) } - pub async fn make(&self) -> Result<()> { - let path = self.path_info.absolute_path(); - let out = std::fs::File::options() - .write(true) - .truncate(true) - .open(&self.nar_file)?; - - debug!("dumping nar for: {}", path); - Ok(Command::new("nix") - .arg("nar") - .arg("pack") - .arg(self.path_info.absolute_path()) - .kill_on_drop(true) - .stdout(out) - .spawn()? - .wait() - .await? - .exit_ok()?) - } - /// Returns a compressed nar reader which can be uploaded. File hash will be available when /// everything is read - pub async fn compress_and_hash(&mut self) -> Result { - let nar_file = File::open(&self.nar_file).await?; + pub fn compress_and_hash(&mut self) -> Result { + let nar_reader = self.store.nar_from_path(self.path_info.path.clone()); // reader that hashes as nar is read - let nar_reader = InspectReader::new(nar_file, |x| { + let nar_reader = InspectReader::new(nar_reader, |x| { self.nar_size += x.len() as u64; self.nar_hasher.update(x); }); diff --git a/src/push.rs b/src/push.rs index 93a28da..57a43d2 100644 --- a/src/push.rs +++ b/src/push.rs @@ -169,8 +169,9 @@ impl Push { uploads.push(tokio::spawn({ let s3 = self.s3.clone(); + let store = self.store.clone(); async move { - let res = uploader.upload(s3).await; + let res = uploader.upload(s3, store).await; drop(permit); self.upload_count.fetch_add(1, Ordering::Relaxed); res diff --git a/src/store.rs b/src/store.rs index 763cb3d..7589e94 100644 --- a/src/store.rs +++ b/src/store.rs @@ -2,9 +2,13 @@ use std::{ffi::OsStr, os::unix::ffi::OsStrExt, sync::Arc}; use anyhow::{Context, Result}; use nix_compat::store_path::StorePath; -use tokio::task; +use tokio::{io::AsyncRead, task}; +use tokio_util::io::StreamReader; -use crate::{bindings, path_info::PathInfo}; +use crate::{ + bindings::{self, AsyncWriteAdapter}, + path_info::PathInfo, +}; pub struct Store { inner: Arc, @@ -75,4 +79,20 @@ impl Store { .await .unwrap() } + + pub fn nar_from_path(&self, store_path: StorePath) -> impl AsyncRead { + let inner = self.inner.clone(); + let (adapter, mut sender) = AsyncWriteAdapter::new(); + let base_name = store_path.to_string().as_bytes().to_vec(); + + tokio::task::spawn_blocking(move || { + // Send all exceptions through the channel, and ignore errors + // during sending (the channel may have been closed). + if let Err(e) = inner.store().nar_from_path(base_name, sender.clone()) { + let _ = sender.rust_error(e); + } + }); + + StreamReader::new(adapter) + } } diff --git a/src/uploader.rs b/src/uploader.rs index a2c2766..c829a79 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -7,7 +7,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tracing::{debug, trace}; use ulid::Ulid; -use crate::{make_nar::MakeNar, path_info::PathInfo}; +use crate::{make_nar::MakeNar, path_info::PathInfo, store::Store}; const CHUNK_SIZE: usize = 1024 * 1024 * 5; @@ -24,9 +24,8 @@ impl<'a> Uploader<'a> { Ok(Self { signing_key, path }) } - pub async fn upload(&self, s3: Arc) -> Result<()> { - let mut nar = MakeNar::new(&self.path)?; - nar.make().await?; + pub async fn upload(&self, s3: Arc, store: Arc) -> Result<()> { + let mut nar = MakeNar::new(&self.path, store)?; // we don't know what the hash of the compressed file will be so upload to a // temp location for now @@ -35,7 +34,7 @@ impl<'a> Uploader<'a> { debug!("uploading to temp path: {}", temp_path); // compress and upload nar - let mut file_reader = nar.compress_and_hash().await?; + let mut file_reader = nar.compress_and_hash()?; loop { let mut buf = BytesMut::with_capacity(CHUNK_SIZE); let n = file_reader.read_buf(&mut buf).await?; From ca97aebd7ac84b2fbeb88a545c6968795cc186ce Mon Sep 17 00:00:00 2001 From: cy Date: Sat, 26 Apr 2025 23:07:56 -0400 Subject: [PATCH 17/32] limit directories even more --- src/push.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/push.rs b/src/push.rs index 57a43d2..9a9e156 100644 --- a/src/push.rs +++ b/src/push.rs @@ -18,8 +18,6 @@ use url::Url; use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; -const UPLOAD_CONCURRENCY: usize = 5; - pub struct Push { upstream_caches: Vec, store_paths: Arc>>, @@ -105,7 +103,7 @@ impl Push { } pub async fn run(&'static self) -> Result<()> { - let (tx, rx) = mpsc::channel(UPLOAD_CONCURRENCY); + let (tx, rx) = mpsc::channel(1); let filter = tokio::spawn(self.filter_from_upstream(tx)); let upload = tokio::spawn(self.upload(rx)); @@ -155,19 +153,25 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(UPLOAD_CONCURRENCY)); + let permits = Arc::new(Semaphore::new(10)); + let big_permits = Arc::new(Semaphore::new(2)); loop { let permits = permits.clone(); - debug!("upload permits available: {}", permits.available_permits()); - let permit = permits.acquire_owned().await.unwrap(); + let big_permits = big_permits.clone(); if let Some(path_to_upload) = rx.recv().await { - println!("uploading: {}", path_to_upload.absolute_path()); - - let uploader = Uploader::new(&self.signing_key, path_to_upload)?; + let mut permit = permits.acquire_owned().await.unwrap(); uploads.push(tokio::spawn({ + // directory may have many files and end up causing "too many open files" + if PathBuf::from(path_to_upload.absolute_path()).is_dir() { + // drop regular permit and take the big one + permit = big_permits.acquire_owned().await.unwrap(); + } + + println!("uploading: {}", path_to_upload.absolute_path()); + let uploader = Uploader::new(&self.signing_key, path_to_upload)?; let s3 = self.s3.clone(); let store = self.store.clone(); async move { From e5336d304d41d80cb9b07a4d9ffe3d9d480e6cf4 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 27 Apr 2025 01:23:45 -0400 Subject: [PATCH 18/32] improve concurrency control; use nar_size from cpathinfo --- src/bindings/mod.rs | 2 ++ src/bindings/nix.cpp | 4 ++++ src/bindings/nix.hpp | 1 + src/path_info.rs | 3 ++- src/push.rs | 29 ++++++++++++++++++++++------- src/store.rs | 2 ++ 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 8084dff..450bc98 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -228,6 +228,8 @@ mod ffi { /// Mid-level wrapper for the `nix::ValidPathInfo` struct. type CPathInfo; + /// Returns the size of the NAR. + fn nar_size(self: Pin<&mut CPathInfo>) -> u64; /// Returns the references of the store path. fn references(self: Pin<&mut CPathInfo>) -> UniquePtr>; diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp index 326e878..8bf2cb3 100644 --- a/src/bindings/nix.cpp +++ b/src/bindings/nix.cpp @@ -57,6 +57,10 @@ void RustSink::eof() { CPathInfo::CPathInfo(nix::ref pi) : pi(pi) {} +uint64_t CPathInfo::nar_size() { + return this->pi->narSize; +} + std::unique_ptr> CPathInfo::sigs() { std::vector result; for (auto&& elem : this->pi->sigs) { diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp index 5c79a33..87b3ebf 100644 --- a/src/bindings/nix.hpp +++ b/src/bindings/nix.hpp @@ -65,6 +65,7 @@ public: CPathInfo(nix::ref pi); std::unique_ptr> sigs(); std::unique_ptr> references(); + uint64_t nar_size(); }; class CNixStore { diff --git a/src/path_info.rs b/src/path_info.rs index f88e464..beea6b9 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -18,6 +18,7 @@ pub struct PathInfo { pub path: StorePath, pub signatures: Vec, pub references: Vec>, + pub nar_size: u64, } impl PathInfo { @@ -90,7 +91,7 @@ impl PathInfo { let upstream = upstream .join(self.narinfo_path().as_ref()) .expect("adding .narinfo should make a valid url"); - debug!("querying {}", upstream); + trace!("querying {}", upstream); let res_status = reqwest::Client::new() .head(upstream.as_str()) .send() diff --git a/src/push.rs b/src/push.rs index 9a9e156..8943355 100644 --- a/src/push.rs +++ b/src/push.rs @@ -114,18 +114,22 @@ impl Push { /// filter paths that are on upstream and send to `tx` async fn filter_from_upstream(&'static self, tx: mpsc::Sender) { - let mut handles = Vec::with_capacity(10); + let mut handles = Vec::new(); let store_paths = self.store_paths.read().await.clone(); + // limit number of inflight requests + let inflight_permits = Arc::new(Semaphore::new(32)); for path in store_paths.into_iter() { if path.check_upstream_signature(&self.upstream_caches) { debug!("skip {} (signature match)", path.absolute_path()); - self.signature_hit_count.fetch_add(1, Ordering::Release); + self.signature_hit_count.fetch_add(1, Ordering::Relaxed); continue; } handles.push({ let tx = tx.clone(); + let inflight_permits = inflight_permits.clone(); tokio::spawn(async move { + let _permit = inflight_permits.acquire().await.unwrap(); if !path .check_upstream_hit(self.upstream_caches.as_slice()) .await @@ -153,24 +157,35 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(10)); - let big_permits = Arc::new(Semaphore::new(2)); + let permits = Arc::new(Semaphore::new(16)); + let big_permits = Arc::new(Semaphore::new(5)); loop { let permits = permits.clone(); let big_permits = big_permits.clone(); if let Some(path_to_upload) = rx.recv().await { + debug!("upload permits available: {}", permits.available_permits()); let mut permit = permits.acquire_owned().await.unwrap(); uploads.push(tokio::spawn({ - // directory may have many files and end up causing "too many open files" - if PathBuf::from(path_to_upload.absolute_path()).is_dir() { + // a large directory may have many files and end up causing "too many open files" + if PathBuf::from(path_to_upload.absolute_path()).is_dir() + && path_to_upload.nar_size > 5 * 1024 * 1024 + { + debug!( + "upload big permits available: {}", + big_permits.available_permits() + ); // drop regular permit and take the big one permit = big_permits.acquire_owned().await.unwrap(); } - println!("uploading: {}", path_to_upload.absolute_path()); + println!( + "uploading: {} (size: {})", + path_to_upload.absolute_path(), + path_to_upload.nar_size + ); let uploader = Uploader::new(&self.signing_key, path_to_upload)?; let s3 = self.s3.clone(); let store = self.store.clone(); diff --git a/src/store.rs b/src/store.rs index 7589e94..9b925b2 100644 --- a/src/store.rs +++ b/src/store.rs @@ -69,11 +69,13 @@ impl Store { .map(|x| StorePath::from_bytes(x.as_bytes())) .collect::>() .context("get references from pathinfo")?; + let nar_size = c_path_info.pin_mut().nar_size(); Ok(PathInfo { path, signatures, references, + nar_size, }) }) .await From b49be95d09987702c5aba1a2af279014bc332302 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 27 Apr 2025 02:32:15 -0400 Subject: [PATCH 19/32] simplify here since the problem was somewhere else --- Cargo.lock | 16 ++++++++++++++++ Cargo.toml | 1 + src/push.rs | 22 ++++------------------ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3e049c..2a630e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -968,6 +968,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "humantime" version = "2.2.0" @@ -1304,6 +1313,12 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libm" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9627da5196e5d8ed0b0495e61e518847578da83483c37288316d9b2e03a7f72" + [[package]] name = "libmimalloc-sys" version = "0.1.42" @@ -1491,6 +1506,7 @@ dependencies = [ "cxx-build", "ed25519-dalek", "futures", + "humansize", "nix-compat", "object_store", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 6ca052f..272be3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ bytes = "1.10.1" object_store = { version = "0.12.0", features = ["aws"] } ulid = "1.2.1" tracing-subscriber = "0.3.19" +humansize = "2.1.3" [build-dependencies] cxx-build = "1.0" diff --git a/src/push.rs b/src/push.rs index 8943355..cd1373c 100644 --- a/src/push.rs +++ b/src/push.rs @@ -10,6 +10,7 @@ use std::{ use anyhow::{Context, Result}; use futures::future::join_all; +use humansize::{DECIMAL, format_size}; use nix_compat::narinfo::{self, SigningKey}; use object_store::aws::{AmazonS3, AmazonS3Builder}; use tokio::sync::{RwLock, Semaphore, mpsc}; @@ -157,34 +158,19 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(16)); - let big_permits = Arc::new(Semaphore::new(5)); + let permits = Arc::new(Semaphore::new(32)); loop { let permits = permits.clone(); - let big_permits = big_permits.clone(); if let Some(path_to_upload) = rx.recv().await { - debug!("upload permits available: {}", permits.available_permits()); - let mut permit = permits.acquire_owned().await.unwrap(); + let permit = permits.acquire_owned().await.unwrap(); uploads.push(tokio::spawn({ - // a large directory may have many files and end up causing "too many open files" - if PathBuf::from(path_to_upload.absolute_path()).is_dir() - && path_to_upload.nar_size > 5 * 1024 * 1024 - { - debug!( - "upload big permits available: {}", - big_permits.available_permits() - ); - // drop regular permit and take the big one - permit = big_permits.acquire_owned().await.unwrap(); - } - println!( "uploading: {} (size: {})", path_to_upload.absolute_path(), - path_to_upload.nar_size + format_size(path_to_upload.nar_size, DECIMAL) ); let uploader = Uploader::new(&self.signing_key, path_to_upload)?; let s3 = self.s3.clone(); From 01443d0d992cf1134a438076adccce9c96c7686e Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 02:46:11 -0400 Subject: [PATCH 20/32] use 10 permits but skip nars bigger than 15mb --- src/push.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/push.rs b/src/push.rs index cd1373c..c147053 100644 --- a/src/push.rs +++ b/src/push.rs @@ -158,15 +158,20 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(32)); + let permits = Arc::new(Semaphore::new(10)); loop { let permits = permits.clone(); if let Some(path_to_upload) = rx.recv().await { - let permit = permits.acquire_owned().await.unwrap(); - uploads.push(tokio::spawn({ + // large uploads will be concurrently uploaded with multipart anyway so don't spawn + // too much of them + let permit = if path_to_upload.nar_size > 15 * 1024 * 1024 { + Some(permits.acquire_owned().await.unwrap()) + } else { + None + }; println!( "uploading: {} (size: {})", path_to_upload.absolute_path(), From 9d2e9e38bd7582703698f108e17384b0d45b922e Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 03:01:24 -0400 Subject: [PATCH 21/32] split lib.rs and main.rs --- src/lib.rs | 65 ++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 74 +++-------------------------------------------------- 2 files changed, 69 insertions(+), 70 deletions(-) create mode 100644 src/lib.rs diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..0df8b02 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,65 @@ +use std::path::PathBuf; + +use clap::{Args, Parser, Subcommand}; + +mod bindings; +mod cli; +mod make_nar; +mod path_info; +pub mod push; +pub mod store; +mod uploader; + +#[derive(Parser, Debug)] +#[command(version)] +#[command(name = "nixcp")] +#[command(about = "Upload store paths to a s3 binary cache")] +#[command(long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, + + /// Whether to enable tokio-console + #[arg(long)] + pub tokio_console: bool, +} + +#[derive(Debug, Subcommand)] +pub enum Commands { + #[command(arg_required_else_help = true)] + Push(PushArgs), +} + +#[derive(Debug, Args)] +pub struct PushArgs { + /// The s3 bucket to upload to + #[arg(long, value_name = "bucket name")] + bucket: String, + + /// Upstream cache to check against. Can be specified multiple times. + /// cache.nixos.org is always included. + #[arg(long = "upstream", short, value_name = "nixcache.example.com")] + upstreams: Vec, + + /// Path to the file containing signing key + /// e.g. ~/cache-priv-key.pem + #[arg(long)] + signing_key: String, + + /// If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 + #[arg(long)] + region: Option, + + /// If unspecifed, will get it from AWS_ENDPOINT envar + /// e.g. https://s3.example.com + #[arg(long)] + endpoint: Option, + + #[arg(long)] + skip_signature_check: bool, + + /// Path to upload + /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 + #[arg(value_name = "PATH")] + pub paths: Vec, +} diff --git a/src/main.rs b/src/main.rs index 0fefdf5..1afe4b6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,76 +1,10 @@ -#![feature(let_chains)] -#![feature(exit_status_error)] - -use std::path::PathBuf; - use anyhow::{Context, Result}; -use clap::{Args, Parser, Subcommand}; +use clap::Parser; use tracing_subscriber::{EnvFilter, prelude::*}; -use push::Push; -use store::Store; - -mod bindings; -mod cli; -mod make_nar; -mod path_info; -mod push; -mod store; -mod uploader; - -#[derive(Parser, Debug)] -#[command(version)] -#[command(name = "nixcp")] -#[command(about = "Upload store paths to a s3 binary cache")] -#[command(long_about = None)] -struct Cli { - #[command(subcommand)] - command: Commands, - - /// Whether to enable tokio-console - #[arg(long)] - tokio_console: bool, -} - -#[derive(Debug, Subcommand)] -enum Commands { - #[command(arg_required_else_help = true)] - Push(PushArgs), -} - -#[derive(Debug, Args)] -pub struct PushArgs { - /// The s3 bucket to upload to - #[arg(long, value_name = "bucket name")] - bucket: String, - - /// Upstream cache to check against. Can be specified multiple times. - /// cache.nixos.org is always included. - #[arg(long = "upstream", short, value_name = "nixcache.example.com")] - upstreams: Vec, - - /// Path to the file containing signing key - /// e.g. ~/cache-priv-key.pem - #[arg(long)] - signing_key: String, - - /// If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 - #[arg(long)] - region: Option, - - /// If unspecifed, will get it from AWS_ENDPOINT envar - /// e.g. https://s3.example.com - #[arg(long)] - endpoint: Option, - - #[arg(long)] - skip_signature_check: bool, - - /// Path to upload - /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 - #[arg(value_name = "PATH")] - paths: Vec, -} +use nixcp::push::Push; +use nixcp::store::Store; +use nixcp::{Cli, Commands}; #[tokio::main] async fn main() -> Result<()> { From 7285c29e88012e58f5caeca97ba3a33a15f3bb80 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 11:11:46 -0400 Subject: [PATCH 22/32] use io::Error::other --- src/bindings/mod.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 450bc98..46b86dd 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -150,10 +150,7 @@ impl Stream for AsyncWriteAdapter { match message { Data(v) => Poll::Ready(Some(Ok(v.into()))), Error(exception) => { - let error = std::io::Error::new( - io::ErrorKind::Other, - format!("cxx error: {exception}"), - ); + let error = std::io::Error::other(format!("cxx error: {exception}")); Poll::Ready(Some(Err(error))) } Eof => { From 878e09649459062cafa43617cfd34baa1c39854f Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 13:30:49 -0400 Subject: [PATCH 23/32] add path_info tests --- src/lib.rs | 2 +- src/path_info.rs | 8 +++++++- tests/common/mod.rs | 16 ++++++++++++++++ tests/path_info.rs | 46 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 tests/common/mod.rs create mode 100644 tests/path_info.rs diff --git a/src/lib.rs b/src/lib.rs index 0df8b02..df3ba9e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,7 @@ use clap::{Args, Parser, Subcommand}; mod bindings; mod cli; mod make_nar; -mod path_info; +pub mod path_info; pub mod push; pub mod store; mod uploader; diff --git a/src/path_info.rs b/src/path_info.rs index beea6b9..5a70c39 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, anyhow}; use futures::future::join_all; use nix_compat::nixbase32; use nix_compat::store_path::StorePath; @@ -41,6 +41,12 @@ impl PathInfo { let derivation = String::from_utf8_lossy(derivation); debug!("derivation: {derivation}"); + if derivation.is_empty() { + return Err(anyhow!( + "nix path-info did not return a derivation for {path:#?}" + )); + } + let store_path = StorePath::from_absolute_path(derivation.trim().as_bytes()) .context("storepath from derivation")?; store diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..96ba300 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,16 @@ +use nixcp::store::Store; + +pub struct Context { + pub store: Store, +} + +impl Context { + fn new() -> Self { + let store = Store::connect().expect("connect to nix store"); + Self { store } + } +} + +pub fn context() -> Context { + Context::new() +} diff --git a/tests/path_info.rs b/tests/path_info.rs new file mode 100644 index 0000000..2653820 --- /dev/null +++ b/tests/path_info.rs @@ -0,0 +1,46 @@ +use nixcp::path_info::PathInfo; +use std::path::PathBuf; +use std::process::Command; + +mod common; + +const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; +const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; + +#[tokio::test] +async fn path_info_from_package() { + let ctx = common::context(); + let path = PathBuf::from(HELLO); + let path_info = PathInfo::from_path(&path, &ctx.store) + .await + .expect("get pathinfo from package"); + assert_eq!(path_info.path.to_string(), HELLO_DRV); +} + +#[tokio::test] +async fn path_info_from_path() { + // the path must be in the store + Command::new("nix") + .arg("build") + .arg("--no-link") + .arg(HELLO) + .status() + .unwrap(); + let ctx = common::context(); + let path = PathBuf::from("/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"); + let path_info = PathInfo::from_path(&path, &ctx.store) + .await + .expect("get pathinfo from package"); + assert_eq!(path_info.path.to_string(), HELLO_DRV); +} + +#[tokio::test] +async fn closure() { + let ctx = common::context(); + let path = PathBuf::from(HELLO); + let path_info = PathInfo::from_path(&path, &ctx.store) + .await + .expect("get pathinfo from package"); + let closure = path_info.get_closure(&ctx.store).await.unwrap(); + assert_eq!(closure.len(), 466); +} From 54d4c714afe54ea2be3fa12ef55982928f6be487 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 20:50:16 -0400 Subject: [PATCH 24/32] rename from_path to from_derivation --- src/path_info.rs | 24 ++++++++++++++---------- src/push.rs | 2 +- tests/path_info.rs | 6 +++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 5a70c39..0698533 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -22,19 +22,19 @@ pub struct PathInfo { } impl PathInfo { - pub async fn from_path(path: &Path, store: &Store) -> Result { - debug!("query path info for {:?}", path); + pub async fn from_derivation(drv: &Path, store: &Store) -> Result { + debug!("query path info for {:?}", drv); - let derivation = match path.extension() { - Some(ext) if ext == "drv" => path.as_os_str().as_encoded_bytes(), + let derivation = match drv.extension() { + Some(ext) if ext == "drv" => drv.as_os_str().as_encoded_bytes(), _ => { &Command::new("nix") .arg("path-info") .arg("--derivation") - .arg(path) + .arg(drv) .output() .await - .context(format!("run command: nix path-info --derivaiton {path:?}"))? + .context(format!("run command: nix path-info --derivaiton {drv:?}"))? .stdout } }; @@ -43,16 +43,20 @@ impl PathInfo { if derivation.is_empty() { return Err(anyhow!( - "nix path-info did not return a derivation for {path:#?}" + "nix path-info did not return a derivation for {drv:#?}" )); } - let store_path = StorePath::from_absolute_path(derivation.trim().as_bytes()) - .context("storepath from derivation")?; + Self::from_path(derivation.trim(), store).await + } + + pub async fn from_path(path: &str, store: &Store) -> Result { + let store_path = + StorePath::from_absolute_path(path.as_bytes()).context("storepath from path")?; store .query_path_info(store_path) .await - .context("query pathinfo for derivation") + .context("query pathinfo for path") } pub async fn get_closure(&self, store: &Store) -> Result> { diff --git a/src/push.rs b/src/push.rs index c147053..5f7a591 100644 --- a/src/push.rs +++ b/src/push.rs @@ -79,7 +79,7 @@ impl Push { let store = self.store.clone(); futs.push(tokio::spawn(async move { - let path_info = PathInfo::from_path(path.as_path(), &store) + let path_info = PathInfo::from_derivation(path.as_path(), &store) .await .context("get path info for path")?; debug!("path-info for {path:?}: {path_info:?}"); diff --git a/tests/path_info.rs b/tests/path_info.rs index 2653820..59b3dfd 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -11,7 +11,7 @@ const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; async fn path_info_from_package() { let ctx = common::context(); let path = PathBuf::from(HELLO); - let path_info = PathInfo::from_path(&path, &ctx.store) + let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); assert_eq!(path_info.path.to_string(), HELLO_DRV); @@ -28,7 +28,7 @@ async fn path_info_from_path() { .unwrap(); let ctx = common::context(); let path = PathBuf::from("/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"); - let path_info = PathInfo::from_path(&path, &ctx.store) + let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); assert_eq!(path_info.path.to_string(), HELLO_DRV); @@ -38,7 +38,7 @@ async fn path_info_from_path() { async fn closure() { let ctx = common::context(); let path = PathBuf::from(HELLO); - let path_info = PathInfo::from_path(&path, &ctx.store) + let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); let closure = path_info.get_closure(&ctx.store).await.unwrap(); From 09181ae7852fb399a9f575e62ab3a98ba0b4e93d Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 17:37:04 -0400 Subject: [PATCH 25/32] add nar tests --- src/lib.rs | 2 +- src/make_nar.rs | 4 ++-- tests/common/mod.rs | 12 ++++++++++-- tests/nar.rs | 26 ++++++++++++++++++++++++++ tests/path_info.rs | 7 +++---- 5 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 tests/nar.rs diff --git a/src/lib.rs b/src/lib.rs index df3ba9e..dfbab4f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ use clap::{Args, Parser, Subcommand}; mod bindings; mod cli; -mod make_nar; +pub mod make_nar; pub mod path_info; pub mod push; pub mod store; diff --git a/src/make_nar.rs b/src/make_nar.rs index b5c1ab2..97d6b1f 100644 --- a/src/make_nar.rs +++ b/src/make_nar.rs @@ -15,10 +15,10 @@ use crate::store::Store; pub struct MakeNar<'a> { path_info: &'a PathInfo, store: Arc, - nar_hasher: Sha256, + pub nar_hasher: Sha256, /// hash of compressed nar file file_hasher: Sha256, - nar_size: u64, + pub nar_size: u64, file_size: u64, } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 96ba300..aed468f 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,12 +1,20 @@ +#![allow(dead_code)] + +use std::sync::Arc; + use nixcp::store::Store; +pub const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; +pub const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; +pub const HELLO_PATH: &str = "/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"; + pub struct Context { - pub store: Store, + pub store: Arc, } impl Context { fn new() -> Self { - let store = Store::connect().expect("connect to nix store"); + let store = Arc::new(Store::connect().expect("connect to nix store")); Self { store } } } diff --git a/tests/nar.rs b/tests/nar.rs new file mode 100644 index 0000000..5ebadc5 --- /dev/null +++ b/tests/nar.rs @@ -0,0 +1,26 @@ +use crate::common::HELLO_PATH; +use nix_compat::nixbase32; +use nixcp::make_nar::MakeNar; +use nixcp::path_info::PathInfo; +use sha2::Digest; +use tokio::io::AsyncReadExt; + +mod common; + +#[tokio::test] +async fn nar_size_and_hash() { + let ctx = common::context(); + let path_info = PathInfo::from_path(HELLO_PATH, &ctx.store).await.unwrap(); + + let mut nar = MakeNar::new(&path_info, ctx.store).unwrap(); + let mut reader = nar.compress_and_hash().unwrap(); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + drop(reader); + + assert_eq!(nar.nar_size, 234680); + + let nar_hash = nar.nar_hasher.finalize(); + let real_nar_hash = "08za7nnjda8kpdsd73v3mhykjvp0rsmskwsr37winhmzgm6iw79w"; + assert_eq!(nixbase32::encode(nar_hash.as_slice()), real_nar_hash); +} diff --git a/tests/path_info.rs b/tests/path_info.rs index 59b3dfd..2d698d0 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -2,10 +2,9 @@ use nixcp::path_info::PathInfo; use std::path::PathBuf; use std::process::Command; -mod common; +use crate::common::{HELLO, HELLO_DRV, HELLO_PATH}; -const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; -const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; +mod common; #[tokio::test] async fn path_info_from_package() { @@ -27,7 +26,7 @@ async fn path_info_from_path() { .status() .unwrap(); let ctx = common::context(); - let path = PathBuf::from("/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"); + let path = PathBuf::from(HELLO_PATH); let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); From 76cbc85032965cea0cb9188b21900f3d3e683bd7 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 21:39:14 -0400 Subject: [PATCH 26/32] use hashset for closure --- src/path_info.rs | 3 ++- src/push.rs | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/path_info.rs b/src/path_info.rs index 0698533..1e1282d 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -13,7 +13,7 @@ use url::Url; use crate::store::Store; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct PathInfo { pub path: StorePath, pub signatures: Vec, @@ -59,6 +59,7 @@ impl PathInfo { .context("query pathinfo for path") } + // TODO: skip call to query_path_info and return Vec? pub async fn get_closure(&self, store: &Store) -> Result> { let futs = store .compute_fs_closure(self.path.clone()) diff --git a/src/push.rs b/src/push.rs index 5f7a591..bf25ea1 100644 --- a/src/push.rs +++ b/src/push.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashSet, fs, iter::once, path::PathBuf, @@ -21,7 +22,7 @@ use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; pub struct Push { upstream_caches: Vec, - store_paths: Arc>>, + store_paths: Arc>>, signing_key: SigningKey, store: Arc, s3: Arc, @@ -61,7 +62,7 @@ impl Push { Ok(Self { upstream_caches: upstreams, - store_paths: Arc::new(RwLock::new(Vec::new())), + store_paths: Arc::new(RwLock::new(HashSet::new())), signing_key, store: Arc::new(store), s3: Arc::new(s3_builder.build()?), From 05589641cfd59f093a6fd38d30bd34467713f4e4 Mon Sep 17 00:00:00 2001 From: cy Date: Mon, 28 Apr 2025 21:39:14 -0400 Subject: [PATCH 27/32] move "nix build" to tests/common --- tests/common/mod.rs | 8 ++++++++ tests/path_info.rs | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index aed468f..3870a1d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] +use std::process::Command; use std::sync::Arc; use nixcp::store::Store; @@ -14,6 +15,13 @@ pub struct Context { impl Context { fn new() -> Self { + // hello must be in the store + Command::new("nix") + .arg("build") + .arg("--no-link") + .arg(HELLO) + .status() + .unwrap(); let store = Arc::new(Store::connect().expect("connect to nix store")); Self { store } } diff --git a/tests/path_info.rs b/tests/path_info.rs index 2d698d0..90d113d 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -1,6 +1,5 @@ use nixcp::path_info::PathInfo; use std::path::PathBuf; -use std::process::Command; use crate::common::{HELLO, HELLO_DRV, HELLO_PATH}; @@ -18,13 +17,6 @@ async fn path_info_from_package() { #[tokio::test] async fn path_info_from_path() { - // the path must be in the store - Command::new("nix") - .arg("build") - .arg("--no-link") - .arg(HELLO) - .status() - .unwrap(); let ctx = common::context(); let path = PathBuf::from(HELLO_PATH); let path_info = PathInfo::from_derivation(&path, &ctx.store) From a0794b03564c6f045c25cc96c93922a43afea367 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 01:15:10 -0400 Subject: [PATCH 28/32] fix closure test check; rm RUST_LOG envar in devshell --- flake.nix | 1 - tests/path_info.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 01d135a..fb86d40 100644 --- a/flake.nix +++ b/flake.nix @@ -59,7 +59,6 @@ devShells.default = craneLib.devShell { inputsFrom = [ nixcp ]; - RUST_LOG = "nixcp=debug"; RUST_BACKGRACE = 1; # for cpp bindings to work NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; diff --git a/tests/path_info.rs b/tests/path_info.rs index 90d113d..57738fd 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -33,5 +33,5 @@ async fn closure() { .await .expect("get pathinfo from package"); let closure = path_info.get_closure(&ctx.store).await.unwrap(); - assert_eq!(closure.len(), 466); + assert_eq!(closure.len(), 472); } From 0e97d1174503c33a40810dbec0922278867636fe Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 01:48:13 -0400 Subject: [PATCH 29/32] skip integration tests in nix package These tests need a connection to the nix store and we can't have that in the nix build environment. --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index fb86d40..2d1191f 100644 --- a/flake.nix +++ b/flake.nix @@ -48,6 +48,8 @@ ]; # for cpp bindings to work NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; + # skip integration tests (they need a connection to the nix store) + cargoTestExtraArgs = "--bins"; }; cargoArtifacts = craneLib.buildDepsOnly commonArgs; From 14d6e9d29eca6fea2ef32f99ec452b68da208c77 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 01:48:13 -0400 Subject: [PATCH 30/32] path_info: check for and resolve symlink --- Cargo.toml | 10 ++++++---- src/path_info.rs | 8 ++++++++ tests/path_info.rs | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 272be3b..c1de8ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,15 +16,14 @@ futures = "0.3.31" nix-compat = { git = "https://github.com/tvlfyi/tvix.git", version = "0.1.0" } regex = "1.11.1" reqwest = "0.12.15" -serde = { version = "1.0.219", features = [ "derive" ]} +serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" sha2 = "0.10.8" -tokio = { version = "1.44.1", features = [ "full", "tracing", "parking_lot" ]} +tokio = { version = "1.44.1", features = ["full", "tracing", "parking_lot"] } tracing = "0.1.41" -url = { version = "2.5.4", features = [ "serde" ]} +url = { version = "2.5.4", features = ["serde"] } cxx = "1.0" console-subscriber = "0.4.1" -tempfile = "3.19.1" tokio-util = { version = "0.7.15", features = ["io"] } bytes = "1.10.1" object_store = { version = "0.12.0", features = ["aws"] } @@ -35,3 +34,6 @@ humansize = "2.1.3" [build-dependencies] cxx-build = "1.0" pkg-config = "0.3.32" + +[dev-dependencies] +tempfile = "3.19.1" diff --git a/src/path_info.rs b/src/path_info.rs index 1e1282d..213fd1a 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -28,6 +28,14 @@ impl PathInfo { let derivation = match drv.extension() { Some(ext) if ext == "drv" => drv.as_os_str().as_encoded_bytes(), _ => { + let drv = { + // resolve symlink + if drv.is_symlink() { + &drv.canonicalize()? + } else { + drv + } + }; &Command::new("nix") .arg("path-info") .arg("--derivation") diff --git a/tests/path_info.rs b/tests/path_info.rs index 57738fd..0f9543b 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -1,6 +1,8 @@ use nixcp::path_info::PathInfo; use std::path::PathBuf; +use tempfile::TempDir; + use crate::common::{HELLO, HELLO_DRV, HELLO_PATH}; mod common; @@ -25,6 +27,23 @@ async fn path_info_from_path() { assert_eq!(path_info.path.to_string(), HELLO_DRV); } +#[tokio::test] +async fn path_info_symlink() { + let ctx = common::context(); + + let temp_path = TempDir::new().unwrap(); + let link_path = temp_path.path().join("result"); + + // symlink at ./result (like `nix build`) + std::os::unix::fs::symlink(HELLO_PATH, &link_path).unwrap(); + + // should resolve symlink + let path_info = PathInfo::from_derivation(&link_path, &ctx.store) + .await + .expect("get pathinfo from package"); + assert_eq!(path_info.path.to_string(), HELLO_DRV); +} + #[tokio::test] async fn closure() { let ctx = common::context(); From 9b0c6aece69147aeeaf300b1c8db8d7b131226db Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 01:48:13 -0400 Subject: [PATCH 31/32] add .editorconfig --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d17805f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,3 @@ +[*.nix] +indent_size = 2 +indent_stype = space From 3c40776981d6d1a82a537c0fc6c026ef043b3111 Mon Sep 17 00:00:00 2001 From: cy Date: Sun, 4 May 2025 03:06:43 -0400 Subject: [PATCH 32/32] update readme --- README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ff71137..55fdaef 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,13 @@ The signing key is generated with: nix-store --generate-binary-cache-key nixcache.cy7.sh cache-priv-key.pem cache-pub-key.pem ``` -`AWS_ACCESS_KEY_ID` and `AWS_ENDPOINT_URL` environment variables should be set with your s3 credentials. +`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables should be set with your s3 credentials. ``` -Usage: nixcp [OPTIONS] --bucket --signing-key +Usage: nixcp push [OPTIONS] --bucket --signing-key [PATH]... -Commands: - push - help Print this message or the help of the given subcommand(s) +Arguments: + [PATH]... Path to upload e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 Options: --bucket @@ -28,15 +27,13 @@ Options: --signing-key Path to the file containing signing key e.g. ~/cache-priv-key.pem --region - If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default + If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 --endpoint - If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default e.g. https://s3.example.com - --profile - AWS profile to use + If unspecifed, will get it from AWS_ENDPOINT envar e.g. https://s3.example.com + --skip-signature-check + -h, --help Print help - -V, --version - Print version ``` ## Install with nix