Compare commits
14 commits
2025-04-12
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
39792cdd40
|
|||
|
55a097d45c
|
|||
|
20c13a86be
|
|||
|
4895720831
|
|||
|
03530a2afa
|
|||
|
2f7ae745f5
|
|||
|
d4cac65247
|
|||
|
c37a05ba9f
|
|||
|
307109d9f1
|
|||
|
6b959f583b
|
|||
|
df8d3a0bc6
|
|||
|
5111af1d34
|
|||
|
abdff736f5
|
|||
|
a95c094063
|
5 changed files with 118 additions and 89 deletions
84
Cargo.lock
generated
84
Cargo.lock
generated
|
|
@ -84,9 +84,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.97"
|
||||
version = "1.0.98"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
|
||||
checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
|
|
@ -414,7 +414,7 @@ dependencies = [
|
|||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"h2 0.4.8",
|
||||
"h2 0.4.9",
|
||||
"http 0.2.12",
|
||||
"http 1.3.1",
|
||||
"http-body 0.4.6",
|
||||
|
|
@ -425,7 +425,7 @@ dependencies = [
|
|||
"hyper-util",
|
||||
"pin-project-lite",
|
||||
"rustls 0.21.12",
|
||||
"rustls 0.23.25",
|
||||
"rustls 0.23.26",
|
||||
"rustls-native-certs 0.8.1",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
|
|
@ -675,9 +675,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.17"
|
||||
version = "1.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a"
|
||||
checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
|
|
@ -712,9 +712,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.5.34"
|
||||
version = "4.5.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e958897981290da2a852763fe9cdb89cd36977a5d729023127095fa94d95e2ff"
|
||||
checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
|
|
@ -722,9 +722,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.5.34"
|
||||
version = "4.5.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83b0f35019843db2160b5bb19ae09b4e6411ac33fc6a712003c33e03090e2489"
|
||||
checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
|
|
@ -1060,9 +1060,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
|||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.10"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
|
||||
checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
|
|
@ -1298,9 +1298,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.8"
|
||||
version = "0.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2"
|
||||
checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
|
|
@ -1457,7 +1457,7 @@ dependencies = [
|
|||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"h2 0.4.8",
|
||||
"h2 0.4.9",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"httparse",
|
||||
|
|
@ -1494,7 +1494,7 @@ dependencies = [
|
|||
"http 1.3.1",
|
||||
"hyper 1.6.0",
|
||||
"hyper-util",
|
||||
"rustls 0.23.25",
|
||||
"rustls 0.23.26",
|
||||
"rustls-native-certs 0.8.1",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
|
|
@ -1679,9 +1679,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.8.0"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
|
||||
checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
|
|
@ -1748,9 +1748,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.171"
|
||||
version = "0.2.172"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
|
||||
checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
|
|
@ -1780,9 +1780,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
|||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.9.3"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413"
|
||||
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
|
|
@ -1863,9 +1863,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
|||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.5"
|
||||
version = "0.8.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
|
||||
checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
]
|
||||
|
|
@ -2050,9 +2050,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
|||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.71"
|
||||
version = "0.10.72"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd"
|
||||
checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
|
|
@ -2082,9 +2082,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
|
|||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.106"
|
||||
version = "0.9.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd"
|
||||
checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
|
|
@ -2242,9 +2242,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.10"
|
||||
version = "0.5.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
|
||||
checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
|
@ -2310,7 +2310,7 @@ dependencies = [
|
|||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.4.8",
|
||||
"h2 0.4.9",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
|
|
@ -2404,14 +2404,14 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.0.3"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96"
|
||||
checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.9.3",
|
||||
"linux-raw-sys 0.9.4",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
|
|
@ -2429,9 +2429,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.25"
|
||||
version = "0.23.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c"
|
||||
checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"once_cell",
|
||||
|
|
@ -2724,9 +2724,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.14.0"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
|
||||
checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
|
|
@ -2837,7 +2837,7 @@ dependencies = [
|
|||
"fastrand",
|
||||
"getrandom 0.3.2",
|
||||
"once_cell",
|
||||
"rustix 1.0.3",
|
||||
"rustix 1.0.5",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
|
|
@ -2913,9 +2913,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.44.1"
|
||||
version = "1.44.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a"
|
||||
checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"bytes",
|
||||
|
|
@ -2966,7 +2966,7 @@ version = "0.26.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b"
|
||||
dependencies = [
|
||||
"rustls 0.23.25",
|
||||
"rustls 0.23.26",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -44,11 +44,11 @@ pub struct PushArgs {
|
|||
#[arg(long)]
|
||||
signing_key: String,
|
||||
|
||||
/// If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default
|
||||
/// If unspecified, will get it form AWS_DEFAULT_REGION envar
|
||||
#[arg(long)]
|
||||
region: Option<String>,
|
||||
|
||||
/// If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default
|
||||
/// If unspecifed, will get it from AWS_ENDPOINT_URL envar
|
||||
/// e.g. https://s3.example.com
|
||||
#[arg(long)]
|
||||
endpoint: Option<String>,
|
||||
|
|
|
|||
|
|
@ -2,29 +2,37 @@ use std::collections::HashSet;
|
|||
|
||||
use anyhow::{Context, Error, Result};
|
||||
use aws_sdk_s3 as s3;
|
||||
use nix_compat::nixbase32;
|
||||
use nix_compat::store_path::StorePath;
|
||||
use nix_compat::{nixbase32, nixhash::CAHash};
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::Deserialize;
|
||||
use tokio::process::Command;
|
||||
use tracing::{debug, error, trace};
|
||||
use url::Url;
|
||||
|
||||
// nix path-info --derivation --json
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct PathInfo {
|
||||
pub deriver: StorePath<String>,
|
||||
pub deriver: Option<StorePath<String>>,
|
||||
pub path: StorePath<String>,
|
||||
signatures: Vec<String>,
|
||||
signatures: Option<Vec<String>>,
|
||||
pub references: Vec<StorePath<String>>,
|
||||
pub ca: Option<CAHash>,
|
||||
}
|
||||
|
||||
impl PathInfo {
|
||||
/// get PathInfo for a package or a store path
|
||||
// get PathInfo for a package or a store path
|
||||
// we deserialize this as an array of `PathInfo` below
|
||||
pub async fn from_path(path: &str) -> Result<Self> {
|
||||
debug!("query nix path-info for {path}");
|
||||
// use lix cause nix would return a json map instead of an array
|
||||
// json output is not stable and could break in future
|
||||
// TODO figure out a better way
|
||||
let nix_cmd = Command::new("nix")
|
||||
.arg("run")
|
||||
.arg("--experimental-features")
|
||||
.arg("nix-command flakes")
|
||||
.arg("github:nixos/nixpkgs/nixos-unstable#lix")
|
||||
.arg("--")
|
||||
.arg("path-info")
|
||||
.arg("--json")
|
||||
.arg(path)
|
||||
|
|
@ -32,6 +40,11 @@ impl PathInfo {
|
|||
.await
|
||||
.context("`nix path-info` failed for {package}")?;
|
||||
|
||||
trace!(
|
||||
"nix path-info output: {}",
|
||||
String::from_utf8_lossy(&nix_cmd.stdout)
|
||||
);
|
||||
|
||||
// nix path-info returns an array with one element
|
||||
match serde_json::from_slice::<Vec<_>>(&nix_cmd.stdout)
|
||||
.context("parse path info from stdout")
|
||||
|
|
@ -88,13 +101,15 @@ impl PathInfo {
|
|||
}
|
||||
|
||||
fn signees(&self) -> Vec<&str> {
|
||||
let signees: Vec<_> = self
|
||||
.signatures
|
||||
.iter()
|
||||
.filter_map(|signature| Some(signature.split_once(":")?.0))
|
||||
.collect();
|
||||
trace!("signees for {}: {:?}", self.path, signees);
|
||||
signees
|
||||
if let Some(signatures) = self.signatures.as_ref() {
|
||||
let signees: Vec<_> = signatures
|
||||
.iter()
|
||||
.filter_map(|signature| Some(signature.split_once(":")?.0))
|
||||
.collect();
|
||||
trace!("signees for {}: {:?}", self.path, signees);
|
||||
return signees;
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool {
|
||||
|
|
@ -144,10 +159,10 @@ mod tests {
|
|||
let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#;
|
||||
let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize");
|
||||
|
||||
path_info.signatures = vec![
|
||||
path_info.signatures = Some(vec![
|
||||
"cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(),
|
||||
"nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(),
|
||||
];
|
||||
]);
|
||||
let signees = path_info.signees();
|
||||
assert_eq!(signees, vec!["cache.nixos.org-1", "nixcache.cy7.sh"]);
|
||||
}
|
||||
|
|
@ -157,29 +172,46 @@ mod tests {
|
|||
let path_info_json = r#"{"deriver":"/nix/store/idy9slp6835nm6x2i41vzm4g1kai1m2p-nixcp-0.1.0.drv.drv","narHash":"sha256-BG5iQEKKOM7d4199942ReE+bZxQDGDuOZqQ5jkTp45o=","narSize":27851376,"path":"/nix/store/giv6gcnv0ymqgi60dx0fsk2l1pxdd1n0-nixcp-0.1.0","references":["/nix/store/954l60hahqvr0hbs7ww6lmgkxvk8akdf-openssl-3.4.1","/nix/store/ik84lbv5jvjm1xxvdl8mhg52ry3xycvm-gcc-14-20241116-lib","/nix/store/rmy663w9p7xb202rcln4jjzmvivznmz8-glibc-2.40-66"],"registrationTime":1744643248,"signatures":["nixcache.cy7.sh:n1lnCoT16xHcuV+tc+/TbZ2m+UKuI15ok+3cg2i5yFHO8+QVUn0x+tOSy6bZ+KxWl4PvmIjUQN1Kus0efn46Cw=="],"valid":true}"#;
|
||||
let mut path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize");
|
||||
|
||||
path_info.signatures = vec![
|
||||
path_info.signatures = Some(vec![
|
||||
"cache.nixos.org-1:sRAGxSFkQ6PGzPGs9caX6y81tqfevIemSSWZjeD7/v1X0J9kEeafaFgz+zBD/0k8imHSWi/leCoIXSCG6/MrCw==".to_string(),
|
||||
"nixcache.cy7.sh:hV1VQvztp8UY7hq/G22uzC3vQp4syBtnpJh21I1CRJykqweohb4mdS3enyi+9xXqAUZMfNrZuRFSySqa5WK1Dg==".to_string(),
|
||||
"nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=".to_string(),
|
||||
];
|
||||
assert_eq!(
|
||||
path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()]),
|
||||
true
|
||||
]);
|
||||
assert!(
|
||||
path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()])
|
||||
);
|
||||
assert_eq!(
|
||||
path_info.check_upstream_signature(&[Url::parse("https://nixcache.cy7.sh").unwrap()]),
|
||||
true
|
||||
assert!(
|
||||
path_info.check_upstream_signature(&[Url::parse("https://nixcache.cy7.sh").unwrap()])
|
||||
);
|
||||
assert_eq!(
|
||||
assert!(
|
||||
path_info.check_upstream_signature(&[
|
||||
Url::parse("https://nix-community.cachix.org").unwrap()
|
||||
]),
|
||||
true
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
path_info
|
||||
assert!(
|
||||
!path_info
|
||||
.check_upstream_signature(&[Url::parse("https://fake-cache.cachix.org").unwrap()]),
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_info_without_signature() {
|
||||
let path_info_json = r#"{"ca":"fixed:r:sha256:1q10p04pgx9sk6xbvrkn4nvh0ys2lzplgcni5368f4z3cr8ikbmz","narHash":"sha256-v64ZUWbjE4fMKNGyR++nQnsAtyV25r26mTr1dwm4IOA=","narSize":5520,"path":"/nix/store/gj6hz9mj23v01yvq1nn5f655jrcky1qq-nixos-option.nix","references":[],"registrationTime":1744740942,"valid":true}"#;
|
||||
let path_info: PathInfo = serde_json::from_str(path_info_json).expect("must serialize");
|
||||
|
||||
assert!(
|
||||
!path_info.check_upstream_signature(&[Url::parse("https://cache.nixos.org").unwrap()])
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
#[test]
|
||||
fn path_info_deserialize_nix_map() {
|
||||
let path_info_json = r#"{"/nix/store/8vm1jxsc0jphd65vb7r6g5ysgqw0yh9f-home-manager-generation":{"ca":null,"deriver":"/nix/store/h8z25s6arcrns5nmrq1yhgbamywjivpn-home-manager-generation.drv","narHash":"sha256-o4qwqyJ5UVm9cyC/nBNcNYVnIM14Pewgw7fou+wUVSY=","narSize":13608,"references":["/nix/store/40yifhx34v4g4llrdn3v2ag8w02j10fv-gnugrep-3.11","/nix/store/4d0ix5djms3n2jnjdc58l916cwack1rp-empty-directory","/nix/store/56zmgla8443qfrkrh2ch0vz0mh8ywrw1-home-manager-files","/nix/store/58br4vk3q5akf4g8lx0pqzfhn47k3j8d-bash-5.2p37","/nix/store/80l1sb3vcmrkcdd7ihlizkcnv19rq9fj-ncurses-6.5","/nix/store/8vm1jxsc0jphd65vb7r6g5ysgqw0yh9f-home-manager-generation","/nix/store/92as847i10kl6s19fi910ddyk9l83835-check-link-targets.sh","/nix/store/9c90iz95yynyh3vsc67zndch6j01vgz3-home-manager-path","/nix/store/b2cfj7yk3wfg1jdwjzim7306hvsc5gnl-systemd-257.3","/nix/store/bm5fi6wj0w4r2wjll2448k307bzfcjwx-cleanup","/nix/store/c244fsb3a7i5837lzn94m4bmav9i5p9b-link","/nix/store/cvlbhhrvzfkjl2hrrzhq3vr5gzan1r60-bash-interactive-5.2p37","/nix/store/gpxsdrrd4x93fs75395vr2dfys1ki9mq-jq-1.7.1-bin","/nix/store/jlf743lqxbvad6dbgndsgqfg20m2np5i-sd-switch-0.5.3","/nix/store/mhmgm739aagj4x7hr6ag2wjmxhmpy8mf-gettext-0.22.5","/nix/store/w9db12j05yv5hl31s6jndd9cfm1g1gw4-hm-modules-messages","/nix/store/wj1c3gsiajabnq50ifxqnlv60i5rhqj7-diffutils-3.10","/nix/store/xhql0ilzbiqwnmz4z8y0phk611wynxf2-gnused-4.9","/nix/store/xq5f95pp297afc2xjgrmhmf9w631qp7m-findutils-4.10.0","/nix/store/yh6qg1nsi5h2xblcr67030pz58fsaxx3-coreutils-9.6","/nix/store/zhrjg6wxrxmdlpn6iapzpp2z2vylpvw5-home-manager.sh"],"registrationTime":1744742989,"signatures":["nixcache.cy7.sh:Vq4X95kSzum7BwrBhjmmM2yVipfBI3AE3jgZ3b3RoYrP4/ghotbDdlwCvwK3qx4BQdEOLSgrC1tDwiMNb6oRBw=="],"ultimate":false}}"#;
|
||||
serde_json::from_str::<HashMap<String, PathInfo>>(path_info_json).expect("must serialize");
|
||||
|
||||
let path_info_json = r#"{"/nix/store/3a2ahdaprw6df0lml1pj9jhbi038dsjh-nixos-system-chunk-25.05.20250412.2631b0b":{"ca":null,"deriver":"/nix/store/12ssi931481jlkizgfk1c1jnawvwjbhh-nixos-system-chunk-25.05.20250412.2631b0b.drv","narHash":"sha256-CHhBIzMD4v/FKqKgGroq0UC1k3GrK5lcNwQPMpv2xLc=","narSize":20704,"references":["/nix/store/0yjiyixxsr137iw93hnaacdsssy1li9h-switch-to-configuration-0.1.0","/nix/store/14rby7cpwrzjsjym44cl5h6nj6qpn1gs-etc","/nix/store/3a2ahdaprw6df0lml1pj9jhbi038dsjh-nixos-system-chunk-25.05.20250412.2631b0b","/nix/store/3wjljpj30fvv2cdb60apr4126pa5bm87-shadow-4.17.2","/nix/store/40yifhx34v4g4llrdn3v2ag8w02j10fv-gnugrep-3.11","/nix/store/58br4vk3q5akf4g8lx0pqzfhn47k3j8d-bash-5.2p37","/nix/store/5dyh8l59kfvf89zjkbmjfnx7fix93n4f-net-tools-2.10","/nix/store/aq9wdsz12bg9252790l9awiry2bml4ls-sops-install-secrets-0.0.1","/nix/store/b00kq6fjhgisdrykg621vml8505nnmb3-users-groups.json","/nix/store/b2cfj7yk3wfg1jdwjzim7306hvsc5gnl-systemd-257.3","/nix/store/bfr68wi6k8icb3j9fy3fzchva56djfhd-mounts.sh","/nix/store/cjnihsds5hhnji9r85hglph07q9y9hgc-system-path","/nix/store/cvlbhhrvzfkjl2hrrzhq3vr5gzan1r60-bash-interactive-5.2p37","/nix/store/f9jll96j74f5ykvs062718b98lfjbn9g-util-linux-2.40.4-bin","/nix/store/h7zih134d3n5yk8pnhv1fa38n6qkyrn2-pre-switch-checks","/nix/store/idn5n51246piyxcr3v6gxnj5a5l9mzpn-linux-6.14.2","/nix/store/ipn5793y61x2904xqnkgbjnp91svjjzx-perl-5.40.0-env","/nix/store/j1rikvl25pz0b5ham1ijq0nbg1q2fqfy-initrd-linux-6.14.2","/nix/store/jgawnqyh6piwcl79gxpmq5czx9rfr9xh-glibc-locales-2.40-66","/nix/store/jqgmcv8j4gj59218hcbiyn8z951rycdj-install-grub.sh","/nix/store/kpmybhxy3gz6k1znbdirwsp3c6wvsgg9-manifest.json","/nix/store/lgainx4gl6q7mhiwmls81d3n51p5jz7z-linux-6.14.2-modules","/nix/store/mhxn5kwnri3z9hdzi3x0980id65p0icn-lib.sh","/nix/store/n8n0faszqlnf3mdg0fj6abnknrhjsw5j-perl-5.40.0-env","/nix/store/nq61v7a601gjndijq5nndprkzpwz4q9g-glibc-2.40-66-bin","/nix/store/nx27idxpvi3fk3p7admvhipny73nr25n-kmod-31","/nix/store/pggww1d2pg24fcg5v36xn63n53vanyyi-gnupg-2.4.7","/nix/store/rg5rf512szdxmnj9qal3wfdnpfsx38qi-setup-etc.pl","/nix/store/vvlfaafnz3pdhw7lx5kc5gb9pl4zhz5l-local-cmds","/nix/store/w142vx7ij1fz6qwhp5dprkf59cizvv1v-update-users-groups.pl","/nix/store/xq5f95pp297afc2xjgrmhmf9w631qp7m-findutils-4.10.0","/nix/store/yh6qg1nsi5h2xblcr67030pz58fsaxx3-coreutils-9.6","/nix/store/zlsmh0ccgvncg30qb4y0mp5pahnk1wnw-append-initrd-secrets","/nix/store/zs07icpv5ykf8m36xcv717hh26bp09fa-firmware","/nix/store/zy2n4id5gcxcbx2x8jbblkmcpdlpsypk-getent-glibc-2.40-66"],"registrationTime":1744743136,"signatures":["nixcache.cy7.sh:dZ1XiKQNe0fRX48gBj03PIABYJGV6BPwb72YpMqEBONZMF+JrkVKhRCF0ur/4Bf5prHxg6Qfg1ytP/4csRC9DQ=="],"ultimate":false}}"#;
|
||||
serde_json::from_str::<HashMap<String, PathInfo>>(path_info_json).expect("must serialize");
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
|
|
|||
21
src/push.rs
21
src/push.rs
|
|
@ -12,7 +12,7 @@ use aws_config::Region;
|
|||
use aws_sdk_s3 as s3;
|
||||
use futures::future::join_all;
|
||||
use nix_compat::narinfo::{self, SigningKey};
|
||||
use tokio::sync::{RwLock, Semaphore, mpsc};
|
||||
use tokio::sync::{RwLock, mpsc};
|
||||
use tracing::{debug, info, trace};
|
||||
use url::Url;
|
||||
|
||||
|
|
@ -30,6 +30,8 @@ pub struct Push {
|
|||
upstream_hit_count: AtomicUsize,
|
||||
// paths that we skipped cause they are already on our cache
|
||||
already_exists_count: AtomicUsize,
|
||||
// paths that we uploaded
|
||||
upload_count: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Push {
|
||||
|
|
@ -68,6 +70,7 @@ impl Push {
|
|||
signature_hit_count: AtomicUsize::new(0),
|
||||
upstream_hit_count: AtomicUsize::new(0),
|
||||
already_exists_count: AtomicUsize::new(0),
|
||||
upload_count: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -98,7 +101,6 @@ impl Push {
|
|||
|
||||
/// filter paths that are on upstream and send to `tx`
|
||||
async fn filter_from_upstream(&'static self, tx: mpsc::Sender<PathInfo>) {
|
||||
let permits = Arc::new(Semaphore::new(10));
|
||||
let mut handles = Vec::with_capacity(10);
|
||||
let store_paths = self.store_paths.read().await.clone();
|
||||
|
||||
|
|
@ -109,11 +111,8 @@ impl Push {
|
|||
continue;
|
||||
}
|
||||
handles.push({
|
||||
let permits = permits.clone();
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let _permit = permits.acquire().await.unwrap();
|
||||
|
||||
if !path
|
||||
.check_upstream_hit(self.upstream_caches.as_slice())
|
||||
.await
|
||||
|
|
@ -143,16 +142,13 @@ impl Push {
|
|||
}
|
||||
|
||||
async fn upload(&'static self, mut rx: mpsc::Receiver<PathInfo>) -> Result<()> {
|
||||
let upload_count = AtomicUsize::new(0);
|
||||
let permits = Arc::new(Semaphore::new(10));
|
||||
let mut uploads = Vec::with_capacity(10);
|
||||
|
||||
loop {
|
||||
if let Some(path_to_upload) = rx.recv().await {
|
||||
let permits = Arc::clone(&permits);
|
||||
let absolute_path = path_to_upload.absolute_path();
|
||||
|
||||
info!("uploading: {}", absolute_path);
|
||||
println!("uploading: {}", absolute_path);
|
||||
let uploader = Uploader::new(
|
||||
&self.signing_key,
|
||||
path_to_upload,
|
||||
|
|
@ -161,8 +157,9 @@ impl Push {
|
|||
)?;
|
||||
|
||||
uploads.push(tokio::spawn(async move {
|
||||
let _permit = permits.acquire().await.unwrap();
|
||||
uploader.upload().await
|
||||
let res = uploader.upload().await;
|
||||
self.upload_count.fetch_add(1, Ordering::Relaxed);
|
||||
res
|
||||
}));
|
||||
} else {
|
||||
join_all(uploads)
|
||||
|
|
@ -171,7 +168,7 @@ impl Push {
|
|||
.flatten()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
println!("uploaded: {}", upload_count.load(Ordering::Relaxed));
|
||||
println!("uploaded: {}", self.upload_count.load(Ordering::Relaxed));
|
||||
println!(
|
||||
"skipped because of signature match: {}",
|
||||
self.signature_hit_count.load(Ordering::Relaxed)
|
||||
|
|
|
|||
|
|
@ -157,9 +157,9 @@ impl<'a> Uploader<'a> {
|
|||
nar_size: nar.len() as u64,
|
||||
references: self.path.references.iter().map(StorePath::as_ref).collect(),
|
||||
signatures: Vec::new(),
|
||||
ca: self.path.ca.clone(),
|
||||
ca: None,
|
||||
system: None,
|
||||
deriver: Some(self.path.deriver.as_ref()),
|
||||
deriver: self.path.deriver.as_ref().map(|x| x.as_ref()),
|
||||
compression: Some("zstd"),
|
||||
file_hash: None,
|
||||
file_size: None,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue