diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index d17805f..0000000 --- a/.editorconfig +++ /dev/null @@ -1,3 +0,0 @@ -[*.nix] -indent_size = 2 -indent_stype = space diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index a4560bf..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: build -on: - workflow_dispatch: - push: - pull_request: - -env: - TERM: ansi - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }} - AWS_ENDPOINT: https://s3.cy7.sh - -jobs: - build-packages: - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - - ubuntu-24.04-arm - - macos-latest # arm64 - - macos-13 # x86 - - runs-on: ${{ matrix.os }} - - steps: - - name: setup binary cache key - run: echo -n "${{ secrets.NIX_CACHE_SECRET_KEY }}" | xxd -p -r > ${{ runner.temp }}/cache-priv-key.pem - - - name: Install Nix - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 - with: - enable_kvm: true - extra_nix_config: | - show-trace = true - experimental-features = nix-command flakes - secret-key-files = ${{ runner.temp }}/cache-priv-key.pem - extra-substituters = https://nixcache.cy7.sh - extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 - with: - persist-credentials: false - - - name: cache devshell - run: | - nix build .#devShells.$(nix eval --impure --raw --expr 'builtins.currentSystem').default - nix run \ - github:cything/nixcp -- push \ - --bucket nixcache \ - --signing-key ${{ runner.temp }}/cache-priv-key.pem \ - result - - - name: build - run: nix build -L . - - - name: cache - run: | - nix run \ - github:cything/nixcp -- push \ - --bucket nixcache \ - --signing-key ${{ runner.temp }}/cache-priv-key.pem \ - result - - - name: prepare tarball to upload - run: nix run github:nixos/nixpkgs#gnutar hcvf result.tar result - - - name: upload result - uses: actions/upload-artifact@6027e3dd177782cd8ab9af838c04fd81a07f1d47 - with: - name: ${{ matrix.os }}.tar - path: result.tar - if-no-files-found: error diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml deleted file mode 100644 index 0b9ac66..0000000 --- a/.github/workflows/check.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: check -on: - workflow_dispatch: - push: - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Install Nix - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 - with: - enable_kvm: true - extra_nix_config: | - show-trace = true - experimental-features = nix-command flakes - extra-substituters = https://nixcache.cy7.sh - extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 - with: - persist-credentials: false - - - name: Run checks - run: nix flake check -L diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index cc7fabc..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: test -on: - workflow_dispatch: - push: - pull_request: - -env: - CARGO_TERM_COLOR: always - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Install Nix - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 - with: - enable_kvm: true - extra_nix_config: | - show-trace = true - experimental-features = nix-command flakes - extra-substituters = https://nixcache.cy7.sh - extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 - with: - persist-credentials: false - - - name: Run tests - run: nix develop -c cargo test --verbose diff --git a/Cargo.toml b/Cargo.toml index c1de8ac..272be3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,14 +16,15 @@ futures = "0.3.31" nix-compat = { git = "https://github.com/tvlfyi/tvix.git", version = "0.1.0" } regex = "1.11.1" reqwest = "0.12.15" -serde = { version = "1.0.219", features = ["derive"] } +serde = { version = "1.0.219", features = [ "derive" ]} serde_json = "1.0.140" sha2 = "0.10.8" -tokio = { version = "1.44.1", features = ["full", "tracing", "parking_lot"] } +tokio = { version = "1.44.1", features = [ "full", "tracing", "parking_lot" ]} tracing = "0.1.41" -url = { version = "2.5.4", features = ["serde"] } +url = { version = "2.5.4", features = [ "serde" ]} cxx = "1.0" console-subscriber = "0.4.1" +tempfile = "3.19.1" tokio-util = { version = "0.7.15", features = ["io"] } bytes = "1.10.1" object_store = { version = "0.12.0", features = ["aws"] } @@ -34,6 +35,3 @@ humansize = "2.1.3" [build-dependencies] cxx-build = "1.0" pkg-config = "0.3.32" - -[dev-dependencies] -tempfile = "3.19.1" diff --git a/LICENSE b/LICENSE deleted file mode 100644 index e886c24..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Cy - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/README.md b/README.md index f9317c8..ff71137 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,14 @@ The signing key is generated with: nix-store --generate-binary-cache-key nixcache.cy7.sh cache-priv-key.pem cache-pub-key.pem ``` -`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables should be set with your s3 credentials. +`AWS_ACCESS_KEY_ID` and `AWS_ENDPOINT_URL` environment variables should be set with your s3 credentials. ``` -Usage: nixcp push [OPTIONS] --bucket --signing-key [PATH]... +Usage: nixcp [OPTIONS] --bucket --signing-key -Arguments: - [PATH]... Path to upload e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 +Commands: + push + help Print this message or the help of the given subcommand(s) Options: --bucket @@ -27,13 +28,15 @@ Options: --signing-key Path to the file containing signing key e.g. ~/cache-priv-key.pem --region - If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 + If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default --endpoint - If unspecifed, will get it from AWS_ENDPOINT envar e.g. https://s3.example.com - --no-default-upstream - Do not include cache.nixos.org as upstream + If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default e.g. https://s3.example.com + --profile + AWS profile to use -h, --help Print help + -V, --version + Print version ``` ## Install with nix diff --git a/flake.nix b/flake.nix index 16b57e6..fb86d40 100644 --- a/flake.nix +++ b/flake.nix @@ -11,15 +11,8 @@ }; }; - outputs = - inputs@{ - nixpkgs, - flake-utils, - crane, - ... - }: - flake-utils.lib.eachDefaultSystem ( - system: + outputs = inputs@{ nixpkgs, flake-utils, crane, ... }: + flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; @@ -28,12 +21,13 @@ ]; }; toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; - craneLib = (crane.mkLib pkgs).overrideToolchain (_: toolchain); + craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); lib = pkgs.lib; # don't clean cpp files cppFilter = path: _type: builtins.match ".*(cpp|hpp)$" path != null; - cppOrCargo = path: type: (cppFilter path type) || (craneLib.filterCargoSources path type); + cppOrCargo = path: type: + (cppFilter path type) || (craneLib.filterCargoSources path type); src = lib.cleanSourceWith { src = ./.; filter = cppOrCargo; @@ -54,38 +48,14 @@ ]; # for cpp bindings to work NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; - # skip integration tests (they need a connection to the nix store) - cargoTestExtraArgs = "--bins"; }; cargoArtifacts = craneLib.buildDepsOnly commonArgs; - nixcp = craneLib.buildPackage ( - commonArgs - // { - inherit cargoArtifacts; - } - ); + nixcp = craneLib.buildPackage (commonArgs // { + inherit cargoArtifacts; + }); in { - checks = { - # clippy with all warnings denied - clippy = craneLib.cargoClippy ( - commonArgs - // { - inherit cargoArtifacts; - cargoClippyExtraArgs = "--all-targets -- --deny warnings"; - } - ); - - # check formatting - cargoFmt = craneLib.cargoFmt { - inherit src; - }; - tomlFmt = craneLib.taploFmt { - src = lib.sources.sourceFilesBySuffices src [ ".toml" ]; - }; - }; - devShells.default = craneLib.devShell { inputsFrom = [ nixcp ]; @@ -96,12 +66,9 @@ packages = with pkgs; [ tokio-console cargo-udeps - cargo-audit ]; }; - formatter = pkgs.nixfmt-rfc-style; - packages.default = nixcp; } ); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c96aa24..eceaf24 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,9 @@ [toolchain] channel = "nightly" profile = "minimal" -components = ["rust-src", "rust-analyzer", "rustfmt", "clippy"] +components = [ + "rust-src", + "rust-analyzer", + "rustfmt", + "clippy", +] \ No newline at end of file diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..139597f --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,2 @@ + + diff --git a/src/lib.rs b/src/lib.rs index 8b1fc18..dfbab4f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use clap::{Args, Parser, Subcommand}; mod bindings; +mod cli; pub mod make_nar; pub mod path_info; pub mod push; @@ -54,9 +55,8 @@ pub struct PushArgs { #[arg(long)] endpoint: Option, - /// Do not include cache.nixos.org as upstream #[arg(long)] - no_default_upstream: bool, + skip_signature_check: bool, /// Path to upload /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 diff --git a/src/path_info.rs b/src/path_info.rs index 213fd1a..1e1282d 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -28,14 +28,6 @@ impl PathInfo { let derivation = match drv.extension() { Some(ext) if ext == "drv" => drv.as_os_str().as_encoded_bytes(), _ => { - let drv = { - // resolve symlink - if drv.is_symlink() { - &drv.canonicalize()? - } else { - drv - } - }; &Command::new("nix") .arg("path-info") .arg("--derivation") diff --git a/src/push.rs b/src/push.rs index 9fc043d..bf25ea1 100644 --- a/src/push.rs +++ b/src/push.rs @@ -1,6 +1,7 @@ use std::{ collections::HashSet, fs, + iter::once, path::PathBuf, sync::{ Arc, @@ -38,13 +39,11 @@ pub struct Push { impl Push { pub async fn new(cli: &PushArgs, store: Store) -> Result { let mut upstreams = Vec::with_capacity(cli.upstreams.len() + 1); - if !cli.no_default_upstream { - upstreams.push( - Url::parse("https://cache.nixos.org") - .expect("default upstream must be a valid url"), - ); - } - for upstream in &cli.upstreams { + for upstream in cli + .upstreams + .iter() + .chain(once(&"https://cache.nixos.org".to_string())) + { upstreams .push(Url::parse(upstream).context(format!("failed to parse {upstream} as url"))?); } @@ -133,7 +132,10 @@ impl Push { let inflight_permits = inflight_permits.clone(); tokio::spawn(async move { let _permit = inflight_permits.acquire().await.unwrap(); - if !path.check_upstream_hit(&self.upstream_caches).await { + if !path + .check_upstream_hit(self.upstream_caches.as_slice()) + .await + { if path.check_if_already_exists(&self.s3).await { debug!("skip {} (already exists)", path.absolute_path()); self.already_exists_count.fetch_add(1, Ordering::Relaxed); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 4c2d932..3870a1d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,10 +6,8 @@ use std::sync::Arc; use nixcp::store::Store; pub const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; -pub const HELLO_DRV: &str = "/nix/store/iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; +pub const HELLO_DRV: &str = "iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; pub const HELLO_PATH: &str = "/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"; -pub const NIXCP_PKG: &str = "github:cything/nixcp?ref=6cfe67af0e8da502702b31f34a941753e64d9561"; -pub const NIXCP_DRV: &str = "/nix/store/ldjvf9qjp980dyvka2hj99q4c0w6901x-nixcp-0.1.0.drv"; pub struct Context { pub store: Arc, @@ -18,7 +16,12 @@ pub struct Context { impl Context { fn new() -> Self { // hello must be in the store - ensure_exists(HELLO); + Command::new("nix") + .arg("build") + .arg("--no-link") + .arg(HELLO) + .status() + .unwrap(); let store = Arc::new(Store::connect().expect("connect to nix store")); Self { store } } @@ -27,12 +30,3 @@ impl Context { pub fn context() -> Context { Context::new() } - -pub fn ensure_exists(pkg: &str) { - Command::new("nix") - .arg("build") - .arg("--no-link") - .arg(pkg) - .status() - .unwrap(); -} diff --git a/tests/path_info.rs b/tests/path_info.rs index d71f9d6..57738fd 100644 --- a/tests/path_info.rs +++ b/tests/path_info.rs @@ -1,9 +1,7 @@ use nixcp::path_info::PathInfo; -use std::{collections::HashSet, path::PathBuf, process::Command}; +use std::path::PathBuf; -use tempfile::TempDir; - -use crate::common::{HELLO, HELLO_DRV, HELLO_PATH, NIXCP_DRV, NIXCP_PKG}; +use crate::common::{HELLO, HELLO_DRV, HELLO_PATH}; mod common; @@ -14,7 +12,7 @@ async fn path_info_from_package() { let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); + assert_eq!(path_info.path.to_string(), HELLO_DRV); } #[tokio::test] @@ -24,74 +22,16 @@ async fn path_info_from_path() { let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); + assert_eq!(path_info.path.to_string(), HELLO_DRV); } #[tokio::test] -async fn path_info_symlink() { - let ctx = common::context(); - - let temp_path = TempDir::new().unwrap(); - let link_path = temp_path.path().join("result"); - - // symlink at ./result (like `nix build`) - std::os::unix::fs::symlink(HELLO_PATH, &link_path).unwrap(); - - // should resolve symlink - let path_info = PathInfo::from_derivation(&link_path, &ctx.store) - .await - .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); -} - -#[tokio::test] -async fn closure_includes_nix_store_requisites() { +async fn closure() { let ctx = common::context(); let path = PathBuf::from(HELLO); let path_info = PathInfo::from_derivation(&path, &ctx.store) .await .expect("get pathinfo from package"); - - // get what we think is the closure - let mut closure: HashSet = path_info - .get_closure(&ctx.store) - .await - .unwrap() - .iter() - .map(|x| x.path.to_absolute_path()) - .collect(); - - // for a somewhat more complicated case - common::ensure_exists(NIXCP_PKG); - let path = PathBuf::from(NIXCP_PKG); - let path_info = PathInfo::from_derivation(&path, &ctx.store) - .await - .expect("get pathinfo from package"); - closure.extend( - path_info - .get_closure(&ctx.store) - .await - .unwrap() - .iter() - .map(|x| x.path.to_absolute_path()), - ); - - // get output of `nix-store --query --requisites --include-outputs` - let nix_store_out = Command::new("nix-store") - .arg("--query") - .arg("--requisites") - .arg("--include-outputs") - .arg(HELLO_DRV) - .arg(NIXCP_DRV) - .output() - .unwrap() - .stdout; - assert!(!nix_store_out.is_empty()); - let ref_closure = String::from_utf8_lossy(&nix_store_out); - let ref_closure = ref_closure.split_whitespace(); - - // check that we didn't miss anything nix-store would catch - for path in ref_closure { - assert!(closure.contains(path)); - } + let closure = path_info.get_closure(&ctx.store).await.unwrap(); + assert_eq!(closure.len(), 472); }