diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index bff29e6..0000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build] -rustflags = ["--cfg", "tokio_unstable"] diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index d17805f..0000000 --- a/.editorconfig +++ /dev/null @@ -1,3 +0,0 @@ -[*.nix] -indent_size = 2 -indent_stype = space diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index a4560bf..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: build -on: - workflow_dispatch: - push: - pull_request: - -env: - TERM: ansi - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }} - AWS_ENDPOINT: https://s3.cy7.sh - -jobs: - build-packages: - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - - ubuntu-24.04-arm - - macos-latest # arm64 - - macos-13 # x86 - - runs-on: ${{ matrix.os }} - - steps: - - name: setup binary cache key - run: echo -n "${{ secrets.NIX_CACHE_SECRET_KEY }}" | xxd -p -r > ${{ runner.temp }}/cache-priv-key.pem - - - name: Install Nix - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 - with: - enable_kvm: true - extra_nix_config: | - show-trace = true - experimental-features = nix-command flakes - secret-key-files = ${{ runner.temp }}/cache-priv-key.pem - extra-substituters = https://nixcache.cy7.sh - extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 - with: - persist-credentials: false - - - name: cache devshell - run: | - nix build .#devShells.$(nix eval --impure --raw --expr 'builtins.currentSystem').default - nix run \ - github:cything/nixcp -- push \ - --bucket nixcache \ - --signing-key ${{ runner.temp }}/cache-priv-key.pem \ - result - - - name: build - run: nix build -L . - - - name: cache - run: | - nix run \ - github:cything/nixcp -- push \ - --bucket nixcache \ - --signing-key ${{ runner.temp }}/cache-priv-key.pem \ - result - - - name: prepare tarball to upload - run: nix run github:nixos/nixpkgs#gnutar hcvf result.tar result - - - name: upload result - uses: actions/upload-artifact@6027e3dd177782cd8ab9af838c04fd81a07f1d47 - with: - name: ${{ matrix.os }}.tar - path: result.tar - if-no-files-found: error diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml deleted file mode 100644 index 0b9ac66..0000000 --- a/.github/workflows/check.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: check -on: - workflow_dispatch: - push: - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Install Nix - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 - with: - enable_kvm: true - extra_nix_config: | - show-trace = true - experimental-features = nix-command flakes - extra-substituters = https://nixcache.cy7.sh - extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 - with: - persist-credentials: false - - - name: Run checks - run: nix flake check -L diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index cc7fabc..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: test -on: - workflow_dispatch: - push: - pull_request: - -env: - CARGO_TERM_COLOR: always - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Install Nix - uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 - with: - enable_kvm: true - extra_nix_config: | - show-trace = true - experimental-features = nix-command flakes - extra-substituters = https://nixcache.cy7.sh - extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= - - - uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2 - with: - persist-credentials: false - - - name: Run tests - run: nix develop -c cargo test --verbose diff --git a/Cargo.lock b/Cargo.lock index 2a630e0..2608cf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,19 +27,10 @@ dependencies = [ ] [[package]] -name = "android-tzdata" -version = "0.1.1" +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "anstream" @@ -156,6 +147,445 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.3.1", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.82.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6eab2900764411ab01c8e91a76fd11a63b4e12bc3da97d9e14a0ce1343d86d3" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.63.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65d21e1ba6f2cdec92044f904356a19f5ad86961acf015741106cdfafd747c0" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "crc64fast-nvme", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.4.9", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.6.0", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.5", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.26", + "rustls-native-certs 0.8.1", + "rustls-pki-types", + "tokio", + "tower 0.5.2", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0" +dependencies = [ + "aws-smithy-runtime-api", + "once_cell", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "once_cell", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.3.1", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.7.9" @@ -166,8 +596,8 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "itoa", "matchit", @@ -192,8 +622,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -218,6 +648,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.21.7" @@ -230,12 +666,45 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + [[package]] name = "bitflags" version = "2.9.0" @@ -280,6 +749,16 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.2.19" @@ -291,6 +770,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -298,22 +786,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "cfg_aliases" -version = "0.2.1" +name = "clang-sys" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "chrono" -version = "0.4.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ - "android-tzdata", - "iana-time-zone", - "num-traits", - "serde", - "windows-link", + "glob", + "libc", + "libloading", ] [[package]] @@ -356,6 +836,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.12.0" @@ -453,6 +942,30 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -462,6 +975,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" +dependencies = [ + "crc", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -477,6 +999,28 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -579,6 +1123,16 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.9" @@ -589,6 +1143,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -597,6 +1160,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -610,14 +1174,32 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "ed25519" version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -640,6 +1222,26 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff", + "generic-array", + "group", + "pkcs8 0.9.0", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -682,6 +1284,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -734,6 +1346,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -840,10 +1458,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "wasm-bindgen", ] [[package]] @@ -853,11 +1469,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", - "wasm-bindgen", ] [[package]] @@ -872,6 +1486,36 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.9.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.9" @@ -883,7 +1527,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.3.1", "indexmap 2.9.0", "slab", "tokio", @@ -902,6 +1546,11 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "hdrhistogram" @@ -922,6 +1571,41 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" @@ -933,6 +1617,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -940,7 +1635,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -951,8 +1646,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -968,21 +1663,36 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humansize" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" -dependencies = [ - "libm", -] - [[package]] name = "humantime" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.6.0" @@ -992,9 +1702,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.9", + "http 1.3.1", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1004,6 +1714,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.5" @@ -1011,14 +1737,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.3.1", + "hyper 1.6.0", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.26", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tower-service", ] @@ -1028,7 +1754,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper", + "hyper 1.6.0", "hyper-util", "pin-project-lite", "tokio", @@ -1043,7 +1769,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-util", "native-tls", "tokio", @@ -1060,9 +1786,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.6.0", "libc", "pin-project-lite", "socket2", @@ -1071,30 +1797,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "iana-time-zone" -version = "0.1.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "icu_collections" version = "1.5.0" @@ -1268,9 +1970,9 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" -version = "0.14.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -1307,6 +2009,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.172" @@ -1314,10 +2022,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] -name = "libm" -version = "0.2.13" +name = "libloading" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9627da5196e5d8ed0b0495e61e518847578da83483c37288316d9b2e03a7f72" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] [[package]] name = "libmimalloc-sys" @@ -1338,6 +2050,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -1366,6 +2084,15 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "matchers" version = "0.1.0" @@ -1499,6 +2226,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-compression", + "aws-config", + "aws-sdk-s3", "bytes", "clap", "console-subscriber", @@ -1506,9 +2235,7 @@ dependencies = [ "cxx-build", "ed25519-dalek", "futures", - "humansize", "nix-compat", - "object_store", "pkg-config", "regex", "reqwest", @@ -1517,10 +2244,9 @@ dependencies = [ "sha2", "tempfile", "tokio", + "tokio-stream", "tokio-util", "tracing", - "tracing-subscriber", - "ulid", "url", ] @@ -1544,13 +2270,18 @@ dependencies = [ ] [[package]] -name = "nu-ansi-term" -version = "0.46.0" +name = "num-conv" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "overload", - "winapi", + "num-traits", ] [[package]] @@ -1592,40 +2323,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "object_store" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "chrono", - "form_urlencoded", - "futures", - "http", - "http-body-util", - "humantime", - "hyper", - "itertools", - "md-5", - "parking_lot", - "percent-encoding", - "quick-xml", - "rand 0.8.5", - "reqwest", - "ring", - "serde", - "serde_json", - "serde_urlencoded", - "thiserror", - "tokio", - "tracing", - "url", - "walkdir", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -1677,10 +2374,21 @@ dependencies = [ ] [[package]] -name = "overload" -version = "0.1.1" +name = "outref" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] [[package]] name = "parking_lot" @@ -1743,14 +2451,24 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", +] + [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.9", + "spki 0.7.3", ] [[package]] @@ -1759,6 +2477,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1768,6 +2492,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -1818,70 +2552,6 @@ dependencies = [ "prost", ] -[[package]] -name = "quick-xml" -version = "0.37.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quinn" -version = "0.11.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" -dependencies = [ - "bytes", - "getrandom 0.3.2", - "rand 0.9.1", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.59.0", -] - [[package]] name = "quote" version = "1.0.40" @@ -1904,18 +2574,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_chacha", + "rand_core", ] [[package]] @@ -1925,17 +2585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", + "rand_core", ] [[package]] @@ -1947,15 +2597,6 @@ dependencies = [ "getrandom 0.2.15", ] -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.11" @@ -1997,6 +2638,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -2020,12 +2667,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.9", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.6.0", + "hyper-rustls 0.27.5", "hyper-tls", "hyper-util", "ipnet", @@ -2036,11 +2683,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pemfile", - "rustls-pki-types", + "rustls-pemfile 2.2.0", "serde", "serde_json", "serde_urlencoded", @@ -2048,18 +2691,26 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls", - "tokio-util", "tower 0.5.2", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", "windows-registry", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.14" @@ -2082,9 +2733,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc_version" @@ -2095,6 +2746,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.0.5" @@ -2104,24 +2768,48 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ + "aws-lc-rs", "once_cell", - "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.1", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -2134,6 +2822,15 @@ dependencies = [ "security-framework 3.2.0", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -2148,8 +2845,15 @@ name = "rustls-pki-types" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "web-time", + "ring", + "untrusted", ] [[package]] @@ -2158,6 +2862,7 @@ version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2175,15 +2880,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" version = "0.1.27" @@ -2205,6 +2901,30 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -2291,6 +3011,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" @@ -2326,13 +3057,23 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "signature" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "rand_core 0.6.4", + "rand_core", ] [[package]] @@ -2360,6 +3101,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -2367,7 +3118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.9", ] [[package]] @@ -2449,7 +3200,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix", + "rustix 1.0.5", "windows-sys 0.59.0", ] @@ -2492,6 +3243,36 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -2502,21 +3283,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinyvec" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.44.2" @@ -2557,13 +3323,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls", + "rustls 0.23.26", "tokio", ] @@ -2580,9 +3356,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.15" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", @@ -2619,11 +3395,11 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2", - "http", - "http-body", + "h2 0.4.9", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -2649,7 +3425,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand", "slab", "tokio", "tokio-util", @@ -2717,17 +3493,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.19" @@ -2735,15 +3500,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", - "nu-ansi-term", "once_cell", "regex", "sharded-slab", - "smallvec", "thread_local", "tracing", "tracing-core", - "tracing-log", ] [[package]] @@ -2758,16 +3520,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" -[[package]] -name = "ulid" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" -dependencies = [ - "rand 0.9.1", - "web-time", -] - [[package]] name = "unicode-ident" version = "1.0.18" @@ -2798,6 +3550,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -2816,6 +3574,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" + [[package]] name = "valuable" version = "0.1.1" @@ -2835,14 +3599,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "walkdir" -version = "2.5.0" +name = "vsimd" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "want" @@ -2939,19 +3699,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "web-sys" version = "0.3.77" @@ -2963,31 +3710,17 @@ dependencies = [ ] [[package]] -name = "web-time" -version = "1.1.0" +name = "which" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ - "js-sys", - "wasm-bindgen", + "either", + "home", + "once_cell", + "rustix 0.38.44", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.9" @@ -2997,47 +3730,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.61.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings 0.4.0", -] - -[[package]] -name = "windows-implement" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "windows-link" version = "0.1.1" @@ -3051,7 +3743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", - "windows-strings 0.3.1", + "windows-strings", "windows-targets 0.53.0", ] @@ -3073,15 +3765,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -3258,6 +3941,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index c1de8ac..2430fdd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,37 +3,30 @@ name = "nixcp" version = "0.1.0" edition = "2024" -[profile.release] -lto = true -codegen-units = 1 - [dependencies] anyhow = "1.0.97" async-compression = { version = "0.4.22", features = ["tokio", "zstd"] } +aws-config = { version = "1.6.1", features = ["behavior-version-latest"] } +aws-sdk-s3 = "1.82.0" clap = { version = "4.5.34", features = ["derive"] } ed25519-dalek = "2.1.1" futures = "0.3.31" nix-compat = { git = "https://github.com/tvlfyi/tvix.git", version = "0.1.0" } regex = "1.11.1" reqwest = "0.12.15" -serde = { version = "1.0.219", features = ["derive"] } +serde = { version = "1.0.219", features = [ "derive" ]} serde_json = "1.0.140" sha2 = "0.10.8" -tokio = { version = "1.44.1", features = ["full", "tracing", "parking_lot"] } +tokio = { version = "1.44.1", features = [ "full", "tracing", "parking_lot" ]} tracing = "0.1.41" -url = { version = "2.5.4", features = ["serde"] } +url = { version = "2.5.4", features = [ "serde" ]} cxx = "1.0" console-subscriber = "0.4.1" -tokio-util = { version = "0.7.15", features = ["io"] } bytes = "1.10.1" -object_store = { version = "0.12.0", features = ["aws"] } -ulid = "1.2.1" -tracing-subscriber = "0.3.19" -humansize = "2.1.3" +tokio-stream = { version = "0.1.17", features = ["fs"] } +tempfile = "3.19.1" +tokio-util = { version = "0.7.14", features = ["io"] } [build-dependencies] cxx-build = "1.0" pkg-config = "0.3.32" - -[dev-dependencies] -tempfile = "3.19.1" diff --git a/LICENSE b/LICENSE deleted file mode 100644 index e886c24..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Cy - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/README.md b/README.md index f9317c8..ff71137 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,14 @@ The signing key is generated with: nix-store --generate-binary-cache-key nixcache.cy7.sh cache-priv-key.pem cache-pub-key.pem ``` -`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables should be set with your s3 credentials. +`AWS_ACCESS_KEY_ID` and `AWS_ENDPOINT_URL` environment variables should be set with your s3 credentials. ``` -Usage: nixcp push [OPTIONS] --bucket --signing-key [PATH]... +Usage: nixcp [OPTIONS] --bucket --signing-key -Arguments: - [PATH]... Path to upload e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 +Commands: + push + help Print this message or the help of the given subcommand(s) Options: --bucket @@ -27,13 +28,15 @@ Options: --signing-key Path to the file containing signing key e.g. ~/cache-priv-key.pem --region - If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 + If unspecified, will get it form AWS_DEFAULT_REGION envar or the AWS default --endpoint - If unspecifed, will get it from AWS_ENDPOINT envar e.g. https://s3.example.com - --no-default-upstream - Do not include cache.nixos.org as upstream + If unspecifed, will get it from AWS_ENDPOINT_URL envar or the AWS default e.g. https://s3.example.com + --profile + AWS profile to use -h, --help Print help + -V, --version + Print version ``` ## Install with nix diff --git a/flake.nix b/flake.nix index 16b57e6..46caa41 100644 --- a/flake.nix +++ b/flake.nix @@ -11,15 +11,8 @@ }; }; - outputs = - inputs@{ - nixpkgs, - flake-utils, - crane, - ... - }: - flake-utils.lib.eachDefaultSystem ( - system: + outputs = inputs@{ nixpkgs, flake-utils, crane, ... }: + flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; @@ -28,21 +21,11 @@ ]; }; toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; - craneLib = (crane.mkLib pkgs).overrideToolchain (_: toolchain); + craneLib = (crane.mkLib pkgs).overrideToolchain(_: toolchain); lib = pkgs.lib; - - # don't clean cpp files - cppFilter = path: _type: builtins.match ".*(cpp|hpp)$" path != null; - cppOrCargo = path: type: (cppFilter path type) || (craneLib.filterCargoSources path type); - src = lib.cleanSourceWith { - src = ./.; - filter = cppOrCargo; - name = "source"; - }; - - commonArgs = { - inherit src; - strictDeps = true; + in + { + devShells.default = pkgs.mkShell { nativeBuildInputs = with pkgs; [ pkg-config ]; @@ -51,58 +34,23 @@ openssl nix boost - ]; - # for cpp bindings to work - NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; - # skip integration tests (they need a connection to the nix store) - cargoTestExtraArgs = "--bins"; - }; - - cargoArtifacts = craneLib.buildDepsOnly commonArgs; - nixcp = craneLib.buildPackage ( - commonArgs - // { - inherit cargoArtifacts; - } - ); - in - { - checks = { - # clippy with all warnings denied - clippy = craneLib.cargoClippy ( - commonArgs - // { - inherit cargoArtifacts; - cargoClippyExtraArgs = "--all-targets -- --deny warnings"; - } - ); - - # check formatting - cargoFmt = craneLib.cargoFmt { - inherit src; - }; - tomlFmt = craneLib.taploFmt { - src = lib.sources.sourceFilesBySuffices src [ ".toml" ]; - }; - }; - - devShells.default = craneLib.devShell { - inputsFrom = [ nixcp ]; - - RUST_BACKGRACE = 1; - # for cpp bindings to work - NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; - - packages = with pkgs; [ tokio-console - cargo-udeps - cargo-audit ]; + NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; + RUST_LOG = "nixcp=debug"; + RUST_BACKGRACE = 1; }; - formatter = pkgs.nixfmt-rfc-style; - - packages.default = nixcp; + packages.default = craneLib.buildPackage { + src = craneLib.cleanCargoSource ./.; + strictDeps = true; + nativeBuildInputs = with pkgs; [ + pkg-config + ]; + buildInputs = with pkgs; [ + openssl + ]; + }; } ); } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c96aa24..eceaf24 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,9 @@ [toolchain] channel = "nightly" profile = "minimal" -components = ["rust-src", "rust-analyzer", "rustfmt", "clippy"] +components = [ + "rust-src", + "rust-analyzer", + "rustfmt", + "clippy", +] \ No newline at end of file diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs index 46b86dd..701a15e 100644 --- a/src/bindings/mod.rs +++ b/src/bindings/mod.rs @@ -23,7 +23,6 @@ use std::pin::Pin; use std::task::{Context, Poll}; use anyhow::Result; -use bytes::Bytes; use futures::stream::{Stream, StreamExt}; use tokio::io::{AsyncWrite, AsyncWriteExt}; @@ -126,7 +125,7 @@ impl AsyncWriteAdapter { writer.write_all(&v).await?; } Err(e) => { - return Err(e.into()); + return Err(e); } } } @@ -140,7 +139,7 @@ impl AsyncWriteAdapter { } impl Stream for AsyncWriteAdapter { - type Item = std::io::Result; + type Item = Result>; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.receiver.poll_recv(cx) { @@ -148,9 +147,9 @@ impl Stream for AsyncWriteAdapter { Poll::Ready(Some(message)) => { use AsyncWriteMessage::*; match message { - Data(v) => Poll::Ready(Some(Ok(v.into()))), + Data(v) => Poll::Ready(Some(Ok(v))), Error(exception) => { - let error = std::io::Error::other(format!("cxx error: {exception}")); + let error = anyhow::Error::msg(format!("cxx error: {exception}")); Poll::Ready(Some(Err(error))) } Eof => { @@ -161,7 +160,7 @@ impl Stream for AsyncWriteAdapter { } Poll::Ready(None) => { if !self.eof { - Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe)))) + Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe).into()))) } else { Poll::Ready(None) } @@ -209,6 +208,9 @@ mod ffi { include_derivers: bool, ) -> Result>>; + /// Obtains a handle to the Nix store. + fn open_nix_store() -> Result>; + /// Creates a NAR dump from a path. fn nar_from_path( self: Pin<&mut CNixStore>, @@ -216,17 +218,12 @@ mod ffi { sender: Box, ) -> Result<()>; - /// Obtains a handle to the Nix store. - fn open_nix_store() -> Result>; - // ========= // CPathInfo // ========= /// Mid-level wrapper for the `nix::ValidPathInfo` struct. type CPathInfo; - /// Returns the size of the NAR. - fn nar_size(self: Pin<&mut CPathInfo>) -> u64; /// Returns the references of the store path. fn references(self: Pin<&mut CPathInfo>) -> UniquePtr>; diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp index 8bf2cb3..326e878 100644 --- a/src/bindings/nix.cpp +++ b/src/bindings/nix.cpp @@ -57,10 +57,6 @@ void RustSink::eof() { CPathInfo::CPathInfo(nix::ref pi) : pi(pi) {} -uint64_t CPathInfo::nar_size() { - return this->pi->narSize; -} - std::unique_ptr> CPathInfo::sigs() { std::vector result; for (auto&& elem : this->pi->sigs) { diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp index 87b3ebf..5c79a33 100644 --- a/src/bindings/nix.hpp +++ b/src/bindings/nix.hpp @@ -65,7 +65,6 @@ public: CPathInfo(nix::ref pi); std::unique_ptr> sigs(); std::unique_ptr> references(); - uint64_t nar_size(); }; class CNixStore { diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..139597f --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,2 @@ + + diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 8b1fc18..0000000 --- a/src/lib.rs +++ /dev/null @@ -1,65 +0,0 @@ -use std::path::PathBuf; - -use clap::{Args, Parser, Subcommand}; - -mod bindings; -pub mod make_nar; -pub mod path_info; -pub mod push; -pub mod store; -mod uploader; - -#[derive(Parser, Debug)] -#[command(version)] -#[command(name = "nixcp")] -#[command(about = "Upload store paths to a s3 binary cache")] -#[command(long_about = None)] -pub struct Cli { - #[command(subcommand)] - pub command: Commands, - - /// Whether to enable tokio-console - #[arg(long)] - pub tokio_console: bool, -} - -#[derive(Debug, Subcommand)] -pub enum Commands { - #[command(arg_required_else_help = true)] - Push(PushArgs), -} - -#[derive(Debug, Args)] -pub struct PushArgs { - /// The s3 bucket to upload to - #[arg(long, value_name = "bucket name")] - bucket: String, - - /// Upstream cache to check against. Can be specified multiple times. - /// cache.nixos.org is always included. - #[arg(long = "upstream", short, value_name = "nixcache.example.com")] - upstreams: Vec, - - /// Path to the file containing signing key - /// e.g. ~/cache-priv-key.pem - #[arg(long)] - signing_key: String, - - /// If unspecified, will get it form AWS_DEFAULT_REGION envar or default to us-east-1 - #[arg(long)] - region: Option, - - /// If unspecifed, will get it from AWS_ENDPOINT envar - /// e.g. https://s3.example.com - #[arg(long)] - endpoint: Option, - - /// Do not include cache.nixos.org as upstream - #[arg(long)] - no_default_upstream: bool, - - /// Path to upload - /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 - #[arg(value_name = "PATH")] - pub paths: Vec, -} diff --git a/src/main.rs b/src/main.rs index 1afe4b6..f4c0f4f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,80 @@ -use anyhow::{Context, Result}; -use clap::Parser; -use tracing_subscriber::{EnvFilter, prelude::*}; +#![feature(let_chains)] +#![feature(extend_one)] -use nixcp::push::Push; -use nixcp::store::Store; -use nixcp::{Cli, Commands}; +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use clap::{Args, Parser, Subcommand}; + +use push::Push; +use store::Store; + +mod bindings; +mod cli; +mod path_info; +mod push; +mod store; +mod uploader; + +#[derive(Parser, Debug)] +#[command(version)] +#[command(name = "nixcp")] +#[command(about = "Upload store paths to a s3 binary cache")] +#[command(long_about = None)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Debug, Subcommand)] +enum Commands { + #[command(arg_required_else_help = true)] + Push(PushArgs), +} + +#[derive(Debug, Args)] +pub struct PushArgs { + /// The s3 bucket to upload to + #[arg(long, value_name = "bucket name")] + bucket: String, + + /// Upstream cache to check against. Can be specified multiple times. + /// cache.nixos.org is always included. + #[arg(long = "upstream", short, value_name = "nixcache.example.com")] + upstreams: Vec, + + /// Path to the file containing signing key + /// e.g. ~/cache-priv-key.pem + #[arg(long)] + signing_key: String, + + /// If unspecified, will get it form AWS_DEFAULT_REGION envar + #[arg(long)] + region: Option, + + /// If unspecifed, will get it from AWS_ENDPOINT_URL envar + /// e.g. https://s3.example.com + #[arg(long)] + endpoint: Option, + + /// AWS profile to use + #[arg(long)] + profile: Option, + + #[arg(long)] + skip_signature_check: bool, + + /// Path to upload + /// e.g. ./result or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1 + #[arg(value_name = "PATH")] + paths: Vec, +} #[tokio::main] async fn main() -> Result<()> { + console_subscriber::init(); + let cli = Cli::parse(); - init_logging(cli.tokio_console); match &cli.command { Commands::Push(cli) => { @@ -24,23 +89,3 @@ async fn main() -> Result<()> { Ok(()) } - -fn init_logging(tokio_console: bool) { - let env_filter = EnvFilter::from_default_env(); - let fmt_layer = tracing_subscriber::fmt::layer().with_filter(env_filter); - - let console_layer = if tokio_console { - Some(console_subscriber::spawn()) - } else { - None - }; - - tracing_subscriber::registry() - .with(fmt_layer) - .with(console_layer) - .init(); - - if tokio_console { - println!("tokio-console is enabled"); - } -} diff --git a/src/make_nar.rs b/src/make_nar.rs deleted file mode 100644 index 97d6b1f..0000000 --- a/src/make_nar.rs +++ /dev/null @@ -1,81 +0,0 @@ -use anyhow::Result; -use async_compression::{Level, tokio::bufread::ZstdEncoder}; -use nix_compat::{ - narinfo::{self, NarInfo}, - store_path::StorePath, -}; -use sha2::{Digest, Sha256}; -use std::{mem::take, sync::Arc}; -use tokio::io::{AsyncRead, BufReader}; -use tokio_util::io::InspectReader; - -use crate::path_info::PathInfo; -use crate::store::Store; - -pub struct MakeNar<'a> { - path_info: &'a PathInfo, - store: Arc, - pub nar_hasher: Sha256, - /// hash of compressed nar file - file_hasher: Sha256, - pub nar_size: u64, - file_size: u64, -} - -impl<'a> MakeNar<'a> { - pub fn new(path_info: &'a PathInfo, store: Arc) -> Result { - Ok(Self { - path_info, - store, - nar_hasher: Sha256::new(), - file_hasher: Sha256::new(), - nar_size: 0, - file_size: 0, - }) - } - - /// Returns a compressed nar reader which can be uploaded. File hash will be available when - /// everything is read - pub fn compress_and_hash(&mut self) -> Result { - let nar_reader = self.store.nar_from_path(self.path_info.path.clone()); - // reader that hashes as nar is read - let nar_reader = InspectReader::new(nar_reader, |x| { - self.nar_size += x.len() as u64; - self.nar_hasher.update(x); - }); - - let encoder = ZstdEncoder::with_quality(BufReader::new(nar_reader), Level::Default); - // reader that updates file_hash as the compressed nar is read - Ok(InspectReader::new(encoder, |x| { - self.file_size += x.len() as u64; - self.file_hasher.update(x); - })) - } - - /// Returns *unsigned* narinfo. `url` must be updated before uploading - pub fn get_narinfo(&mut self) -> Result { - let file_hash = take(&mut self.file_hasher).finalize().into(); - let nar_hash = take(&mut self.nar_hasher).finalize().into(); - - Ok(NarInfo { - flags: narinfo::Flags::empty(), - store_path: self.path_info.path.as_ref(), - nar_hash, - nar_size: self.nar_size, - references: self - .path_info - .references - .iter() - .map(StorePath::as_ref) - .collect(), - signatures: Vec::new(), - ca: None, - system: None, - deriver: None, - compression: Some("zstd"), - file_hash: Some(file_hash), - file_size: Some(self.file_size), - url: "", - }) - } -} diff --git a/src/path_info.rs b/src/path_info.rs index 213fd1a..e62b68a 100644 --- a/src/path_info.rs +++ b/src/path_info.rs @@ -1,10 +1,10 @@ use std::collections::HashSet; -use anyhow::{Context, Result, anyhow}; +use anyhow::{Context, Result}; +use aws_sdk_s3 as s3; use futures::future::join_all; use nix_compat::nixbase32; use nix_compat::store_path::StorePath; -use object_store::{ObjectStore, aws::AmazonS3, path::Path as ObjectPath}; use regex::Regex; use std::path::Path; use tokio::process::Command; @@ -13,61 +13,41 @@ use url::Url; use crate::store::Store; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone)] pub struct PathInfo { pub path: StorePath, pub signatures: Vec, pub references: Vec>, - pub nar_size: u64, } impl PathInfo { - pub async fn from_derivation(drv: &Path, store: &Store) -> Result { - debug!("query path info for {:?}", drv); + pub async fn from_path(path: &Path, store: &Store) -> Result { + debug!("query path info for {:?}", path); - let derivation = match drv.extension() { - Some(ext) if ext == "drv" => drv.as_os_str().as_encoded_bytes(), + let derivation = match path.extension() { + Some(ext) if ext == "drv" => path.as_os_str().as_encoded_bytes(), _ => { - let drv = { - // resolve symlink - if drv.is_symlink() { - &drv.canonicalize()? - } else { - drv - } - }; &Command::new("nix") .arg("path-info") .arg("--derivation") - .arg(drv) + .arg(path) .output() .await - .context(format!("run command: nix path-info --derivaiton {drv:?}"))? + .context(format!("run command: nix path-info --derivaiton {path:?}"))? .stdout } }; let derivation = String::from_utf8_lossy(derivation); debug!("derivation: {derivation}"); - if derivation.is_empty() { - return Err(anyhow!( - "nix path-info did not return a derivation for {drv:#?}" - )); - } - - Self::from_path(derivation.trim(), store).await - } - - pub async fn from_path(path: &str, store: &Store) -> Result { - let store_path = - StorePath::from_absolute_path(path.as_bytes()).context("storepath from path")?; + let store_path = StorePath::from_absolute_path(derivation.trim().as_bytes()) + .context("storepath from derivation")?; store .query_path_info(store_path) .await - .context("query pathinfo for path") + .context("query pathinfo for derivation") } - // TODO: skip call to query_path_info and return Vec? pub async fn get_closure(&self, store: &Store) -> Result> { let futs = store .compute_fs_closure(self.path.clone()) @@ -102,15 +82,15 @@ impl PathInfo { .filter_map(|signature| Some(signature.split_once(":")?.0)) .collect(); trace!("signers for {}: {:?}", self.path, signers); - signers + return signers; } pub async fn check_upstream_hit(&self, upstreams: &[Url]) -> bool { for upstream in upstreams { let upstream = upstream - .join(self.narinfo_path().as_ref()) + .join(format!("{}.narinfo", self.digest()).as_str()) .expect("adding .narinfo should make a valid url"); - trace!("querying {}", upstream); + debug!("querying {}", upstream); let res_status = reqwest::Client::new() .head(upstream.as_str()) .send() @@ -128,12 +108,17 @@ impl PathInfo { self.path.to_absolute_path() } - pub fn narinfo_path(&self) -> ObjectPath { - ObjectPath::parse(format!("{}.narinfo", nixbase32::encode(self.path.digest()))) - .expect("must parse to a valid object_store path") + pub fn digest(&self) -> String { + nixbase32::encode(self.path.digest()) } - pub async fn check_if_already_exists(&self, s3: &AmazonS3) -> bool { - s3.head(&self.narinfo_path()).await.is_ok() + pub async fn check_if_already_exists(&self, s3_client: &s3::Client, bucket: String) -> bool { + s3_client + .head_object() + .bucket(bucket) + .key(format!("{}.narinfo", self.digest())) + .send() + .await + .is_ok() } } diff --git a/src/push.rs b/src/push.rs index 9fc043d..18f74b9 100644 --- a/src/push.rs +++ b/src/push.rs @@ -1,6 +1,6 @@ use std::{ - collections::HashSet, fs, + iter::once, path::PathBuf, sync::{ Arc, @@ -9,11 +9,11 @@ use std::{ }; use anyhow::{Context, Result}; +use aws_config::Region; +use aws_sdk_s3 as s3; use futures::future::join_all; -use humansize::{DECIMAL, format_size}; use nix_compat::narinfo::{self, SigningKey}; -use object_store::aws::{AmazonS3, AmazonS3Builder}; -use tokio::sync::{RwLock, Semaphore, mpsc}; +use tokio::sync::{RwLock, mpsc}; use tracing::debug; use url::Url; @@ -21,10 +21,11 @@ use crate::{PushArgs, path_info::PathInfo, store::Store, uploader::Uploader}; pub struct Push { upstream_caches: Vec, - store_paths: Arc>>, + store_paths: Arc>>, + s3_client: s3::Client, signing_key: SigningKey, + bucket: String, store: Arc, - s3: Arc, // paths that we skipped cause of a signature match signature_hit_count: AtomicUsize, // paths that we skipped cause we found it on an upstream @@ -38,13 +39,11 @@ pub struct Push { impl Push { pub async fn new(cli: &PushArgs, store: Store) -> Result { let mut upstreams = Vec::with_capacity(cli.upstreams.len() + 1); - if !cli.no_default_upstream { - upstreams.push( - Url::parse("https://cache.nixos.org") - .expect("default upstream must be a valid url"), - ); - } - for upstream in &cli.upstreams { + for upstream in cli + .upstreams + .iter() + .chain(once(&"https://cache.nixos.org".to_string())) + { upstreams .push(Url::parse(upstream).context(format!("failed to parse {upstream} as url"))?); } @@ -52,21 +51,25 @@ impl Push { let key = fs::read_to_string(&cli.signing_key)?; let signing_key = narinfo::parse_keypair(key.as_str())?.0; - let mut s3_builder = AmazonS3Builder::from_env().with_bucket_name(&cli.bucket); - + let mut s3_config = aws_config::from_env(); if let Some(region) = &cli.region { - s3_builder = s3_builder.with_region(region); + s3_config = s3_config.region(Region::new(region.clone())); } if let Some(endpoint) = &cli.endpoint { - s3_builder = s3_builder.with_endpoint(endpoint); + s3_config = s3_config.endpoint_url(endpoint); + } + if let Some(profile) = &cli.profile { + s3_config = s3_config.profile_name(profile); } + let s3_client = s3::Client::new(&s3_config.load().await); Ok(Self { upstream_caches: upstreams, - store_paths: Arc::new(RwLock::new(HashSet::new())), + store_paths: Arc::new(RwLock::new(Vec::new())), + s3_client, signing_key, + bucket: cli.bucket.clone(), store: Arc::new(store), - s3: Arc::new(s3_builder.build()?), signature_hit_count: AtomicUsize::new(0), upstream_hit_count: AtomicUsize::new(0), already_exists_count: AtomicUsize::new(0), @@ -81,7 +84,7 @@ impl Push { let store = self.store.clone(); futs.push(tokio::spawn(async move { - let path_info = PathInfo::from_derivation(path.as_path(), &store) + let path_info = PathInfo::from_path(path.as_path(), &store) .await .context("get path info for path")?; debug!("path-info for {path:?}: {path_info:?}"); @@ -106,10 +109,9 @@ impl Push { } pub async fn run(&'static self) -> Result<()> { - let (tx, rx) = mpsc::channel(1); + let (tx, rx) = mpsc::channel(10); let filter = tokio::spawn(self.filter_from_upstream(tx)); let upload = tokio::spawn(self.upload(rx)); - filter.await?; upload.await??; Ok(()) @@ -117,24 +119,26 @@ impl Push { /// filter paths that are on upstream and send to `tx` async fn filter_from_upstream(&'static self, tx: mpsc::Sender) { - let mut handles = Vec::new(); + let mut handles = Vec::with_capacity(10); let store_paths = self.store_paths.read().await.clone(); - // limit number of inflight requests - let inflight_permits = Arc::new(Semaphore::new(32)); for path in store_paths.into_iter() { if path.check_upstream_signature(&self.upstream_caches) { debug!("skip {} (signature match)", path.absolute_path()); - self.signature_hit_count.fetch_add(1, Ordering::Relaxed); + self.signature_hit_count.fetch_add(1, Ordering::Release); continue; } handles.push({ let tx = tx.clone(); - let inflight_permits = inflight_permits.clone(); tokio::spawn(async move { - let _permit = inflight_permits.acquire().await.unwrap(); - if !path.check_upstream_hit(&self.upstream_caches).await { - if path.check_if_already_exists(&self.s3).await { + if !path + .check_upstream_hit(self.upstream_caches.as_slice()) + .await + { + if path + .check_if_already_exists(&self.s3_client, self.bucket.clone()) + .await + { debug!("skip {} (already exists)", path.absolute_path()); self.already_exists_count.fetch_add(1, Ordering::Relaxed); } else { @@ -157,31 +161,21 @@ impl Push { async fn upload(&'static self, mut rx: mpsc::Receiver) -> Result<()> { let mut uploads = Vec::new(); - let permits = Arc::new(Semaphore::new(10)); loop { - let permits = permits.clone(); - if let Some(path_to_upload) = rx.recv().await { + println!("uploading: {}", path_to_upload.absolute_path()); + + let uploader = Uploader::new( + &self.signing_key, + path_to_upload, + &self.s3_client, + self.bucket.clone(), + )?; + uploads.push(tokio::spawn({ - // large uploads will be concurrently uploaded with multipart anyway so don't spawn - // too much of them - let permit = if path_to_upload.nar_size > 15 * 1024 * 1024 { - Some(permits.acquire_owned().await.unwrap()) - } else { - None - }; - println!( - "uploading: {} (size: {})", - path_to_upload.absolute_path(), - format_size(path_to_upload.nar_size, DECIMAL) - ); - let uploader = Uploader::new(&self.signing_key, path_to_upload)?; - let s3 = self.s3.clone(); - let store = self.store.clone(); async move { - let res = uploader.upload(s3, store).await; - drop(permit); + let res = uploader.upload().await; self.upload_count.fetch_add(1, Ordering::Relaxed); res } diff --git a/src/store.rs b/src/store.rs index 9b925b2..0433362 100644 --- a/src/store.rs +++ b/src/store.rs @@ -2,8 +2,7 @@ use std::{ffi::OsStr, os::unix::ffi::OsStrExt, sync::Arc}; use anyhow::{Context, Result}; use nix_compat::store_path::StorePath; -use tokio::{io::AsyncRead, task}; -use tokio_util::io::StreamReader; +use tokio::task; use crate::{ bindings::{self, AsyncWriteAdapter}, @@ -32,13 +31,13 @@ impl Store { inner .store() .compute_fs_closure(path.to_string().as_bytes(), false, true, true)?; - cxx_vector + Ok(cxx_vector .iter() .map(|x| { StorePath::from_bytes(x.as_bytes()) .context("make StorePath from vector returned by compute_fs_closure") }) - .collect::>() + .collect::>()?) }) .await .unwrap() @@ -69,32 +68,29 @@ impl Store { .map(|x| StorePath::from_bytes(x.as_bytes())) .collect::>() .context("get references from pathinfo")?; - let nar_size = c_path_info.pin_mut().nar_size(); Ok(PathInfo { path, signatures, references, - nar_size, }) }) .await .unwrap() } - pub fn nar_from_path(&self, store_path: StorePath) -> impl AsyncRead { + pub fn stream_nar(&self, path: StorePath) -> AsyncWriteAdapter { let inner = self.inner.clone(); let (adapter, mut sender) = AsyncWriteAdapter::new(); - let base_name = store_path.to_string().as_bytes().to_vec(); - tokio::task::spawn_blocking(move || { - // Send all exceptions through the channel, and ignore errors - // during sending (the channel may have been closed). - if let Err(e) = inner.store().nar_from_path(base_name, sender.clone()) { + task::spawn_blocking(move || { + if let Err(e) = inner + .store() + .nar_from_path(path.to_string().as_bytes().to_vec(), sender.clone()) + { let _ = sender.rust_error(e); } }); - - StreamReader::new(adapter) + adapter } } diff --git a/src/uploader.rs b/src/uploader.rs index c829a79..8f4efaa 100644 --- a/src/uploader.rs +++ b/src/uploader.rs @@ -1,80 +1,260 @@ +use std::{collections::BTreeMap, os::unix::fs::PermissionsExt, path::PathBuf}; + use anyhow::Result; -use bytes::BytesMut; -use nix_compat::{narinfo::SigningKey, nixbase32}; -use object_store::{ObjectStore, aws::AmazonS3, buffered::BufWriter, path::Path}; -use std::sync::Arc; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tracing::{debug, trace}; -use ulid::Ulid; +use async_compression::{Level, tokio::bufread::ZstdEncoder}; +use aws_sdk_s3::{ + self as s3, + types::{CompletedMultipartUpload, CompletedPart}, +}; +use bytes::{BufMut, Bytes, BytesMut}; +use futures::{future::join_all, stream::TryStreamExt}; +use nix_compat::{ + nar::writer::r#async as nar, + narinfo::{self, NarInfo, SigningKey}, + nixbase32, + store_path::StorePath, +}; +use sha2::{Digest, Sha256}; +use tokio::{ + fs::{File, read_dir, read_link}, + io::{AsyncRead, BufReader}, + pin, +}; +use tokio_stream::wrappers::ReadDirStream; +use tokio_util::io::{InspectReader, read_buf}; +use tracing::debug; -use crate::{make_nar::MakeNar, path_info::PathInfo, store::Store}; +use crate::{bindings::AsyncWriteAdapter, path_info::PathInfo, store::Store}; -const CHUNK_SIZE: usize = 1024 * 1024 * 5; +const MULTIPART_CUTOFF: usize = 1024 * 1024 * 5; pub struct Uploader<'a> { signing_key: &'a SigningKey, path: PathInfo, + s3_client: &'a s3::Client, + bucket: String, + store: &'a Store, } impl<'a> Uploader<'a> { pub fn new( signing_key: &'a SigningKey, path: PathInfo, + s3_client: &'a s3::Client, + bucket: String, + store: &'a Store, ) -> Result { - Ok(Self { signing_key, path }) + Ok(Self { + signing_key, + path, + s3_client, + bucket, + store, + }) } - pub async fn upload(&self, s3: Arc, store: Arc) -> Result<()> { - let mut nar = MakeNar::new(&self.path, store)?; + pub async fn upload(&self) -> Result<()> { + let mut nar_temp = File::open(tempfile::Builder::new().tempfile()?.path()).await?; + self.make_nar(&mut nar_temp).await; - // we don't know what the hash of the compressed file will be so upload to a - // temp location for now - let temp_path = Path::parse(Ulid::new().to_string())?; - let mut s3_writer = BufWriter::new(s3.clone(), temp_path.clone()); - debug!("uploading to temp path: {}", temp_path); + // this goes to the .narinfo file + let mut nar_hasher = Sha256::new(); + // this is the URL for file .narinfo points to + let mut file_hasher = Sha256::new(); + let nar_reader = compress_and_hash_nar(nar_temp, &mut nar_hasher, &mut file_hasher); - // compress and upload nar - let mut file_reader = nar.compress_and_hash()?; - loop { - let mut buf = BytesMut::with_capacity(CHUNK_SIZE); - let n = file_reader.read_buf(&mut buf).await?; - s3_writer.put(buf.freeze()).await?; - if n == 0 { - break; + let buf = BytesMut::with_capacity(MULTIPART_CUTOFF); + let + + if first_chunk.len() < MULTIPART_CUTOFF { + let put_object = self + .s3_client + .put_object() + .bucket(&self.bucket) + .key(&nar_url) + .body(first_chunk.into()) + .send() + .await?; + debug!("put object: {:#?}", put_object); + } else { + let multipart = self + .s3_client + .create_multipart_upload() + .bucket(&self.bucket) + .key(&nar_url) + .send() + .await?; + let upload_id = multipart.upload_id().unwrap(); + + let mut parts = Vec::with_capacity(nar.len() / MULTIPART_CUTOFF); + let chunks = nar.chunks(MULTIPART_CUTOFF); + for (i, chunk) in chunks.enumerate() { + parts.push(tokio::task::spawn( + self.s3_client + .upload_part() + .bucket(&self.bucket) + .key(&nar_url) + .upload_id(upload_id) + .part_number(i as i32 + 1) + .body(chunk.to_vec().into()) + .send(), + )); } + + let completed_parts = join_all(parts) + .await + .into_iter() + .flatten() + .collect::, _>>()? + .into_iter() + .enumerate() + .map(|(i, part)| { + CompletedPart::builder() + .set_e_tag(part.e_tag().map(ToString::to_string)) + .set_part_number(Some(i as i32 + 1)) + .set_checksum_sha256(part.checksum_sha256().map(ToString::to_string)) + .build() + }) + .collect::>(); + + let completed_mp_upload = CompletedMultipartUpload::builder() + .set_parts(Some(completed_parts)) + .build(); + + let complete_mp_upload = self + .s3_client + .complete_multipart_upload() + .bucket(&self.bucket) + .key(&nar_url) + .upload_id(upload_id) + .multipart_upload(completed_mp_upload) + .send() + .await?; + + debug!("complete multipart upload: {:#?}", complete_mp_upload); } - drop(file_reader); - let mut nar_info = nar.get_narinfo()?; + let narinfo_url = format!("{}.narinfo", self.path.digest()); + debug!("uploading narinfo with key {narinfo_url}"); + self.s3_client + .put_object() + .bucket(&self.bucket) + .key(narinfo_url) + .body(nar_info.to_string().as_bytes().to_vec().into()) + .send() + .await?; + debug!("done uploading narinfo"); + + Ok(()) + } + + fn narinfo_from_nar(&self, nar: &[u8]) -> Result { + let mut hasher = Sha256::new(); + hasher.update(nar); + let nar_hash: [u8; 32] = hasher.finalize().into(); + let mut nar_info = NarInfo { + flags: narinfo::Flags::empty(), + store_path: self.path.path.as_ref(), + nar_hash, + nar_size: nar.len() as u64, + references: self.path.references.iter().map(StorePath::as_ref).collect(), + signatures: Vec::new(), + ca: None, + system: None, + deriver: None, + compression: Some("zstd"), + file_hash: None, + file_size: None, + url: "", + }; + // signature consists of: store_path, nar_hash, nar_size, and references nar_info.add_signature(self.signing_key); + Ok(nar_info) + } - // now that we can calculate the file_hash move the nar to where it should be - let real_path = nar_url( - &nar_info - .file_hash - .expect("file hash must be known at this point"), - ); - debug!("moving {} to {}", temp_path, real_path); - // the temp object must be done uploading - s3_writer.shutdown().await?; - // this is implemented as a copy-and-delete - s3.rename(&temp_path, &real_path).await?; - // set nar url in narinfo - nar_info.url = real_path.as_ref(); + async fn make_nar(&self, sink: &mut File) -> Result<()> { + let nar = nar::open(sink).await?; + let path = self.path.absolute_path(); + let metadata = File::open(&path).await?.metadata().await?; - // upload narinfo - let narinfo_path = self.path.narinfo_path(); - debug!("uploading narinfo: {}", narinfo_path); - trace!("narinfo: {:#}", nar_info); - s3.put(&narinfo_path, nar_info.to_string().into()).await?; + if metadata.is_symlink() { + let target = read_link(&path).await?; + nar.symlink(target.as_os_str().as_encoded_bytes()).await; + } else if metadata.is_dir() { + let mut nar = nar.directory().await?; + nar_from_dir(path.into(), &mut nar).await; + nar.close().await; + } else if metadata.is_file() { + let perms = metadata.permissions().mode(); + let mut executable = false; + if (perms & 0o700) == 0o700 { + executable = true; + } + + let mut file = BufReader::new(File::open(&path).await?); + nar.file(executable, metadata.len(), &mut file).await; + } Ok(()) } } -/// calculate url where the compressed nar should be uploaded -fn nar_url(file_hash: &[u8]) -> Path { - let compressed_nar_hash = nixbase32::encode(file_hash); - Path::parse(format!("nar/{compressed_nar_hash}.nar.zst")) - .expect("should parse to a valid object_store::path::Path") +async fn nar_from_dir(path: PathBuf, node: &mut nar::Directory<'_, '_>) -> Result<()> { + let root = ReadDirStream::new(read_dir(&path).await?); + let entries = root + .map_ok(|x| (x.file_name(), x)) + .try_collect::>() + .await?; + + // directory entries must be written in ascending order of name + for (name, entry) in entries.iter() { + let node = node.entry(name.as_encoded_bytes()).await?; + let metadata = entry.metadata().await?; + + if metadata.is_symlink() { + let target = read_link(entry.path()).await?; + node.symlink(target.as_os_str().as_encoded_bytes()).await; + } else if metadata.is_dir() { + let mut node = node.directory().await?; + Box::pin(nar_from_dir(entry.path(), &mut node)).await; + node.close().await; + } else if metadata.is_file() { + let perms = metadata.permissions().mode(); + let mut executable = false; + if (perms & 0o700) == 0o700 { + executable = true; + } + + let mut file = BufReader::new(File::open(entry.path()).await?); + node.file(executable, metadata.len(), &mut file).await; + } + } + Ok(()) +} + +fn compress_and_hash_nar( + nar_file: File, + nar_hasher: &mut Sha256, + compressed_nar_hasher: &mut Sha256, +) -> impl AsyncRead { + let nar_reader = InspectReader::new(nar_file, |x| nar_hasher.update(x)); + let nar_buf_reader = BufReader::new(nar_reader); + + let compressed_nar_reader = ZstdEncoder::with_quality(nar_buf_reader, Level::Default); + InspectReader::new(compressed_nar_reader, |x| compressed_nar_hasher.update(x)) +} + +fn nar_url(compressed_nar_hash: &[u8]) -> String { + format!("nar/{}.nar.zst", nixbase32::encode(compressed_nar_hash)) +} + +async fn read_buf_nar(stream: &mut S, mut buf: BytesMut) -> Result { + while buf.len() < buf.capacity() { + let n = read_buf(stream, &mut buf).await?; + + if n == 0 { + break; + } + } + Ok(buf.freeze()) } diff --git a/tests/common/mod.rs b/tests/common/mod.rs deleted file mode 100644 index 4c2d932..0000000 --- a/tests/common/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -#![allow(dead_code)] - -use std::process::Command; -use std::sync::Arc; - -use nixcp::store::Store; - -pub const HELLO: &str = "github:nixos/nixpkgs?ref=f771eb401a46846c1aebd20552521b233dd7e18b#hello"; -pub const HELLO_DRV: &str = "/nix/store/iqbwkm8mjjjlmw6x6ry9rhzin2cp9372-hello-2.12.1.drv"; -pub const HELLO_PATH: &str = "/nix/store/9bwryidal9q3g91cjm6xschfn4ikd82q-hello-2.12.1"; -pub const NIXCP_PKG: &str = "github:cything/nixcp?ref=6cfe67af0e8da502702b31f34a941753e64d9561"; -pub const NIXCP_DRV: &str = "/nix/store/ldjvf9qjp980dyvka2hj99q4c0w6901x-nixcp-0.1.0.drv"; - -pub struct Context { - pub store: Arc, -} - -impl Context { - fn new() -> Self { - // hello must be in the store - ensure_exists(HELLO); - let store = Arc::new(Store::connect().expect("connect to nix store")); - Self { store } - } -} - -pub fn context() -> Context { - Context::new() -} - -pub fn ensure_exists(pkg: &str) { - Command::new("nix") - .arg("build") - .arg("--no-link") - .arg(pkg) - .status() - .unwrap(); -} diff --git a/tests/nar.rs b/tests/nar.rs deleted file mode 100644 index 5ebadc5..0000000 --- a/tests/nar.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::common::HELLO_PATH; -use nix_compat::nixbase32; -use nixcp::make_nar::MakeNar; -use nixcp::path_info::PathInfo; -use sha2::Digest; -use tokio::io::AsyncReadExt; - -mod common; - -#[tokio::test] -async fn nar_size_and_hash() { - let ctx = common::context(); - let path_info = PathInfo::from_path(HELLO_PATH, &ctx.store).await.unwrap(); - - let mut nar = MakeNar::new(&path_info, ctx.store).unwrap(); - let mut reader = nar.compress_and_hash().unwrap(); - let mut buf = Vec::new(); - reader.read_to_end(&mut buf).await.unwrap(); - drop(reader); - - assert_eq!(nar.nar_size, 234680); - - let nar_hash = nar.nar_hasher.finalize(); - let real_nar_hash = "08za7nnjda8kpdsd73v3mhykjvp0rsmskwsr37winhmzgm6iw79w"; - assert_eq!(nixbase32::encode(nar_hash.as_slice()), real_nar_hash); -} diff --git a/tests/path_info.rs b/tests/path_info.rs deleted file mode 100644 index d71f9d6..0000000 --- a/tests/path_info.rs +++ /dev/null @@ -1,97 +0,0 @@ -use nixcp::path_info::PathInfo; -use std::{collections::HashSet, path::PathBuf, process::Command}; - -use tempfile::TempDir; - -use crate::common::{HELLO, HELLO_DRV, HELLO_PATH, NIXCP_DRV, NIXCP_PKG}; - -mod common; - -#[tokio::test] -async fn path_info_from_package() { - let ctx = common::context(); - let path = PathBuf::from(HELLO); - let path_info = PathInfo::from_derivation(&path, &ctx.store) - .await - .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); -} - -#[tokio::test] -async fn path_info_from_path() { - let ctx = common::context(); - let path = PathBuf::from(HELLO_PATH); - let path_info = PathInfo::from_derivation(&path, &ctx.store) - .await - .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); -} - -#[tokio::test] -async fn path_info_symlink() { - let ctx = common::context(); - - let temp_path = TempDir::new().unwrap(); - let link_path = temp_path.path().join("result"); - - // symlink at ./result (like `nix build`) - std::os::unix::fs::symlink(HELLO_PATH, &link_path).unwrap(); - - // should resolve symlink - let path_info = PathInfo::from_derivation(&link_path, &ctx.store) - .await - .expect("get pathinfo from package"); - assert_eq!(path_info.path.to_absolute_path(), HELLO_DRV); -} - -#[tokio::test] -async fn closure_includes_nix_store_requisites() { - let ctx = common::context(); - let path = PathBuf::from(HELLO); - let path_info = PathInfo::from_derivation(&path, &ctx.store) - .await - .expect("get pathinfo from package"); - - // get what we think is the closure - let mut closure: HashSet = path_info - .get_closure(&ctx.store) - .await - .unwrap() - .iter() - .map(|x| x.path.to_absolute_path()) - .collect(); - - // for a somewhat more complicated case - common::ensure_exists(NIXCP_PKG); - let path = PathBuf::from(NIXCP_PKG); - let path_info = PathInfo::from_derivation(&path, &ctx.store) - .await - .expect("get pathinfo from package"); - closure.extend( - path_info - .get_closure(&ctx.store) - .await - .unwrap() - .iter() - .map(|x| x.path.to_absolute_path()), - ); - - // get output of `nix-store --query --requisites --include-outputs` - let nix_store_out = Command::new("nix-store") - .arg("--query") - .arg("--requisites") - .arg("--include-outputs") - .arg(HELLO_DRV) - .arg(NIXCP_DRV) - .output() - .unwrap() - .stdout; - assert!(!nix_store_out.is_empty()); - let ref_closure = String::from_utf8_lossy(&nix_store_out); - let ref_closure = ref_closure.split_whitespace(); - - // check that we didn't miss anything nix-store would catch - for path in ref_closure { - assert!(closure.contains(path)); - } -}