From 85fe97c35a6cbe6b1bf50446675d3539d2dddde1 Mon Sep 17 00:00:00 2001 From: cy Date: Thu, 27 Mar 2025 14:42:22 -0400 Subject: [PATCH 1/7] test s3 cache --- .../workflows/build-machines-and-homes.yml | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-machines-and-homes.yml b/.github/workflows/build-machines-and-homes.yml index 1900b0e..e9795a6 100644 --- a/.github/workflows/build-machines-and-homes.yml +++ b/.github/workflows/build-machines-and-homes.yml @@ -7,12 +7,13 @@ env: ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} NIX_CONFIG: | show-trace = true - extra-substituters = https://cdn.cy7.sh/main - extra-trusted-public-keys = main:Ku31HoEWcBtfggge2VGj+QTkVrQuIwRIMGyfV/5VQP0= experimental-features = nix-command flakes accept-flake-config = true system-features = nixos-test benchmark big-parallel kvm + secret-key-files = ~/cache-priv-key.pem TERM: ansi + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }} jobs: build-machines: strategy: @@ -36,16 +37,14 @@ jobs: remove-codeql: 'true' remove-docker-images: 'true' build-mount-path: /nix + - name: setup binary cache key + run: echo "${{ secrets.NIX_CACHE_SECRET_KEY }}" >> ~/cache-priv-key.pem - name: Install Nix uses: cachix/install-nix-action@v30 - name: Sync repository uses: actions/checkout@v4 with: persist-credentials: false - - name: setup attic - run: | - nix profile install github:zhaofengli/attic - attic login cy7 https://cache.cy7.sh "$ATTIC_TOKEN" - name: build run: | package=".#nixosConfigurations."${{ matrix.machine }}".config.system.build.toplevel" @@ -53,10 +52,7 @@ jobs: - name: cache if: always() run: | - package=".#nixosConfigurations."${{ matrix.machine }}".config.system.build.toplevel" - derivation="$(nix path-info --derivation "$package")" - cache="$(nix-store --query --requisites --include-outputs "$derivation")" - xargs attic push main <<< "$cache" + nix copy ".#nixosConfigurations."${{ matrix.machine }}".config.system.build.toplevel" --to "s3://nixcache?endpoint=s3.cy7.sh" build-homes: strategy: fail-fast: false @@ -80,16 +76,14 @@ jobs: remove-codeql: 'true' remove-docker-images: 'true' build-mount-path: /nix + - name: setup binary cache key + run: echo "${{ secrets.NIX_CACHE_SECRET_KEY }}" >> ~/cache-priv-key.pem - name: Install Nix uses: cachix/install-nix-action@v30 - name: Sync repository uses: actions/checkout@v4 with: persist-credentials: false - - name: setup attic - run: | - nix profile install github:zhaofengli/attic - attic login cy7 https://cache.cy7.sh "$ATTIC_TOKEN" - name: build run: | package=".#homeConfigurations."${{ matrix.home }}".activationPackage" @@ -97,7 +91,4 @@ jobs: - name: cache if: always() run: | - package=".#homeConfigurations."${{ matrix.home }}".activationPackage" - derivation="$(nix path-info --derivation "$package")" - cache="$(nix-store --query --requisites --include-outputs "$derivation")" - xargs attic push main <<< "$cache" + nix copy ".#homeConfigurations."${{ matrix.home }}".activationPackage" --to "s3://nixcache?endpoint=s3.cy7.sh" From 146c61cea1e6d0317c9fbbe10d60302f1e837f36 Mon Sep 17 00:00:00 2001 From: cy Date: Thu, 27 Mar 2025 15:00:12 -0400 Subject: [PATCH 2/7] add cache priv and pub key --- .github/workflows/build-machines-and-homes.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-machines-and-homes.yml b/.github/workflows/build-machines-and-homes.yml index e9795a6..e8bb79c 100644 --- a/.github/workflows/build-machines-and-homes.yml +++ b/.github/workflows/build-machines-and-homes.yml @@ -11,6 +11,8 @@ env: accept-flake-config = true system-features = nixos-test benchmark big-parallel kvm secret-key-files = ~/cache-priv-key.pem + extra-substituters = https://nixcache.cy7.sh + extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= TERM: ansi AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }} From 448ac0cedf5a2b825cc013034dc2e839cba96e36 Mon Sep 17 00:00:00 2001 From: cy Date: Thu, 27 Mar 2025 15:09:38 -0400 Subject: [PATCH 3/7] add cache priv key to /home/runner --- .github/workflows/build-machines-and-homes.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-machines-and-homes.yml b/.github/workflows/build-machines-and-homes.yml index e8bb79c..3b36789 100644 --- a/.github/workflows/build-machines-and-homes.yml +++ b/.github/workflows/build-machines-and-homes.yml @@ -10,7 +10,7 @@ env: experimental-features = nix-command flakes accept-flake-config = true system-features = nixos-test benchmark big-parallel kvm - secret-key-files = ~/cache-priv-key.pem + secret-key-files = /home/runner/cache-priv-key.pem extra-substituters = https://nixcache.cy7.sh extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= TERM: ansi @@ -40,7 +40,7 @@ jobs: remove-docker-images: 'true' build-mount-path: /nix - name: setup binary cache key - run: echo "${{ secrets.NIX_CACHE_SECRET_KEY }}" >> ~/cache-priv-key.pem + run: echo "${{ secrets.NIX_CACHE_SECRET_KEY }}" >> /home/runner/cache-priv-key.pem - name: Install Nix uses: cachix/install-nix-action@v30 - name: Sync repository @@ -54,7 +54,7 @@ jobs: - name: cache if: always() run: | - nix copy ".#nixosConfigurations."${{ matrix.machine }}".config.system.build.toplevel" --to "s3://nixcache?endpoint=s3.cy7.sh" + nix copy ".#nixosConfigurations."${{ matrix.machine }}".config.system.build.toplevel" --to "s3://nixcache?endpoint=s3.cy7.sh&secret-key=/home/runner/cache-priv-key.pem" build-homes: strategy: fail-fast: false @@ -79,7 +79,7 @@ jobs: remove-docker-images: 'true' build-mount-path: /nix - name: setup binary cache key - run: echo "${{ secrets.NIX_CACHE_SECRET_KEY }}" >> ~/cache-priv-key.pem + run: echo "${{ secrets.NIX_CACHE_SECRET_KEY }}" >> /home/runner/cache-priv-key.pem - name: Install Nix uses: cachix/install-nix-action@v30 - name: Sync repository @@ -93,4 +93,4 @@ jobs: - name: cache if: always() run: | - nix copy ".#homeConfigurations."${{ matrix.home }}".activationPackage" --to "s3://nixcache?endpoint=s3.cy7.sh" + nix copy ".#homeConfigurations."${{ matrix.home }}".activationPackage" --to "s3://nixcache?endpoint=s3.cy7.sh&secret-key=/home/runner/cache-priv-key.pem" From 694f3c5243ac6358a2785957571ac36d90b6adcc Mon Sep 17 00:00:00 2001 From: cy Date: Thu, 27 Mar 2025 15:50:06 -0400 Subject: [PATCH 4/7] workflow: use new cache for build-packages --- .github/workflows/build-packages.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 1d23fe9..2e926b2 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -10,10 +10,11 @@ env: ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} NIX_CONFIG: | show-trace = true - extra-substituters = https://cdn.cy7.sh/main - extra-trusted-public-keys = main:Ku31HoEWcBtfggge2VGj+QTkVrQuIwRIMGyfV/5VQP0= experimental-features = nix-command flakes accept-flake-config = true + secret-key-files = /home/runner/cache-priv-key.pem + extra-substituters = https://nixcache.cy7.sh + extra-trusted-public-keys = nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8= TERM: ansi jobs: build-packages: @@ -29,23 +30,19 @@ jobs: - macos-13 runs-on: ${{ matrix.os }} steps: + - name: setup binary cache key + run: echo "${{ secrets.NIX_CACHE_SECRET_KEY }}" >> /home/runner/cache-priv-key.pem - name: Install Nix uses: cachix/install-nix-action@v30 - name: Sync repository uses: actions/checkout@v4 with: persist-credentials: false - - name: setup attic - run: | - nix profile install github:zhaofengli/attic - attic login cy7 https://cache.cy7.sh "$ATTIC_TOKEN" - run: nix build -L ${{ matrix.package }} - name: cache result if: always() run: | - derivation="$(nix path-info --derivation "${{ matrix.package }}")" - cache="$(nix-store --query --requisites --include-outputs "$derivation")" - xargs attic push main <<< "$cache" + nix copy "${{ matrix.machine }}" --to "s3://nixcache?endpoint=s3.cy7.sh&secret-key=/home/runner/cache-priv-key.pem" - name: prepare tarball to upload run: nix run github:nixos/nixpkgs#gnutar hcvf result.tar result - name: upload result From ef2285b04beef24429b3674f29337b81f2ae0aa2 Mon Sep 17 00:00:00 2001 From: cy Date: Thu, 27 Mar 2025 16:39:33 -0400 Subject: [PATCH 5/7] use s3 nix cache --- flake.nix | 4 ++-- hosts/chunk/garage.nix | 12 ++++++++++++ hosts/common.nix | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index b5fccc8..52d012f 100644 --- a/flake.nix +++ b/flake.nix @@ -111,11 +111,11 @@ nixConfig = { extra-substituters = [ "https://nix-community.cachix.org" - "https://cdn.cy7.sh/main" + "https://nixcache.cy7.sh" ]; extra-trusted-public-keys = [ "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" - "main:Ku31HoEWcBtfggge2VGj+QTkVrQuIwRIMGyfV/5VQP0=" + "nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8=" ]; }; diff --git a/hosts/chunk/garage.nix b/hosts/chunk/garage.nix index 6a25df2..da9b650 100644 --- a/hosts/chunk/garage.nix +++ b/hosts/chunk/garage.nix @@ -10,6 +10,11 @@ api_bind_addr = "[::]:3900"; root_domain = "s3.cy7.sh"; }; + s3_web = { + bind_addr = "[::]:3902"; + root_domain = ".web.cy7.sh"; + add_host_to_metrics = true; + }; admin.api_bind_addr = "[::]:3903"; rpc_bind_addr = "[::]:3901"; replication_factor = 1; @@ -33,5 +38,12 @@ import common reverse_proxy localhost:3903 ''; + "*.web.cy7.sh" = { + serverAliases = [ "nixcache.cy7.sh" ]; + extraConfig = '' + import common + reverse_proxy localhost:3902 + ''; + }; }; } diff --git a/hosts/common.nix b/hosts/common.nix index 7067008..1d54545 100644 --- a/hosts/common.nix +++ b/hosts/common.nix @@ -11,11 +11,11 @@ ]; trusted-public-keys = [ "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" - "main:Ku31HoEWcBtfggge2VGj+QTkVrQuIwRIMGyfV/5VQP0=" + "nixcache.cy7.sh:DN3d1dt0wnXfTH03oVmTee4KgmdNdB0NY3SuzA8Fwx8=" ]; extra-substituters = [ "https://nix-community.cachix.org" - "https://cdn.cy7.sh/main" + "https://nixcache.cy7.sh" ]; }; channel.enable = false; From 96372deb31f992a29f118e4f4276afabc171572b Mon Sep 17 00:00:00 2001 From: cy Date: Thu, 27 Mar 2025 18:44:53 -0400 Subject: [PATCH 6/7] try to fix rclone memory issue; also disable OOM for now --- hosts/chunk/rclone.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hosts/chunk/rclone.nix b/hosts/chunk/rclone.nix index 31cdf54..1da2ad7 100644 --- a/hosts/chunk/rclone.nix +++ b/hosts/chunk/rclone.nix @@ -14,20 +14,22 @@ let --config ${config.sops.secrets."rclone/config".path} \ --allow-other \ --cache-dir /var/cache/rclone \ - --transfers 32 \ + --transfers 16 \ --vfs-cache-mode full \ --vfs-cache-min-free-space 5G \ --dir-cache-time 30d \ --no-checksum \ --no-modtime \ --vfs-fast-fingerprint \ - --vfs-read-chunk-size 4M \ - --vfs-read-chunk-streams 32 \ + --vfs-read-chunk-size 16M \ + --vfs-read-chunk-streams 16 \ --sftp-concurrency 128 \ --sftp-chunk-size 255k \ + --buffer-size 0 \ ${remote} ${mount} ''; ExecStop = "${lib.getExe' pkgs.fuse "fusermount"} -zu ${mount}"; + OOMScoreAdjust = -1000; }; in { From 90ccdfe4a30e7a1adde8463e23de6cd6494ff57e Mon Sep 17 00:00:00 2001 From: cy Date: Thu, 27 Mar 2025 18:45:14 -0400 Subject: [PATCH 7/7] sops: add binary cache priv key --- .sops.yaml | 5 +++++ secrets/cache-priv-key.pem | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 secrets/cache-priv-key.pem diff --git a/.sops.yaml b/.sops.yaml index cb7e65d..c812080 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -123,3 +123,8 @@ creation_rules: - age: - *chunk - *cy + - path_regex: secrets/cache-priv-key.pem + key_groups: + - age: + - *yt + - *cy diff --git a/secrets/cache-priv-key.pem b/secrets/cache-priv-key.pem new file mode 100644 index 0000000..30618df --- /dev/null +++ b/secrets/cache-priv-key.pem @@ -0,0 +1,24 @@ +{ + "data": "ENC[AES256_GCM,data:IVRg3IqrlV1Cy3xwyVszhUnRzbWP3OSb/XZF1H0N30eKL8d0DxFGngC5qMgRcmSs203/QL3w0fENp1u0f8tVajqJVlzLjlsiQrMdtXmiMv0LKO7E+aj4UZ0wMchB0XgSVUWrKUXxZrA=,iv:3GtA07yuAAI++RsLSwY3U62k1iG9+hvkGn45HjFt/Gk=,tag:PJ13CrjcE06KMC383txqHw==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": [ + { + "recipient": "age1sy0at69err83qyml2vqu8xvwjccfws447aaadfvacj2qluw3p45s2mtrw8", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVdkxBV3NSL1NDRjhPanpZ\nWTQwUmJmTExNOG0xZ1l1M2ZUSlFaTWdzS2hvCkdNT0dmK3FhcC85RE1mQmN1Uncw\nU1A4Znk1Vnk3RmY1UWdBY21VdnRZdWMKLS0tIHhTYUVUeTZsYkJFWk5LdnNDWlc5\ndDFDMUN2RmN0MDNXclpEMFA5Z0F5M0kKsLgc2D73RPNdTo4q7hBPGcBVPGwY73g+\nqQZbkVVzKPHo814ivwIVFYv+i5Qvf+p985Rko/fQ98GxW0G5c9Qfkg==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age10h6pg5qdpc4t0rpmksfv788a57f04n83zgqaezkjjn65nkhv547s0vxfdn", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuZ3B4czExQVZxNUJGNk1X\nbE9Md1Z6blM1S2xhb3VLT0JWbUpjSkxxS1U0Cm41bmt5Rk9LVTJFbVRFT2xxM01i\nTGUyYVU2RWdzRUhBVmNsTzJZWmh6cFUKLS0tIFlXVHQrcTVvbnptRitQVkVXRHBJ\nTjIxVXFvRmRQbUFHRXFjbFIva1IxVG8KciKyUdNjec7ocuKVX8KflMVPKpf/tEVr\nTxudivOoQ0XaqyPVi3cWDpuk2IAWUuJDxjmEctE6JgPtQvs1GsKCdg==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-03-27T20:37:28Z", + "mac": "ENC[AES256_GCM,data:35iUoodcjvIn+VAE20f2sHFaTh3+aqCYQ4HalWdVz4eUSkVNcdXs2uqOZtFv3MszDiH9izM84OyHwykudJ99QE3B/NwpfIoKQaU6Qg5X/g/rC1meffMaZwcASVbepjznahbTKmJqeSrMeybrBIV+6FaSjWXn0+D72GEEM1vgH9c=,iv:N2CbttHJsczm37qdapOCrlNeSSgsZBDlvWyvUpa3mkk=,tag:btniVwaVS9h4jDo4IM2wcA==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.9.4" + } +} \ No newline at end of file