Compare commits
3 commits
48e44628fb
...
b47a778b9e
Author | SHA1 | Date | |
---|---|---|---|
b47a778b9e | |||
f304dae207 | |||
2f9cc77b42 |
3 changed files with 36 additions and 28 deletions
45
src/main.rs
45
src/main.rs
|
@ -2,22 +2,34 @@
|
||||||
#![feature(extend_one)]
|
#![feature(extend_one)]
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Args, Parser, Subcommand};
|
||||||
use tracing_subscriber::{EnvFilter, FmtSubscriber};
|
use tracing_subscriber::{EnvFilter, FmtSubscriber};
|
||||||
|
|
||||||
use nixcp::NixCp;
|
use push::Push;
|
||||||
|
|
||||||
mod cli;
|
mod cli;
|
||||||
mod nixcp;
|
|
||||||
mod path_info;
|
mod path_info;
|
||||||
|
mod push;
|
||||||
mod uploader;
|
mod uploader;
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
#[command(version, name = "nixcp")]
|
#[command(version)]
|
||||||
|
#[command(name = "nixcp")]
|
||||||
|
#[command(about = "Upload store paths to a s3 binary cache")]
|
||||||
|
#[command(long_about = None)]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
command: Commands,
|
command: Commands,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Subcommand)]
|
||||||
|
enum Commands {
|
||||||
|
#[command(arg_required_else_help = true)]
|
||||||
|
Push(PushArgs),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Args)]
|
||||||
|
pub struct PushArgs {
|
||||||
/// The s3 bucket to upload to
|
/// The s3 bucket to upload to
|
||||||
#[arg(long, value_name = "bucket name")]
|
#[arg(long, value_name = "bucket name")]
|
||||||
bucket: String,
|
bucket: String,
|
||||||
|
@ -44,16 +56,14 @@ struct Cli {
|
||||||
/// AWS profile to use
|
/// AWS profile to use
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
profile: Option<String>,
|
profile: Option<String>,
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Subcommand)]
|
#[arg(long)]
|
||||||
enum Commands {
|
skip_signature_check: bool,
|
||||||
Push {
|
|
||||||
/// Package or store path to upload
|
/// Package or store path to upload
|
||||||
/// e.g. nixpkgs#hello or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1
|
/// e.g. nixpkgs#hello or /nix/store/y4qpcibkj767szhjb58i2sidmz8m24hb-hello-2.12.1
|
||||||
#[arg(value_name = "package or store path")]
|
#[arg(value_name = "package or store path")]
|
||||||
package: String,
|
package: String,
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
|
@ -63,15 +73,14 @@ async fn main() -> Result<()> {
|
||||||
tracing::subscriber::set_global_default(subscriber)?;
|
tracing::subscriber::set_global_default(subscriber)?;
|
||||||
|
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
let nixcp = Box::leak(Box::new(NixCp::new(&cli).await?));
|
|
||||||
|
|
||||||
match &cli.command {
|
match &cli.command {
|
||||||
Commands::Push { package } => {
|
Commands::Push(cli) => {
|
||||||
nixcp
|
let push = Box::leak(Box::new(Push::new(cli).await?));
|
||||||
.paths_from_package(package)
|
push.paths_from_package(&cli.package)
|
||||||
.await
|
.await
|
||||||
.context("nixcp get paths from package")?;
|
.context("nixcp get paths from package")?;
|
||||||
nixcp.run().await.context("nixcp run")?;
|
push.run().await.context("nixcp run")?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -125,13 +125,13 @@ impl PathInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn check_if_already_exists(&self, s3_client: &s3::Client, bucket: String) -> bool {
|
pub async fn check_if_already_exists(&self, s3_client: &s3::Client, bucket: String) -> bool {
|
||||||
!s3_client
|
s3_client
|
||||||
.head_object()
|
.head_object()
|
||||||
.bucket(bucket)
|
.bucket(bucket)
|
||||||
.key(format!("{}.narinfo", self.digest()))
|
.key(format!("{}.narinfo", self.digest()))
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.is_err()
|
.is_ok()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,9 +16,9 @@ use tokio::sync::{RwLock, Semaphore, mpsc};
|
||||||
use tracing::{debug, info, trace};
|
use tracing::{debug, info, trace};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::{Cli, path_info::PathInfo, uploader::Uploader};
|
use crate::{PushArgs, path_info::PathInfo, uploader::Uploader};
|
||||||
|
|
||||||
pub struct NixCp {
|
pub struct Push {
|
||||||
upstream_caches: Vec<Url>,
|
upstream_caches: Vec<Url>,
|
||||||
store_paths: Arc<RwLock<Vec<PathInfo>>>,
|
store_paths: Arc<RwLock<Vec<PathInfo>>>,
|
||||||
s3_client: s3::Client,
|
s3_client: s3::Client,
|
||||||
|
@ -32,8 +32,8 @@ pub struct NixCp {
|
||||||
already_exists_count: AtomicUsize,
|
already_exists_count: AtomicUsize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NixCp {
|
impl Push {
|
||||||
pub async fn new(cli: &Cli) -> Result<Self> {
|
pub async fn new(cli: &PushArgs) -> Result<Self> {
|
||||||
let mut upstreams = Vec::with_capacity(cli.upstreams.len() + 1);
|
let mut upstreams = Vec::with_capacity(cli.upstreams.len() + 1);
|
||||||
for upstream in cli
|
for upstream in cli
|
||||||
.upstreams
|
.upstreams
|
||||||
|
@ -160,11 +160,10 @@ impl NixCp {
|
||||||
self.bucket.clone(),
|
self.bucket.clone(),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
let fut = tokio::spawn({
|
uploads.push(tokio::spawn(async move {
|
||||||
let _permit = permits.acquire().await.unwrap();
|
let _permit = permits.acquire().await.unwrap();
|
||||||
async move { uploader.upload().await }
|
uploader.upload().await
|
||||||
});
|
}));
|
||||||
uploads.push(fut);
|
|
||||||
} else {
|
} else {
|
||||||
join_all(uploads)
|
join_all(uploads)
|
||||||
.await
|
.await
|
Loading…
Add table
Add a link
Reference in a new issue