bring code from attic

src/bindings/mod.rs

@@ -0,0 +1,270 @@
+/*
+Copyright 2022 Zhaofeng Li and the Attic contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//! `libnixstore` Bindings
+
+use std::cell::UnsafeCell;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use futures::stream::{Stream, StreamExt};
+use tokio::io::{AsyncWrite, AsyncWriteExt};
+
+use crate::{AtticError, AtticResult};
+
+// The C++ implementation takes care of concurrency
+#[repr(transparent)]
+pub struct FfiNixStore(UnsafeCell<cxx::UniquePtr<ffi::CNixStore>>);
+
+unsafe impl Send for FfiNixStore {}
+unsafe impl Sync for FfiNixStore {}
+
+impl FfiNixStore {
+    pub fn store(&self) -> Pin<&mut ffi::CNixStore> {
+        unsafe {
+            let ptr = self.0.get().as_mut().unwrap();
+            ptr.pin_mut()
+        }
+    }
+}
+
+/// Obtain a handle to the Nix store.
+pub unsafe fn open_nix_store() -> AtticResult<FfiNixStore> {
+    match ffi::open_nix_store() {
+        Ok(ptr) => {
+            let cell = UnsafeCell::new(ptr);
+            Ok(FfiNixStore(cell))
+        }
+        Err(e) => Err(e.into()),
+    }
+}
+
+// TODO: Benchmark different implementations
+// (tokio, crossbeam, flume)
+mod mpsc {
+    // Tokio
+    pub use tokio::sync::mpsc::{
+        UnboundedReceiver, UnboundedSender, error::SendError, unbounded_channel,
+    };
+}
+
+/// Async write request.
+#[derive(Debug)]
+enum AsyncWriteMessage {
+    Data(Vec<u8>),
+    Error(String),
+    Eof,
+}
+
+/// Async write request sender.
+#[derive(Clone)]
+pub struct AsyncWriteSender {
+    sender: mpsc::UnboundedSender<AsyncWriteMessage>,
+}
+
+impl AsyncWriteSender {
+    fn send(&mut self, data: &[u8]) -> Result<(), mpsc::SendError<AsyncWriteMessage>> {
+        let message = AsyncWriteMessage::Data(Vec::from(data));
+        self.sender.send(message)
+    }
+
+    fn eof(&mut self) -> Result<(), mpsc::SendError<AsyncWriteMessage>> {
+        let message = AsyncWriteMessage::Eof;
+        self.sender.send(message)
+    }
+
+    pub(crate) fn rust_error(
+        &mut self,
+        error: impl std::error::Error,
+    ) -> Result<(), impl std::error::Error> {
+        let message = AsyncWriteMessage::Error(error.to_string());
+        self.sender.send(message)
+    }
+}
+
+/// A wrapper of the `AsyncWrite` trait for the synchronous Nix C++ land.
+pub struct AsyncWriteAdapter {
+    receiver: mpsc::UnboundedReceiver<AsyncWriteMessage>,
+    eof: bool,
+}
+
+impl AsyncWriteAdapter {
+    pub fn new() -> (Self, Box<AsyncWriteSender>) {
+        let (sender, receiver) = mpsc::unbounded_channel();
+
+        let r = Self {
+            receiver,
+            eof: false,
+        };
+        let sender = Box::new(AsyncWriteSender { sender });
+
+        (r, sender)
+    }
+
+    /// Write everything the sender sends to us.
+    pub async fn write_all(mut self, mut writer: Box<dyn AsyncWrite + Unpin>) -> AtticResult<()> {
+        let writer = writer.as_mut();
+
+        while let Some(data) = self.next().await {
+            match data {
+                Ok(v) => {
+                    writer.write_all(&v).await?;
+                }
+                Err(e) => {
+                    return Err(e);
+                }
+            }
+        }
+
+        if !self.eof {
+            Err(io::Error::from(io::ErrorKind::BrokenPipe).into())
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl Stream for AsyncWriteAdapter {
+    type Item = AtticResult<Vec<u8>>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        match self.receiver.poll_recv(cx) {
+            Poll::Pending => Poll::Pending,
+            Poll::Ready(Some(message)) => {
+                use AsyncWriteMessage::*;
+                match message {
+                    Data(v) => Poll::Ready(Some(Ok(v))),
+                    Error(exception) => {
+                        let error = AtticError::CxxError { exception };
+                        Poll::Ready(Some(Err(error)))
+                    }
+                    Eof => {
+                        self.eof = true;
+                        Poll::Ready(None)
+                    }
+                }
+            }
+            Poll::Ready(None) => {
+                if !self.eof {
+                    Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe).into())))
+                } else {
+                    Poll::Ready(None)
+                }
+            }
+        }
+    }
+}
+
+#[cxx::bridge]
+/// Generated by `cxx.rs`.
+///
+/// Mid-level wrapper of `libnixstore` implemented in C++.
+mod ffi {
+    extern "Rust" {
+        type AsyncWriteSender;
+        fn send(self: &mut AsyncWriteSender, data: &[u8]) -> Result<()>;
+        fn eof(self: &mut AsyncWriteSender) -> Result<()>;
+    }
+
+    unsafe extern "C++" {
+        include!("attic/src/nix_store/bindings/nix.hpp");
+
+        // =========
+        // CNixStore
+        // =========
+
+        /// Mid-level wrapper for the Unix Domain Socket Nix Store.
+        type CNixStore;
+
+        /// Returns the path of the Nix store itself.
+        fn store_dir(self: Pin<&mut CNixStore>) -> String;
+
+        /*
+        /// Verifies that a path is indeed in the Nix store, then return the base store path.
+        ///
+        /// Use parse_store_path instead.
+        fn to_store_path(self: Pin<&mut CNixStore>, path: &str) -> Result<String>;
+        */
+
+        /// Queries information about a valid path.
+        fn query_path_info(
+            self: Pin<&mut CNixStore>,
+            store_path: &[u8],
+        ) -> Result<UniquePtr<CPathInfo>>;
+
+        /// Computes the closure of a valid path.
+        ///
+        /// If `flip_directions` is true, the set of paths that can reach `store_path` is
+        /// returned.
+        fn compute_fs_closure(
+            self: Pin<&mut CNixStore>,
+            store_path: &[u8],
+            flip_direction: bool,
+            include_outputs: bool,
+            include_derivers: bool,
+        ) -> Result<UniquePtr<CxxVector<CxxString>>>;
+
+        /// Computes the closure of a list of valid paths.
+        ///
+        /// This is the multi-path variant of `compute_fs_closure`.
+        /// If `flip_directions` is true, the set of paths that can reach `store_path` is
+        /// returned.
+        ///
+        /// It's easier and more efficient to just pass a vector of slices
+        /// instead of wrangling with concrete "extern rust" / "extern C++"
+        /// types.
+        fn compute_fs_closure_multi(
+            self: Pin<&mut CNixStore>,
+            base_names: &[&[u8]],
+            flip_direction: bool,
+            include_outputs: bool,
+            include_derivers: bool,
+        ) -> Result<UniquePtr<CxxVector<CxxString>>>;
+
+        /// Creates a NAR dump from a path.
+        fn nar_from_path(
+            self: Pin<&mut CNixStore>,
+            base_name: Vec<u8>,
+            sender: Box<AsyncWriteSender>,
+        ) -> Result<()>;
+
+        /// Obtains a handle to the Nix store.
+        fn open_nix_store() -> Result<UniquePtr<CNixStore>>;
+
+        // =========
+        // CPathInfo
+        // =========
+
+        /// Mid-level wrapper for the `nix::ValidPathInfo` struct.
+        type CPathInfo;
+
+        /// Returns the SHA-256 hash of the store path.
+        fn nar_sha256_hash(self: Pin<&mut CPathInfo>) -> &[u8];
+
+        /// Returns the size of the NAR.
+        fn nar_size(self: Pin<&mut CPathInfo>) -> u64;
+
+        /// Returns the references of the store path.
+        fn references(self: Pin<&mut CPathInfo>) -> UniquePtr<CxxVector<CxxString>>;
+
+        /// Returns the possibly invalid signatures attached to the store path.
+        fn sigs(self: Pin<&mut CPathInfo>) -> UniquePtr<CxxVector<CxxString>>;
+
+        /// Returns the CA field of the store path.
+        fn ca(self: Pin<&mut CPathInfo>) -> String;
+    }
+}

src/bindings/nix.cpp

@@ -0,0 +1,173 @@
+/*
+Copyright 2022 Zhaofeng Li and the Attic contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// C++ side of the libnixstore glue.
+//
+// We implement a mid-level wrapper of the Nix Store interface,
+// which is then wrapped again in the Rust side to enable full
+// async-await operation.
+//
+// Here we stick with the naming conventions of Rust and handle
+// Rust types directly where possible, so that the interfaces are
+// satisfying to use from the Rust side via cxx.rs.
+
+#include "attic/src/nix_store/bindings/nix.hpp"
+
+static std::mutex g_init_nix_mutex;
+static bool g_init_nix_done = false;
+
+static nix::StorePath store_path_from_rust(RBasePathSlice base_name) {
+	std::string_view sv((const char *)base_name.data(), base_name.size());
+	return nix::StorePath(sv);
+}
+
+static bool hash_is_sha256(const nix::Hash &hash) {
+#ifdef ATTIC_NIX_2_20
+	return hash.algo == nix::HashAlgorithm::SHA256;
+#else
+	return hash.type == nix::htSHA256;
+#endif
+}
+
+// ========
+// RustSink
+// ========
+
+RustSink::RustSink(RBox<AsyncWriteSender> sender) : sender(std::move(sender)) {}
+
+void RustSink::operator () (std::string_view data) {
+	RBasePathSlice s((const unsigned char *)data.data(), data.size());
+
+	this->sender->send(s);
+}
+
+void RustSink::eof() {
+	this->sender->eof();
+}
+
+
+// =========
+// CPathInfo
+// =========
+
+CPathInfo::CPathInfo(nix::ref<const nix::ValidPathInfo> pi) : pi(pi) {}
+
+RHashSlice CPathInfo::nar_sha256_hash() {
+	auto &hash = this->pi->narHash;
+
+	if (!hash_is_sha256(hash)) {
+		throw nix::Error("Only SHA-256 hashes are supported at the moment");
+	}
+
+	return RHashSlice(hash.hash, hash.hashSize);
+}
+
+uint64_t CPathInfo::nar_size() {
+	return this->pi->narSize;
+}
+
+std::unique_ptr<std::vector<std::string>> CPathInfo::sigs() {
+	std::vector<std::string> result;
+	for (auto&& elem : this->pi->sigs) {
+		result.push_back(std::string(elem));
+	}
+	return std::make_unique<std::vector<std::string>>(result);
+}
+
+std::unique_ptr<std::vector<std::string>> CPathInfo::references() {
+	std::vector<std::string> result;
+	for (auto&& elem : this->pi->references) {
+		result.push_back(std::string(elem.to_string()));
+	}
+	return std::make_unique<std::vector<std::string>>(result);
+}
+
+RString CPathInfo::ca() {
+	if (this->pi->ca) {
+		return RString(nix::renderContentAddress(this->pi->ca));
+	} else {
+		return RString("");
+	}
+}
+
+// =========
+// CNixStore
+// =========
+
+CNixStore::CNixStore() {
+	std::map<std::string, std::string> params;
+	std::lock_guard<std::mutex> lock(g_init_nix_mutex);
+
+	if (!g_init_nix_done) {
+		nix::initNix();
+		g_init_nix_done = true;
+	}
+
+	this->store = nix::openStore(nix::settings.storeUri.get(), params);
+}
+
+RString CNixStore::store_dir() {
+	return RString(this->store->storeDir);
+}
+
+std::unique_ptr<CPathInfo> CNixStore::query_path_info(RBasePathSlice base_name) {
+	auto store_path = store_path_from_rust(base_name);
+
+	auto r = this->store->queryPathInfo(store_path);
+	return std::make_unique<CPathInfo>(r);
+}
+
+std::unique_ptr<std::vector<std::string>> CNixStore::compute_fs_closure(RBasePathSlice base_name, bool flip_direction, bool include_outputs, bool include_derivers) {
+	std::set<nix::StorePath> out;
+
+	this->store->computeFSClosure(store_path_from_rust(base_name), out, flip_direction, include_outputs, include_derivers);
+
+	std::vector<std::string> result;
+	for (auto&& elem : out) {
+		result.push_back(std::string(elem.to_string()));
+	}
+	return std::make_unique<std::vector<std::string>>(result);
+}
+
+std::unique_ptr<std::vector<std::string>> CNixStore::compute_fs_closure_multi(RSlice<const RBasePathSlice> base_names, bool flip_direction, bool include_outputs, bool include_derivers) {
+	std::set<nix::StorePath> path_set, out;
+	for (auto&& base_name : base_names) {
+		path_set.insert(store_path_from_rust(base_name));
+	}
+
+	this->store->computeFSClosure(path_set, out, flip_direction, include_outputs, include_derivers);
+
+	std::vector<std::string> result;
+	for (auto&& elem : out) {
+		result.push_back(std::string(elem.to_string()));
+	}
+	return std::make_unique<std::vector<std::string>>(result);
+}
+
+void CNixStore::nar_from_path(RVec<unsigned char> base_name, RBox<AsyncWriteSender> sender) {
+	RustSink sink(std::move(sender));
+
+	std::string_view sv((const char *)base_name.data(), base_name.size());
+	nix::StorePath store_path(sv);
+
+	// exceptions will be thrown into Rust
+	this->store->narFromPath(store_path, sink);
+	sink.eof();
+}
+
+std::unique_ptr<CNixStore> open_nix_store() {
+	return std::make_unique<CNixStore>();
+}

src/bindings/nix.hpp

@@ -0,0 +1,96 @@
+/*
+Copyright 2022 Zhaofeng Li and the Attic contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// C++ side of the libnixstore glue.
+//
+// We implement a mid-level wrapper of the Nix Store interface,
+// which is then wrapped again in the Rust side to enable full
+// async-await operation.
+//
+// Here we stick with the naming conventions of Rust and handle
+// Rust types directly where possible, so that the interfaces are
+// satisfying to use from the Rust side via cxx.rs.
+
+#pragma once
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <nix/store-api.hh>
+#include <nix/local-store.hh>
+#include <nix/remote-store.hh>
+#include <nix/uds-remote-store.hh>
+#include <nix/hash.hh>
+#include <nix/path.hh>
+#include <nix/serialise.hh>
+#include <nix/shared.hh>
+#include <rust/cxx.h>
+
+template<class T> using RVec = rust::Vec<T>;
+template<class T> using RBox = rust::Box<T>;
+template<class T> using RSlice = rust::Slice<T>;
+using RString = rust::String;
+using RStr = rust::Str;
+using RBasePathSlice = RSlice<const unsigned char>;
+using RHashSlice = RSlice<const unsigned char>;
+
+struct AsyncWriteSender;
+
+struct RustSink : nix::Sink
+{
+	RBox<AsyncWriteSender> sender;
+public:
+	RustSink(RBox<AsyncWriteSender> sender);
+	void operator () (std::string_view data) override;
+	void eof();
+};
+
+// Opaque wrapper for nix::ValidPathInfo
+class CPathInfo {
+	nix::ref<const nix::ValidPathInfo> pi;
+public:
+	CPathInfo(nix::ref<const nix::ValidPathInfo> pi);
+	RHashSlice nar_sha256_hash();
+	uint64_t nar_size();
+	std::unique_ptr<std::vector<std::string>> sigs();
+	std::unique_ptr<std::vector<std::string>> references();
+	RString ca();
+};
+
+class CNixStore {
+	std::shared_ptr<nix::Store> store;
+public:
+	CNixStore();
+
+	RString store_dir();
+	std::unique_ptr<CPathInfo> query_path_info(RBasePathSlice base_name);
+	std::unique_ptr<std::vector<std::string>> compute_fs_closure(
+		RBasePathSlice base_name,
+		bool flip_direction,
+		bool include_outputs,
+		bool include_derivers);
+	std::unique_ptr<std::vector<std::string>> compute_fs_closure_multi(
+		RSlice<const RBasePathSlice> base_names,
+		bool flip_direction,
+		bool include_outputs,
+		bool include_derivers);
+	void nar_from_path(RVec<unsigned char> base_name, RBox<AsyncWriteSender> sender);
+};
+
+std::unique_ptr<CNixStore> open_nix_store();
+
+// Relies on our definitions
+#include "attic/src/nix_store/bindings/mod.rs.h"