diff --git a/src/bindings/mod.rs b/src/bindings/mod.rs new file mode 100644 index 0000000..521df2e --- /dev/null +++ b/src/bindings/mod.rs @@ -0,0 +1,270 @@ +/* +Copyright 2022 Zhaofeng Li and the Attic contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//! `libnixstore` Bindings + +use std::cell::UnsafeCell; +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use futures::stream::{Stream, StreamExt}; +use tokio::io::{AsyncWrite, AsyncWriteExt}; + +use crate::{AtticError, AtticResult}; + +// The C++ implementation takes care of concurrency +#[repr(transparent)] +pub struct FfiNixStore(UnsafeCell>); + +unsafe impl Send for FfiNixStore {} +unsafe impl Sync for FfiNixStore {} + +impl FfiNixStore { + pub fn store(&self) -> Pin<&mut ffi::CNixStore> { + unsafe { + let ptr = self.0.get().as_mut().unwrap(); + ptr.pin_mut() + } + } +} + +/// Obtain a handle to the Nix store. +pub unsafe fn open_nix_store() -> AtticResult { + match ffi::open_nix_store() { + Ok(ptr) => { + let cell = UnsafeCell::new(ptr); + Ok(FfiNixStore(cell)) + } + Err(e) => Err(e.into()), + } +} + +// TODO: Benchmark different implementations +// (tokio, crossbeam, flume) +mod mpsc { + // Tokio + pub use tokio::sync::mpsc::{ + UnboundedReceiver, UnboundedSender, error::SendError, unbounded_channel, + }; +} + +/// Async write request. +#[derive(Debug)] +enum AsyncWriteMessage { + Data(Vec), + Error(String), + Eof, +} + +/// Async write request sender. +#[derive(Clone)] +pub struct AsyncWriteSender { + sender: mpsc::UnboundedSender, +} + +impl AsyncWriteSender { + fn send(&mut self, data: &[u8]) -> Result<(), mpsc::SendError> { + let message = AsyncWriteMessage::Data(Vec::from(data)); + self.sender.send(message) + } + + fn eof(&mut self) -> Result<(), mpsc::SendError> { + let message = AsyncWriteMessage::Eof; + self.sender.send(message) + } + + pub(crate) fn rust_error( + &mut self, + error: impl std::error::Error, + ) -> Result<(), impl std::error::Error> { + let message = AsyncWriteMessage::Error(error.to_string()); + self.sender.send(message) + } +} + +/// A wrapper of the `AsyncWrite` trait for the synchronous Nix C++ land. +pub struct AsyncWriteAdapter { + receiver: mpsc::UnboundedReceiver, + eof: bool, +} + +impl AsyncWriteAdapter { + pub fn new() -> (Self, Box) { + let (sender, receiver) = mpsc::unbounded_channel(); + + let r = Self { + receiver, + eof: false, + }; + let sender = Box::new(AsyncWriteSender { sender }); + + (r, sender) + } + + /// Write everything the sender sends to us. + pub async fn write_all(mut self, mut writer: Box) -> AtticResult<()> { + let writer = writer.as_mut(); + + while let Some(data) = self.next().await { + match data { + Ok(v) => { + writer.write_all(&v).await?; + } + Err(e) => { + return Err(e); + } + } + } + + if !self.eof { + Err(io::Error::from(io::ErrorKind::BrokenPipe).into()) + } else { + Ok(()) + } + } +} + +impl Stream for AsyncWriteAdapter { + type Item = AtticResult>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.receiver.poll_recv(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Some(message)) => { + use AsyncWriteMessage::*; + match message { + Data(v) => Poll::Ready(Some(Ok(v))), + Error(exception) => { + let error = AtticError::CxxError { exception }; + Poll::Ready(Some(Err(error))) + } + Eof => { + self.eof = true; + Poll::Ready(None) + } + } + } + Poll::Ready(None) => { + if !self.eof { + Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe).into()))) + } else { + Poll::Ready(None) + } + } + } + } +} + +#[cxx::bridge] +/// Generated by `cxx.rs`. +/// +/// Mid-level wrapper of `libnixstore` implemented in C++. +mod ffi { + extern "Rust" { + type AsyncWriteSender; + fn send(self: &mut AsyncWriteSender, data: &[u8]) -> Result<()>; + fn eof(self: &mut AsyncWriteSender) -> Result<()>; + } + + unsafe extern "C++" { + include!("attic/src/nix_store/bindings/nix.hpp"); + + // ========= + // CNixStore + // ========= + + /// Mid-level wrapper for the Unix Domain Socket Nix Store. + type CNixStore; + + /// Returns the path of the Nix store itself. + fn store_dir(self: Pin<&mut CNixStore>) -> String; + + /* + /// Verifies that a path is indeed in the Nix store, then return the base store path. + /// + /// Use parse_store_path instead. + fn to_store_path(self: Pin<&mut CNixStore>, path: &str) -> Result; + */ + + /// Queries information about a valid path. + fn query_path_info( + self: Pin<&mut CNixStore>, + store_path: &[u8], + ) -> Result>; + + /// Computes the closure of a valid path. + /// + /// If `flip_directions` is true, the set of paths that can reach `store_path` is + /// returned. + fn compute_fs_closure( + self: Pin<&mut CNixStore>, + store_path: &[u8], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result>>; + + /// Computes the closure of a list of valid paths. + /// + /// This is the multi-path variant of `compute_fs_closure`. + /// If `flip_directions` is true, the set of paths that can reach `store_path` is + /// returned. + /// + /// It's easier and more efficient to just pass a vector of slices + /// instead of wrangling with concrete "extern rust" / "extern C++" + /// types. + fn compute_fs_closure_multi( + self: Pin<&mut CNixStore>, + base_names: &[&[u8]], + flip_direction: bool, + include_outputs: bool, + include_derivers: bool, + ) -> Result>>; + + /// Creates a NAR dump from a path. + fn nar_from_path( + self: Pin<&mut CNixStore>, + base_name: Vec, + sender: Box, + ) -> Result<()>; + + /// Obtains a handle to the Nix store. + fn open_nix_store() -> Result>; + + // ========= + // CPathInfo + // ========= + + /// Mid-level wrapper for the `nix::ValidPathInfo` struct. + type CPathInfo; + + /// Returns the SHA-256 hash of the store path. + fn nar_sha256_hash(self: Pin<&mut CPathInfo>) -> &[u8]; + + /// Returns the size of the NAR. + fn nar_size(self: Pin<&mut CPathInfo>) -> u64; + + /// Returns the references of the store path. + fn references(self: Pin<&mut CPathInfo>) -> UniquePtr>; + + /// Returns the possibly invalid signatures attached to the store path. + fn sigs(self: Pin<&mut CPathInfo>) -> UniquePtr>; + + /// Returns the CA field of the store path. + fn ca(self: Pin<&mut CPathInfo>) -> String; + } +} diff --git a/src/bindings/nix.cpp b/src/bindings/nix.cpp new file mode 100644 index 0000000..167244c --- /dev/null +++ b/src/bindings/nix.cpp @@ -0,0 +1,173 @@ +/* +Copyright 2022 Zhaofeng Li and the Attic contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// C++ side of the libnixstore glue. +// +// We implement a mid-level wrapper of the Nix Store interface, +// which is then wrapped again in the Rust side to enable full +// async-await operation. +// +// Here we stick with the naming conventions of Rust and handle +// Rust types directly where possible, so that the interfaces are +// satisfying to use from the Rust side via cxx.rs. + +#include "attic/src/nix_store/bindings/nix.hpp" + +static std::mutex g_init_nix_mutex; +static bool g_init_nix_done = false; + +static nix::StorePath store_path_from_rust(RBasePathSlice base_name) { + std::string_view sv((const char *)base_name.data(), base_name.size()); + return nix::StorePath(sv); +} + +static bool hash_is_sha256(const nix::Hash &hash) { +#ifdef ATTIC_NIX_2_20 + return hash.algo == nix::HashAlgorithm::SHA256; +#else + return hash.type == nix::htSHA256; +#endif +} + +// ======== +// RustSink +// ======== + +RustSink::RustSink(RBox sender) : sender(std::move(sender)) {} + +void RustSink::operator () (std::string_view data) { + RBasePathSlice s((const unsigned char *)data.data(), data.size()); + + this->sender->send(s); +} + +void RustSink::eof() { + this->sender->eof(); +} + + +// ========= +// CPathInfo +// ========= + +CPathInfo::CPathInfo(nix::ref pi) : pi(pi) {} + +RHashSlice CPathInfo::nar_sha256_hash() { + auto &hash = this->pi->narHash; + + if (!hash_is_sha256(hash)) { + throw nix::Error("Only SHA-256 hashes are supported at the moment"); + } + + return RHashSlice(hash.hash, hash.hashSize); +} + +uint64_t CPathInfo::nar_size() { + return this->pi->narSize; +} + +std::unique_ptr> CPathInfo::sigs() { + std::vector result; + for (auto&& elem : this->pi->sigs) { + result.push_back(std::string(elem)); + } + return std::make_unique>(result); +} + +std::unique_ptr> CPathInfo::references() { + std::vector result; + for (auto&& elem : this->pi->references) { + result.push_back(std::string(elem.to_string())); + } + return std::make_unique>(result); +} + +RString CPathInfo::ca() { + if (this->pi->ca) { + return RString(nix::renderContentAddress(this->pi->ca)); + } else { + return RString(""); + } +} + +// ========= +// CNixStore +// ========= + +CNixStore::CNixStore() { + std::map params; + std::lock_guard lock(g_init_nix_mutex); + + if (!g_init_nix_done) { + nix::initNix(); + g_init_nix_done = true; + } + + this->store = nix::openStore(nix::settings.storeUri.get(), params); +} + +RString CNixStore::store_dir() { + return RString(this->store->storeDir); +} + +std::unique_ptr CNixStore::query_path_info(RBasePathSlice base_name) { + auto store_path = store_path_from_rust(base_name); + + auto r = this->store->queryPathInfo(store_path); + return std::make_unique(r); +} + +std::unique_ptr> CNixStore::compute_fs_closure(RBasePathSlice base_name, bool flip_direction, bool include_outputs, bool include_derivers) { + std::set out; + + this->store->computeFSClosure(store_path_from_rust(base_name), out, flip_direction, include_outputs, include_derivers); + + std::vector result; + for (auto&& elem : out) { + result.push_back(std::string(elem.to_string())); + } + return std::make_unique>(result); +} + +std::unique_ptr> CNixStore::compute_fs_closure_multi(RSlice base_names, bool flip_direction, bool include_outputs, bool include_derivers) { + std::set path_set, out; + for (auto&& base_name : base_names) { + path_set.insert(store_path_from_rust(base_name)); + } + + this->store->computeFSClosure(path_set, out, flip_direction, include_outputs, include_derivers); + + std::vector result; + for (auto&& elem : out) { + result.push_back(std::string(elem.to_string())); + } + return std::make_unique>(result); +} + +void CNixStore::nar_from_path(RVec base_name, RBox sender) { + RustSink sink(std::move(sender)); + + std::string_view sv((const char *)base_name.data(), base_name.size()); + nix::StorePath store_path(sv); + + // exceptions will be thrown into Rust + this->store->narFromPath(store_path, sink); + sink.eof(); +} + +std::unique_ptr open_nix_store() { + return std::make_unique(); +} diff --git a/src/bindings/nix.hpp b/src/bindings/nix.hpp new file mode 100644 index 0000000..bbab4c5 --- /dev/null +++ b/src/bindings/nix.hpp @@ -0,0 +1,96 @@ +/* +Copyright 2022 Zhaofeng Li and the Attic contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// C++ side of the libnixstore glue. +// +// We implement a mid-level wrapper of the Nix Store interface, +// which is then wrapped again in the Rust side to enable full +// async-await operation. +// +// Here we stick with the naming conventions of Rust and handle +// Rust types directly where possible, so that the interfaces are +// satisfying to use from the Rust side via cxx.rs. + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template using RVec = rust::Vec; +template using RBox = rust::Box; +template using RSlice = rust::Slice; +using RString = rust::String; +using RStr = rust::Str; +using RBasePathSlice = RSlice; +using RHashSlice = RSlice; + +struct AsyncWriteSender; + +struct RustSink : nix::Sink +{ + RBox sender; +public: + RustSink(RBox sender); + void operator () (std::string_view data) override; + void eof(); +}; + +// Opaque wrapper for nix::ValidPathInfo +class CPathInfo { + nix::ref pi; +public: + CPathInfo(nix::ref pi); + RHashSlice nar_sha256_hash(); + uint64_t nar_size(); + std::unique_ptr> sigs(); + std::unique_ptr> references(); + RString ca(); +}; + +class CNixStore { + std::shared_ptr store; +public: + CNixStore(); + + RString store_dir(); + std::unique_ptr query_path_info(RBasePathSlice base_name); + std::unique_ptr> compute_fs_closure( + RBasePathSlice base_name, + bool flip_direction, + bool include_outputs, + bool include_derivers); + std::unique_ptr> compute_fs_closure_multi( + RSlice base_names, + bool flip_direction, + bool include_outputs, + bool include_derivers); + void nar_from_path(RVec base_name, RBox sender); +}; + +std::unique_ptr open_nix_store(); + +// Relies on our definitions +#include "attic/src/nix_store/bindings/mod.rs.h"