bring code from attic
This commit is contained in:
parent
39792cdd40
commit
c956d6741a
SSH key fingerprint:
SHA256:o/geVWV4om1QhUSkKvDQeW/eAihwnjyXkqMwrVdbuts
3 changed files with 539 additions and 0 deletions
270
src/bindings/mod.rs
Normal file
270
src/bindings/mod.rs
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
/*
|
||||
Copyright 2022 Zhaofeng Li and the Attic contributors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
//! `libnixstore` Bindings
|
||||
|
||||
use std::cell::UnsafeCell;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::stream::{Stream, StreamExt};
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt};
|
||||
|
||||
use crate::{AtticError, AtticResult};
|
||||
|
||||
// The C++ implementation takes care of concurrency
|
||||
#[repr(transparent)]
|
||||
pub struct FfiNixStore(UnsafeCell<cxx::UniquePtr<ffi::CNixStore>>);
|
||||
|
||||
unsafe impl Send for FfiNixStore {}
|
||||
unsafe impl Sync for FfiNixStore {}
|
||||
|
||||
impl FfiNixStore {
|
||||
pub fn store(&self) -> Pin<&mut ffi::CNixStore> {
|
||||
unsafe {
|
||||
let ptr = self.0.get().as_mut().unwrap();
|
||||
ptr.pin_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Obtain a handle to the Nix store.
|
||||
pub unsafe fn open_nix_store() -> AtticResult<FfiNixStore> {
|
||||
match ffi::open_nix_store() {
|
||||
Ok(ptr) => {
|
||||
let cell = UnsafeCell::new(ptr);
|
||||
Ok(FfiNixStore(cell))
|
||||
}
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Benchmark different implementations
|
||||
// (tokio, crossbeam, flume)
|
||||
mod mpsc {
|
||||
// Tokio
|
||||
pub use tokio::sync::mpsc::{
|
||||
UnboundedReceiver, UnboundedSender, error::SendError, unbounded_channel,
|
||||
};
|
||||
}
|
||||
|
||||
/// Async write request.
|
||||
#[derive(Debug)]
|
||||
enum AsyncWriteMessage {
|
||||
Data(Vec<u8>),
|
||||
Error(String),
|
||||
Eof,
|
||||
}
|
||||
|
||||
/// Async write request sender.
|
||||
#[derive(Clone)]
|
||||
pub struct AsyncWriteSender {
|
||||
sender: mpsc::UnboundedSender<AsyncWriteMessage>,
|
||||
}
|
||||
|
||||
impl AsyncWriteSender {
|
||||
fn send(&mut self, data: &[u8]) -> Result<(), mpsc::SendError<AsyncWriteMessage>> {
|
||||
let message = AsyncWriteMessage::Data(Vec::from(data));
|
||||
self.sender.send(message)
|
||||
}
|
||||
|
||||
fn eof(&mut self) -> Result<(), mpsc::SendError<AsyncWriteMessage>> {
|
||||
let message = AsyncWriteMessage::Eof;
|
||||
self.sender.send(message)
|
||||
}
|
||||
|
||||
pub(crate) fn rust_error(
|
||||
&mut self,
|
||||
error: impl std::error::Error,
|
||||
) -> Result<(), impl std::error::Error> {
|
||||
let message = AsyncWriteMessage::Error(error.to_string());
|
||||
self.sender.send(message)
|
||||
}
|
||||
}
|
||||
|
||||
/// A wrapper of the `AsyncWrite` trait for the synchronous Nix C++ land.
|
||||
pub struct AsyncWriteAdapter {
|
||||
receiver: mpsc::UnboundedReceiver<AsyncWriteMessage>,
|
||||
eof: bool,
|
||||
}
|
||||
|
||||
impl AsyncWriteAdapter {
|
||||
pub fn new() -> (Self, Box<AsyncWriteSender>) {
|
||||
let (sender, receiver) = mpsc::unbounded_channel();
|
||||
|
||||
let r = Self {
|
||||
receiver,
|
||||
eof: false,
|
||||
};
|
||||
let sender = Box::new(AsyncWriteSender { sender });
|
||||
|
||||
(r, sender)
|
||||
}
|
||||
|
||||
/// Write everything the sender sends to us.
|
||||
pub async fn write_all(mut self, mut writer: Box<dyn AsyncWrite + Unpin>) -> AtticResult<()> {
|
||||
let writer = writer.as_mut();
|
||||
|
||||
while let Some(data) = self.next().await {
|
||||
match data {
|
||||
Ok(v) => {
|
||||
writer.write_all(&v).await?;
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !self.eof {
|
||||
Err(io::Error::from(io::ErrorKind::BrokenPipe).into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for AsyncWriteAdapter {
|
||||
type Item = AtticResult<Vec<u8>>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
match self.receiver.poll_recv(cx) {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(Some(message)) => {
|
||||
use AsyncWriteMessage::*;
|
||||
match message {
|
||||
Data(v) => Poll::Ready(Some(Ok(v))),
|
||||
Error(exception) => {
|
||||
let error = AtticError::CxxError { exception };
|
||||
Poll::Ready(Some(Err(error)))
|
||||
}
|
||||
Eof => {
|
||||
self.eof = true;
|
||||
Poll::Ready(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
Poll::Ready(None) => {
|
||||
if !self.eof {
|
||||
Poll::Ready(Some(Err(io::Error::from(io::ErrorKind::BrokenPipe).into())))
|
||||
} else {
|
||||
Poll::Ready(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cxx::bridge]
|
||||
/// Generated by `cxx.rs`.
|
||||
///
|
||||
/// Mid-level wrapper of `libnixstore` implemented in C++.
|
||||
mod ffi {
|
||||
extern "Rust" {
|
||||
type AsyncWriteSender;
|
||||
fn send(self: &mut AsyncWriteSender, data: &[u8]) -> Result<()>;
|
||||
fn eof(self: &mut AsyncWriteSender) -> Result<()>;
|
||||
}
|
||||
|
||||
unsafe extern "C++" {
|
||||
include!("attic/src/nix_store/bindings/nix.hpp");
|
||||
|
||||
// =========
|
||||
// CNixStore
|
||||
// =========
|
||||
|
||||
/// Mid-level wrapper for the Unix Domain Socket Nix Store.
|
||||
type CNixStore;
|
||||
|
||||
/// Returns the path of the Nix store itself.
|
||||
fn store_dir(self: Pin<&mut CNixStore>) -> String;
|
||||
|
||||
/*
|
||||
/// Verifies that a path is indeed in the Nix store, then return the base store path.
|
||||
///
|
||||
/// Use parse_store_path instead.
|
||||
fn to_store_path(self: Pin<&mut CNixStore>, path: &str) -> Result<String>;
|
||||
*/
|
||||
|
||||
/// Queries information about a valid path.
|
||||
fn query_path_info(
|
||||
self: Pin<&mut CNixStore>,
|
||||
store_path: &[u8],
|
||||
) -> Result<UniquePtr<CPathInfo>>;
|
||||
|
||||
/// Computes the closure of a valid path.
|
||||
///
|
||||
/// If `flip_directions` is true, the set of paths that can reach `store_path` is
|
||||
/// returned.
|
||||
fn compute_fs_closure(
|
||||
self: Pin<&mut CNixStore>,
|
||||
store_path: &[u8],
|
||||
flip_direction: bool,
|
||||
include_outputs: bool,
|
||||
include_derivers: bool,
|
||||
) -> Result<UniquePtr<CxxVector<CxxString>>>;
|
||||
|
||||
/// Computes the closure of a list of valid paths.
|
||||
///
|
||||
/// This is the multi-path variant of `compute_fs_closure`.
|
||||
/// If `flip_directions` is true, the set of paths that can reach `store_path` is
|
||||
/// returned.
|
||||
///
|
||||
/// It's easier and more efficient to just pass a vector of slices
|
||||
/// instead of wrangling with concrete "extern rust" / "extern C++"
|
||||
/// types.
|
||||
fn compute_fs_closure_multi(
|
||||
self: Pin<&mut CNixStore>,
|
||||
base_names: &[&[u8]],
|
||||
flip_direction: bool,
|
||||
include_outputs: bool,
|
||||
include_derivers: bool,
|
||||
) -> Result<UniquePtr<CxxVector<CxxString>>>;
|
||||
|
||||
/// Creates a NAR dump from a path.
|
||||
fn nar_from_path(
|
||||
self: Pin<&mut CNixStore>,
|
||||
base_name: Vec<u8>,
|
||||
sender: Box<AsyncWriteSender>,
|
||||
) -> Result<()>;
|
||||
|
||||
/// Obtains a handle to the Nix store.
|
||||
fn open_nix_store() -> Result<UniquePtr<CNixStore>>;
|
||||
|
||||
// =========
|
||||
// CPathInfo
|
||||
// =========
|
||||
|
||||
/// Mid-level wrapper for the `nix::ValidPathInfo` struct.
|
||||
type CPathInfo;
|
||||
|
||||
/// Returns the SHA-256 hash of the store path.
|
||||
fn nar_sha256_hash(self: Pin<&mut CPathInfo>) -> &[u8];
|
||||
|
||||
/// Returns the size of the NAR.
|
||||
fn nar_size(self: Pin<&mut CPathInfo>) -> u64;
|
||||
|
||||
/// Returns the references of the store path.
|
||||
fn references(self: Pin<&mut CPathInfo>) -> UniquePtr<CxxVector<CxxString>>;
|
||||
|
||||
/// Returns the possibly invalid signatures attached to the store path.
|
||||
fn sigs(self: Pin<&mut CPathInfo>) -> UniquePtr<CxxVector<CxxString>>;
|
||||
|
||||
/// Returns the CA field of the store path.
|
||||
fn ca(self: Pin<&mut CPathInfo>) -> String;
|
||||
}
|
||||
}
|
||||
173
src/bindings/nix.cpp
Normal file
173
src/bindings/nix.cpp
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
/*
|
||||
Copyright 2022 Zhaofeng Li and the Attic contributors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// C++ side of the libnixstore glue.
|
||||
//
|
||||
// We implement a mid-level wrapper of the Nix Store interface,
|
||||
// which is then wrapped again in the Rust side to enable full
|
||||
// async-await operation.
|
||||
//
|
||||
// Here we stick with the naming conventions of Rust and handle
|
||||
// Rust types directly where possible, so that the interfaces are
|
||||
// satisfying to use from the Rust side via cxx.rs.
|
||||
|
||||
#include "attic/src/nix_store/bindings/nix.hpp"
|
||||
|
||||
static std::mutex g_init_nix_mutex;
|
||||
static bool g_init_nix_done = false;
|
||||
|
||||
static nix::StorePath store_path_from_rust(RBasePathSlice base_name) {
|
||||
std::string_view sv((const char *)base_name.data(), base_name.size());
|
||||
return nix::StorePath(sv);
|
||||
}
|
||||
|
||||
static bool hash_is_sha256(const nix::Hash &hash) {
|
||||
#ifdef ATTIC_NIX_2_20
|
||||
return hash.algo == nix::HashAlgorithm::SHA256;
|
||||
#else
|
||||
return hash.type == nix::htSHA256;
|
||||
#endif
|
||||
}
|
||||
|
||||
// ========
|
||||
// RustSink
|
||||
// ========
|
||||
|
||||
RustSink::RustSink(RBox<AsyncWriteSender> sender) : sender(std::move(sender)) {}
|
||||
|
||||
void RustSink::operator () (std::string_view data) {
|
||||
RBasePathSlice s((const unsigned char *)data.data(), data.size());
|
||||
|
||||
this->sender->send(s);
|
||||
}
|
||||
|
||||
void RustSink::eof() {
|
||||
this->sender->eof();
|
||||
}
|
||||
|
||||
|
||||
// =========
|
||||
// CPathInfo
|
||||
// =========
|
||||
|
||||
CPathInfo::CPathInfo(nix::ref<const nix::ValidPathInfo> pi) : pi(pi) {}
|
||||
|
||||
RHashSlice CPathInfo::nar_sha256_hash() {
|
||||
auto &hash = this->pi->narHash;
|
||||
|
||||
if (!hash_is_sha256(hash)) {
|
||||
throw nix::Error("Only SHA-256 hashes are supported at the moment");
|
||||
}
|
||||
|
||||
return RHashSlice(hash.hash, hash.hashSize);
|
||||
}
|
||||
|
||||
uint64_t CPathInfo::nar_size() {
|
||||
return this->pi->narSize;
|
||||
}
|
||||
|
||||
std::unique_ptr<std::vector<std::string>> CPathInfo::sigs() {
|
||||
std::vector<std::string> result;
|
||||
for (auto&& elem : this->pi->sigs) {
|
||||
result.push_back(std::string(elem));
|
||||
}
|
||||
return std::make_unique<std::vector<std::string>>(result);
|
||||
}
|
||||
|
||||
std::unique_ptr<std::vector<std::string>> CPathInfo::references() {
|
||||
std::vector<std::string> result;
|
||||
for (auto&& elem : this->pi->references) {
|
||||
result.push_back(std::string(elem.to_string()));
|
||||
}
|
||||
return std::make_unique<std::vector<std::string>>(result);
|
||||
}
|
||||
|
||||
RString CPathInfo::ca() {
|
||||
if (this->pi->ca) {
|
||||
return RString(nix::renderContentAddress(this->pi->ca));
|
||||
} else {
|
||||
return RString("");
|
||||
}
|
||||
}
|
||||
|
||||
// =========
|
||||
// CNixStore
|
||||
// =========
|
||||
|
||||
CNixStore::CNixStore() {
|
||||
std::map<std::string, std::string> params;
|
||||
std::lock_guard<std::mutex> lock(g_init_nix_mutex);
|
||||
|
||||
if (!g_init_nix_done) {
|
||||
nix::initNix();
|
||||
g_init_nix_done = true;
|
||||
}
|
||||
|
||||
this->store = nix::openStore(nix::settings.storeUri.get(), params);
|
||||
}
|
||||
|
||||
RString CNixStore::store_dir() {
|
||||
return RString(this->store->storeDir);
|
||||
}
|
||||
|
||||
std::unique_ptr<CPathInfo> CNixStore::query_path_info(RBasePathSlice base_name) {
|
||||
auto store_path = store_path_from_rust(base_name);
|
||||
|
||||
auto r = this->store->queryPathInfo(store_path);
|
||||
return std::make_unique<CPathInfo>(r);
|
||||
}
|
||||
|
||||
std::unique_ptr<std::vector<std::string>> CNixStore::compute_fs_closure(RBasePathSlice base_name, bool flip_direction, bool include_outputs, bool include_derivers) {
|
||||
std::set<nix::StorePath> out;
|
||||
|
||||
this->store->computeFSClosure(store_path_from_rust(base_name), out, flip_direction, include_outputs, include_derivers);
|
||||
|
||||
std::vector<std::string> result;
|
||||
for (auto&& elem : out) {
|
||||
result.push_back(std::string(elem.to_string()));
|
||||
}
|
||||
return std::make_unique<std::vector<std::string>>(result);
|
||||
}
|
||||
|
||||
std::unique_ptr<std::vector<std::string>> CNixStore::compute_fs_closure_multi(RSlice<const RBasePathSlice> base_names, bool flip_direction, bool include_outputs, bool include_derivers) {
|
||||
std::set<nix::StorePath> path_set, out;
|
||||
for (auto&& base_name : base_names) {
|
||||
path_set.insert(store_path_from_rust(base_name));
|
||||
}
|
||||
|
||||
this->store->computeFSClosure(path_set, out, flip_direction, include_outputs, include_derivers);
|
||||
|
||||
std::vector<std::string> result;
|
||||
for (auto&& elem : out) {
|
||||
result.push_back(std::string(elem.to_string()));
|
||||
}
|
||||
return std::make_unique<std::vector<std::string>>(result);
|
||||
}
|
||||
|
||||
void CNixStore::nar_from_path(RVec<unsigned char> base_name, RBox<AsyncWriteSender> sender) {
|
||||
RustSink sink(std::move(sender));
|
||||
|
||||
std::string_view sv((const char *)base_name.data(), base_name.size());
|
||||
nix::StorePath store_path(sv);
|
||||
|
||||
// exceptions will be thrown into Rust
|
||||
this->store->narFromPath(store_path, sink);
|
||||
sink.eof();
|
||||
}
|
||||
|
||||
std::unique_ptr<CNixStore> open_nix_store() {
|
||||
return std::make_unique<CNixStore>();
|
||||
}
|
||||
96
src/bindings/nix.hpp
Normal file
96
src/bindings/nix.hpp
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
/*
|
||||
Copyright 2022 Zhaofeng Li and the Attic contributors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// C++ side of the libnixstore glue.
|
||||
//
|
||||
// We implement a mid-level wrapper of the Nix Store interface,
|
||||
// which is then wrapped again in the Rust side to enable full
|
||||
// async-await operation.
|
||||
//
|
||||
// Here we stick with the naming conventions of Rust and handle
|
||||
// Rust types directly where possible, so that the interfaces are
|
||||
// satisfying to use from the Rust side via cxx.rs.
|
||||
|
||||
#pragma once
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <nix/store-api.hh>
|
||||
#include <nix/local-store.hh>
|
||||
#include <nix/remote-store.hh>
|
||||
#include <nix/uds-remote-store.hh>
|
||||
#include <nix/hash.hh>
|
||||
#include <nix/path.hh>
|
||||
#include <nix/serialise.hh>
|
||||
#include <nix/shared.hh>
|
||||
#include <rust/cxx.h>
|
||||
|
||||
template<class T> using RVec = rust::Vec<T>;
|
||||
template<class T> using RBox = rust::Box<T>;
|
||||
template<class T> using RSlice = rust::Slice<T>;
|
||||
using RString = rust::String;
|
||||
using RStr = rust::Str;
|
||||
using RBasePathSlice = RSlice<const unsigned char>;
|
||||
using RHashSlice = RSlice<const unsigned char>;
|
||||
|
||||
struct AsyncWriteSender;
|
||||
|
||||
struct RustSink : nix::Sink
|
||||
{
|
||||
RBox<AsyncWriteSender> sender;
|
||||
public:
|
||||
RustSink(RBox<AsyncWriteSender> sender);
|
||||
void operator () (std::string_view data) override;
|
||||
void eof();
|
||||
};
|
||||
|
||||
// Opaque wrapper for nix::ValidPathInfo
|
||||
class CPathInfo {
|
||||
nix::ref<const nix::ValidPathInfo> pi;
|
||||
public:
|
||||
CPathInfo(nix::ref<const nix::ValidPathInfo> pi);
|
||||
RHashSlice nar_sha256_hash();
|
||||
uint64_t nar_size();
|
||||
std::unique_ptr<std::vector<std::string>> sigs();
|
||||
std::unique_ptr<std::vector<std::string>> references();
|
||||
RString ca();
|
||||
};
|
||||
|
||||
class CNixStore {
|
||||
std::shared_ptr<nix::Store> store;
|
||||
public:
|
||||
CNixStore();
|
||||
|
||||
RString store_dir();
|
||||
std::unique_ptr<CPathInfo> query_path_info(RBasePathSlice base_name);
|
||||
std::unique_ptr<std::vector<std::string>> compute_fs_closure(
|
||||
RBasePathSlice base_name,
|
||||
bool flip_direction,
|
||||
bool include_outputs,
|
||||
bool include_derivers);
|
||||
std::unique_ptr<std::vector<std::string>> compute_fs_closure_multi(
|
||||
RSlice<const RBasePathSlice> base_names,
|
||||
bool flip_direction,
|
||||
bool include_outputs,
|
||||
bool include_derivers);
|
||||
void nar_from_path(RVec<unsigned char> base_name, RBox<AsyncWriteSender> sender);
|
||||
};
|
||||
|
||||
std::unique_ptr<CNixStore> open_nix_store();
|
||||
|
||||
// Relies on our definitions
|
||||
#include "attic/src/nix_store/bindings/mod.rs.h"
|
||||
Loading…
Add table
Add a link
Reference in a new issue