Trusty: provenance library workspace (initial import)
This commit is contained in:
16
crates/adapters/trusty-tokio/Cargo.toml
Normal file
16
crates/adapters/trusty-tokio/Cargo.toml
Normal file
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "trusty-tokio"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
authors.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
description = "Runs a blocking trusty Store on a tokio runtime's blocking pool, so an async caller does not park a worker thread. An adapter, not a core concern: a runtime is a substrate."
|
||||
|
||||
[dependencies]
|
||||
trusty-store = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["rt", "rt-multi-thread", "macros", "time", "sync"] }
|
||||
264
crates/adapters/trusty-tokio/src/lib.rs
Normal file
264
crates/adapters/trusty-tokio/src/lib.rs
Normal file
@@ -0,0 +1,264 @@
|
||||
#![deny(clippy::all)]
|
||||
|
||||
//! An async face for a blocking [`Store`].
|
||||
//!
|
||||
//! # Why this is an adapter and not a core concern
|
||||
//!
|
||||
//! A runtime is a substrate. `trusty-core` touches no filesystem, clock,
|
||||
//! thread, socket or process API, and `trusty-store` is four pure
|
||||
//! dependencies; putting an optional tokio in either would give a crate
|
||||
//! designed to have no platform a platform -- even switched off, it is
|
||||
//! in the manifest and in the lockfile.
|
||||
//!
|
||||
//! So this lives beside `trusty-sqlite` and `trusty-postgres`, named for
|
||||
//! its substrate the same way they are. A consumer on async-std, smol,
|
||||
//! or no runtime at all pays nothing and is not told tokio is the
|
||||
//! answer.
|
||||
//!
|
||||
//! # What this is, and what it is not
|
||||
//!
|
||||
//! This is async **ergonomics**, not async **I/O**. Every call still runs
|
||||
//! the same blocking code on the same backend; it simply runs it on a
|
||||
//! blocking pool instead of on the async runtime's worker threads.
|
||||
//! SQLite writes still serialise, one at a time. A genuinely concurrent
|
||||
//! backend would be a different adapter, not this one.
|
||||
//!
|
||||
//! Saying so plainly matters, because the failure this prevents and the
|
||||
//! failure it does not prevent look alike from outside.
|
||||
//!
|
||||
//! # The problem it does solve
|
||||
//!
|
||||
//! An async runtime has a small pool of worker threads running every
|
||||
//! task. A blocking call inside a task **parks a whole worker** for the
|
||||
//! duration of a write and its fsync, during which that thread runs
|
||||
//! nothing else. Enough concurrent appends and the runtime starves:
|
||||
//! tasks with nothing to do with trusty simply stop making progress.
|
||||
//!
|
||||
//! Distant Star is the consumer this was waiting for -- it is tokio
|
||||
//! throughout, and a relay writing a custody annotation does so inside
|
||||
//! an async task while it is also moving packets.
|
||||
//!
|
||||
//! # Two things deliberately do not cross
|
||||
//!
|
||||
//! [`Store::iter_chain`] returns an iterator borrowing the store, and
|
||||
//! [`Store::with_txn`] takes a closure that is not `'static`. Neither can
|
||||
//! be sent to another thread, so neither is mirrored here:
|
||||
//!
|
||||
//! - **`chain_rows`** materialises the chain into a `Vec` instead of
|
||||
//! streaming it. That is a real difference and not a free one -- the
|
||||
//! result is bounded by memory rather than by patience -- so
|
||||
//! [`AsyncStore::chain_rows_bounded`] exists for callers who would
|
||||
//! rather cap it than find out.
|
||||
//! - **`with_txn_owned`** takes an owned `FnOnce` that can be moved to
|
||||
//! the blocking pool. Callers return values out of it directly rather
|
||||
//! than capturing outer state by reference, because a reference to
|
||||
//! outer state is exactly what cannot travel.
|
||||
|
||||
pub mod pool;
|
||||
pub use pool::BlockingPool;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use trusty_store::{
|
||||
KvCrossCheckReport, LastRow, RowInsert, Store, StoreError, StoreResult, StoredRow, TipRow,
|
||||
TipUpsert,
|
||||
};
|
||||
|
||||
/// The async surface. Mirrors [`Store`] where mirroring is possible, and
|
||||
/// says so where it is not.
|
||||
///
|
||||
/// Not dyn-compatible: `async fn` in a trait cannot be. Consumers that
|
||||
/// need dynamic dispatch should hold [`BlockingStore`] concretely, or
|
||||
/// wrap it themselves.
|
||||
#[allow(async_fn_in_trait)]
|
||||
pub trait AsyncStore {
|
||||
async fn last_row(&self, entity_id: &str) -> StoreResult<Option<LastRow>>;
|
||||
async fn insert_row(&self, row: RowInsert) -> StoreResult<i64>;
|
||||
async fn get_tip(&self, chain_id: &str) -> StoreResult<Option<TipRow>>;
|
||||
async fn get_tip_for_entity(&self, entity_id: &str) -> StoreResult<Option<TipRow>>;
|
||||
async fn set_tip(&self, tip: TipUpsert) -> StoreResult<()>;
|
||||
async fn kv_cross_check(&self) -> StoreResult<KvCrossCheckReport>;
|
||||
|
||||
/// The whole chain, materialised.
|
||||
///
|
||||
/// Unlike [`Store::iter_chain`] this cannot stream: the iterator
|
||||
/// borrows the store, and a borrow cannot be sent to a blocking
|
||||
/// thread. Prefer [`AsyncStore::chain_rows_bounded`] on any chain
|
||||
/// whose length is not known.
|
||||
async fn chain_rows(&self, entity_id: &str) -> StoreResult<Vec<StoredRow>>;
|
||||
|
||||
/// At most `limit` rows, and whether more were left behind.
|
||||
///
|
||||
/// Returning the "there is more" flag rather than silently stopping
|
||||
/// is the whole point: a truncated history that looks complete is
|
||||
/// how a verifier concludes a chain is intact when it has only seen
|
||||
/// the part that is.
|
||||
async fn chain_rows_bounded(
|
||||
&self,
|
||||
entity_id: &str,
|
||||
limit: usize,
|
||||
) -> StoreResult<(Vec<StoredRow>, bool)>;
|
||||
}
|
||||
|
||||
/// Runs a blocking [`Store`] on the runtime's blocking pool.
|
||||
///
|
||||
/// Cloneable and shareable: every clone talks to the same store behind
|
||||
/// the same lock, so concurrent callers serialise rather than race. That
|
||||
/// is not a limitation this type imposes -- it is the backend's, made
|
||||
/// visible instead of hidden.
|
||||
pub struct BlockingStore<S: Store + Send + 'static> {
|
||||
inner: Arc<Mutex<S>>,
|
||||
}
|
||||
|
||||
impl<S: Store + Send + 'static> Clone for BlockingStore<S> {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
inner: Arc::clone(&self.inner),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: Store + Send + 'static> BlockingStore<S> {
|
||||
pub fn new(store: S) -> Self {
|
||||
Self {
|
||||
inner: Arc::new(Mutex::new(store)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Take the store back. Fails if any clone is still alive.
|
||||
pub fn into_inner(self) -> Result<S, Self> {
|
||||
match Arc::try_unwrap(self.inner) {
|
||||
Ok(mutex) => Ok(mutex.into_inner().unwrap_or_else(|e| e.into_inner())),
|
||||
Err(inner) => Err(Self { inner }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Run an owned closure against the store on the blocking pool.
|
||||
///
|
||||
/// The escape hatch for everything this facade does not mirror,
|
||||
/// including transactions. The closure must be `'static` and `Send`,
|
||||
/// which is not a formality: a closure borrowing local state is
|
||||
/// precisely what cannot be moved to another thread.
|
||||
pub async fn with_store<T, F>(&self, f: F) -> StoreResult<T>
|
||||
where
|
||||
F: FnOnce(&mut S) -> StoreResult<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let inner = Arc::clone(&self.inner);
|
||||
run_blocking(move || {
|
||||
let mut guard = inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
f(&mut guard)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// A transaction, with its result carried out by value.
|
||||
///
|
||||
/// [`Store::with_txn`] captures results through outer state because
|
||||
/// its closure cannot return one. Here the closure is owned, so it
|
||||
/// can -- which removes the need for the `Option` dance the blocking
|
||||
/// API requires, and with it the chance of reading the captured slot
|
||||
/// on a path where the transaction rolled back.
|
||||
pub async fn with_txn_owned<T, F>(&self, f: F) -> StoreResult<T>
|
||||
where
|
||||
F: FnOnce(&mut dyn Store) -> StoreResult<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
self.with_store(move |store| {
|
||||
let mut captured: Option<T> = None;
|
||||
let mut once = Some(f);
|
||||
store.with_txn(&mut |txn: &mut dyn Store| {
|
||||
let f = once.take().ok_or_else(|| {
|
||||
StoreError::Backend(
|
||||
"the backend invoked the transaction body more than once, which \
|
||||
this facade cannot support: an owned closure can only run once"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
captured = Some(f(txn)?);
|
||||
Ok(())
|
||||
})?;
|
||||
captured.ok_or_else(|| {
|
||||
StoreError::Backend(
|
||||
"the transaction committed without the body having run".to_string(),
|
||||
)
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
/// Hand work to the blocking pool, and turn a panic there into an error
|
||||
/// rather than letting it take the caller down.
|
||||
///
|
||||
/// A panic inside a store operation is a bug, but a bug that unwinds
|
||||
/// into an unrelated task is a much harder bug to find than one that
|
||||
/// arrives as a `Backend` error at the call site that caused it.
|
||||
async fn run_blocking<T, F>(f: F) -> StoreResult<T>
|
||||
where
|
||||
F: FnOnce() -> StoreResult<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
match tokio::task::spawn_blocking(f).await {
|
||||
Ok(result) => result,
|
||||
Err(join) => Err(StoreError::Backend(format!(
|
||||
"the store operation did not finish: {join}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: Store + Send + 'static> AsyncStore for BlockingStore<S> {
|
||||
async fn last_row(&self, entity_id: &str) -> StoreResult<Option<LastRow>> {
|
||||
let entity_id = entity_id.to_string();
|
||||
self.with_store(move |s| s.last_row(&entity_id)).await
|
||||
}
|
||||
|
||||
async fn insert_row(&self, row: RowInsert) -> StoreResult<i64> {
|
||||
self.with_store(move |s| s.insert_row(&row)).await
|
||||
}
|
||||
|
||||
async fn get_tip(&self, chain_id: &str) -> StoreResult<Option<TipRow>> {
|
||||
let chain_id = chain_id.to_string();
|
||||
self.with_store(move |s| s.get_tip(&chain_id)).await
|
||||
}
|
||||
|
||||
async fn get_tip_for_entity(&self, entity_id: &str) -> StoreResult<Option<TipRow>> {
|
||||
let entity_id = entity_id.to_string();
|
||||
self.with_store(move |s| s.get_tip_for_entity(&entity_id))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn set_tip(&self, tip: TipUpsert) -> StoreResult<()> {
|
||||
self.with_store(move |s| s.set_tip(&tip)).await
|
||||
}
|
||||
|
||||
async fn kv_cross_check(&self) -> StoreResult<KvCrossCheckReport> {
|
||||
self.with_store(|s| s.kv_cross_check()).await
|
||||
}
|
||||
|
||||
async fn chain_rows(&self, entity_id: &str) -> StoreResult<Vec<StoredRow>> {
|
||||
let entity_id = entity_id.to_string();
|
||||
self.with_store(move |s| s.iter_chain(&entity_id)?.collect())
|
||||
.await
|
||||
}
|
||||
|
||||
async fn chain_rows_bounded(
|
||||
&self,
|
||||
entity_id: &str,
|
||||
limit: usize,
|
||||
) -> StoreResult<(Vec<StoredRow>, bool)> {
|
||||
let entity_id = entity_id.to_string();
|
||||
self.with_store(move |s| {
|
||||
let mut iter = s.iter_chain(&entity_id)?;
|
||||
let mut rows = Vec::new();
|
||||
for row in iter.by_ref().take(limit) {
|
||||
rows.push(row?);
|
||||
}
|
||||
// Asking for one more is how "there is more" is learned
|
||||
// without walking the rest of a chain nobody asked for.
|
||||
let more = iter.next().is_some();
|
||||
Ok((rows, more))
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
282
crates/adapters/trusty-tokio/src/pool.rs
Normal file
282
crates/adapters/trusty-tokio/src/pool.rs
Normal file
@@ -0,0 +1,282 @@
|
||||
//! A pool of stores, so concurrent callers are not queued behind one.
|
||||
//!
|
||||
//! # Why a pool rather than removing the lock
|
||||
//!
|
||||
//! [`crate::BlockingStore`] holds one store behind one mutex, so every
|
||||
//! caller serialises. That looks like needless queuing to delete, and it
|
||||
//! is not: `Store::insert_row(&mut self)` requires exclusive access by
|
||||
//! its signature. The mutex is the trait's demand, not this type's
|
||||
//! choice.
|
||||
//!
|
||||
//! Concurrency therefore needs **more stores**, not fewer locks. Each
|
||||
//! holds its own connection, so two calls genuinely run at once.
|
||||
//!
|
||||
//! # The correctness this depends on
|
||||
//!
|
||||
//! Two connections appending to the same entity would race the chain
|
||||
//! tip: appending is read-tip, compute, insert, and those are not
|
||||
//! atomic. `trusty-postgres` holds a per-entity advisory lock across
|
||||
//! that sequence, so same-entity writers serialise in the database while
|
||||
//! different entities proceed in parallel.
|
||||
//!
|
||||
//! **That lock is a precondition of this type, not a detail.** Before it
|
||||
//! existed, the single mutex was the only thing making concurrent
|
||||
//! appends safe, and pooling without it would have converted a slowness
|
||||
//! into a lost update -- a trade that reads as a performance win in a
|
||||
//! diff.
|
||||
//!
|
||||
//! A backend with no such lock, or one writer per file like SQLite,
|
||||
//! should keep using [`crate::BlockingStore`].
|
||||
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use tokio::sync::Semaphore;
|
||||
use trusty_store::{Store, StoreError, StoreResult};
|
||||
|
||||
/// Builds a fresh store, each with its own connection.
|
||||
pub type Factory<S> = Arc<dyn Fn() -> StoreResult<S> + Send + Sync>;
|
||||
|
||||
/// Runs blocking store work across several connections.
|
||||
pub struct BlockingPool<S: Store + Send + 'static> {
|
||||
idle: Arc<Mutex<Vec<S>>>,
|
||||
make: Factory<S>,
|
||||
/// Bounds concurrency to the connection budget. Without it, load
|
||||
/// would open connections until the database refused them, and the
|
||||
/// refusal would arrive as a failed append rather than as waiting.
|
||||
permits: Arc<Semaphore>,
|
||||
live: Arc<AtomicUsize>,
|
||||
max: usize,
|
||||
}
|
||||
|
||||
impl<S: Store + Send + 'static> Clone for BlockingPool<S> {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
idle: Arc::clone(&self.idle),
|
||||
make: Arc::clone(&self.make),
|
||||
permits: Arc::clone(&self.permits),
|
||||
live: Arc::clone(&self.live),
|
||||
max: self.max,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: Store + Send + 'static> BlockingPool<S> {
|
||||
/// A pool of at most `max` connections, opened on demand.
|
||||
///
|
||||
/// Connections are created lazily rather than up front: a pool that
|
||||
/// opens its full budget at startup turns a misconfigured connection
|
||||
/// string into a failure at boot for capacity nobody has asked for
|
||||
/// yet.
|
||||
pub fn new(max: usize, make: impl Fn() -> StoreResult<S> + Send + Sync + 'static) -> Self {
|
||||
let max = max.max(1);
|
||||
Self {
|
||||
idle: Arc::new(Mutex::new(Vec::new())),
|
||||
make: Arc::new(make),
|
||||
permits: Arc::new(Semaphore::new(max)),
|
||||
live: Arc::new(AtomicUsize::new(0)),
|
||||
max,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn capacity(&self) -> usize {
|
||||
self.max
|
||||
}
|
||||
|
||||
/// Connections currently open. Rises to `capacity` under load and
|
||||
/// does not fall; a pool that closed idle connections would pay
|
||||
/// reconnection cost exactly when traffic returned.
|
||||
pub fn open_connections(&self) -> usize {
|
||||
self.live.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Run `f` against a store from the pool.
|
||||
///
|
||||
/// The store is moved into the blocking closure and handed back with
|
||||
/// the result, so ownership never has to be shared and no lock is
|
||||
/// held while the work runs.
|
||||
///
|
||||
/// A store whose work returned an error is still returned to the
|
||||
/// pool: a failed append usually means a rejected row, not a broken
|
||||
/// connection, and discarding the connection would turn every
|
||||
/// validation error into a reconnect.
|
||||
pub async fn with<T, F>(&self, f: F) -> StoreResult<T>
|
||||
where
|
||||
F: FnOnce(&mut S) -> StoreResult<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let _permit = self
|
||||
.permits
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|_| StoreError::Backend("the store pool has been closed".into()))?;
|
||||
|
||||
let mut store = match self.idle.lock().unwrap_or_else(|e| e.into_inner()).pop() {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
let s = (self.make)()?;
|
||||
self.live.fetch_add(1, Ordering::SeqCst);
|
||||
s
|
||||
}
|
||||
};
|
||||
|
||||
let joined = tokio::task::spawn_blocking(move || {
|
||||
let out = f(&mut store);
|
||||
(store, out)
|
||||
})
|
||||
.await;
|
||||
|
||||
match joined {
|
||||
Ok((store, out)) => {
|
||||
self.idle
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.push(store);
|
||||
out
|
||||
}
|
||||
Err(join) => {
|
||||
// The store went down with the panicking task, so the
|
||||
// connection is gone rather than merely idle.
|
||||
self.live.fetch_sub(1, Ordering::SeqCst);
|
||||
Err(StoreError::Backend(format!(
|
||||
"the store operation did not finish and its connection was lost: {join}"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
use trusty_store::{KvCrossCheckReport, LastRow, RowInsert, StoredRow, TipRow, TipUpsert};
|
||||
|
||||
struct Slow {
|
||||
id: usize,
|
||||
delay: Duration,
|
||||
}
|
||||
|
||||
impl Store for Slow {
|
||||
fn last_row(&self, _e: &str) -> StoreResult<Option<LastRow>> {
|
||||
std::thread::sleep(self.delay);
|
||||
Ok(None)
|
||||
}
|
||||
fn insert_row(&mut self, _r: &RowInsert) -> StoreResult<i64> {
|
||||
std::thread::sleep(self.delay);
|
||||
Ok(self.id as i64)
|
||||
}
|
||||
fn iter_chain<'a>(
|
||||
&'a self,
|
||||
_e: &str,
|
||||
) -> StoreResult<Box<dyn Iterator<Item = StoreResult<StoredRow>> + 'a>> {
|
||||
Ok(Box::new(std::iter::empty()))
|
||||
}
|
||||
fn get_tip(&self, _c: &str) -> StoreResult<Option<TipRow>> {
|
||||
Ok(None)
|
||||
}
|
||||
fn get_tip_for_entity(&self, _e: &str) -> StoreResult<Option<TipRow>> {
|
||||
Ok(None)
|
||||
}
|
||||
fn set_tip(&mut self, _t: &TipUpsert) -> StoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn kv_cross_check(&self) -> StoreResult<KvCrossCheckReport> {
|
||||
Err(StoreError::Backend("not implemented".into()))
|
||||
}
|
||||
fn with_txn(
|
||||
&mut self,
|
||||
f: &mut dyn FnMut(&mut dyn Store) -> StoreResult<()>,
|
||||
) -> StoreResult<()> {
|
||||
let mut scratch = Slow {
|
||||
id: self.id,
|
||||
delay: Duration::ZERO,
|
||||
};
|
||||
f(&mut scratch)
|
||||
}
|
||||
}
|
||||
|
||||
fn pool(max: usize, delay_ms: u64) -> BlockingPool<Slow> {
|
||||
let next = Arc::new(AtomicUsize::new(0));
|
||||
BlockingPool::new(max, move || {
|
||||
Ok(Slow {
|
||||
id: next.fetch_add(1, Ordering::SeqCst),
|
||||
delay: Duration::from_millis(delay_ms),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// **The claim.** Four slow calls across four connections finish in
|
||||
/// far less than four times one call.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn calls_run_concurrently_rather_than_queueing() {
|
||||
let p = pool(4, 250);
|
||||
let start = std::time::Instant::now();
|
||||
let mut hs = Vec::new();
|
||||
for _ in 0..4 {
|
||||
let p = p.clone();
|
||||
hs.push(tokio::spawn(async move { p.with(|s| s.last_row("e")).await }));
|
||||
}
|
||||
for h in hs {
|
||||
h.await.expect("join").expect("call");
|
||||
}
|
||||
let elapsed = start.elapsed();
|
||||
assert!(
|
||||
elapsed < Duration::from_millis(700),
|
||||
"four 250ms calls on four connections took {elapsed:?}. Serialised \
|
||||
they would take about a second, which is what a single store \
|
||||
behind one mutex does and what this type exists to stop"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn concurrency_is_bounded_by_the_connection_budget() {
|
||||
let p = pool(2, 60);
|
||||
let mut hs = Vec::new();
|
||||
for _ in 0..8 {
|
||||
let p = p.clone();
|
||||
hs.push(tokio::spawn(async move { p.with(|s| s.last_row("e")).await }));
|
||||
}
|
||||
for h in hs {
|
||||
h.await.expect("join").expect("call");
|
||||
}
|
||||
assert!(
|
||||
p.open_connections() <= 2,
|
||||
"eight callers opened {} connections against a budget of 2. Without \
|
||||
the bound, load opens connections until the database refuses, and \
|
||||
the refusal arrives as a failed append rather than as waiting",
|
||||
p.open_connections()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connections_are_reused_not_opened_per_call() {
|
||||
let p = pool(4, 0);
|
||||
for _ in 0..20 {
|
||||
p.with(|s| s.last_row("e")).await.expect("call");
|
||||
}
|
||||
assert_eq!(
|
||||
p.open_connections(),
|
||||
1,
|
||||
"sequential calls must reuse one connection; opening one per call \
|
||||
would make the pool slower than the mutex it replaces"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_call_returns_its_connection_to_the_pool() {
|
||||
let p = pool(2, 0);
|
||||
let _ = p
|
||||
.with(|s| s.kv_cross_check().map(|_| ()))
|
||||
.await
|
||||
.expect_err("this store returns an error for that call");
|
||||
p.with(|s| s.last_row("e")).await.expect("still usable");
|
||||
assert_eq!(
|
||||
p.open_connections(),
|
||||
1,
|
||||
"a failed append usually means a rejected row, not a broken \
|
||||
connection; discarding it would turn every validation error into \
|
||||
a reconnect"
|
||||
);
|
||||
}
|
||||
}
|
||||
280
crates/adapters/trusty-tokio/tests/runtime.rs
Normal file
280
crates/adapters/trusty-tokio/tests/runtime.rs
Normal file
@@ -0,0 +1,280 @@
|
||||
//! Does the async facade actually keep the runtime free?
|
||||
//!
|
||||
//! That is the only claim it makes, so it is the one worth testing.
|
||||
//! Everything else here exists because it is a way the facade could be
|
||||
//! quietly wrong: a lost value, a swallowed panic, a truncation that
|
||||
//! reads as a complete chain.
|
||||
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use trusty_store::{
|
||||
KvCrossCheckReport, LastRow, RowInsert, Store, StoreError, StoreResult, StoredRow, TipRow,
|
||||
TipUpsert,
|
||||
};
|
||||
use trusty_tokio::{AsyncStore, BlockingStore};
|
||||
|
||||
/// A store that is deliberately slow, so blocking is observable.
|
||||
struct SlowStore {
|
||||
delay: Duration,
|
||||
rows: usize,
|
||||
inserts: Arc<AtomicUsize>,
|
||||
panic_on_insert: bool,
|
||||
}
|
||||
|
||||
impl SlowStore {
|
||||
fn new(delay_ms: u64) -> Self {
|
||||
Self {
|
||||
delay: Duration::from_millis(delay_ms),
|
||||
rows: 0,
|
||||
inserts: Arc::new(AtomicUsize::new(0)),
|
||||
panic_on_insert: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_rows(mut self, rows: usize) -> Self {
|
||||
self.rows = rows;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn stored_row(sequence: i64) -> StoredRow {
|
||||
StoredRow {
|
||||
id: sequence + 1,
|
||||
entity_id: "e".into(),
|
||||
chain_id: "c".into(),
|
||||
event_type: "MUTATION".into(),
|
||||
sequence,
|
||||
prev_hash: String::new(),
|
||||
row_hash: Some(format!("{sequence:064}")),
|
||||
anchor_hash: None,
|
||||
raw_payload: None,
|
||||
detected_at: String::new(),
|
||||
received_at: String::new(),
|
||||
source_url: String::new(),
|
||||
extraction_method: "trusty".into(),
|
||||
}
|
||||
}
|
||||
|
||||
impl Store for SlowStore {
|
||||
fn last_row(&self, _entity_id: &str) -> StoreResult<Option<LastRow>> {
|
||||
std::thread::sleep(self.delay);
|
||||
Ok(Some(LastRow {
|
||||
row_hash: "aa".into(),
|
||||
sequence: 7,
|
||||
}))
|
||||
}
|
||||
|
||||
fn insert_row(&mut self, _row: &RowInsert) -> StoreResult<i64> {
|
||||
std::thread::sleep(self.delay);
|
||||
if self.panic_on_insert {
|
||||
panic!("a bug inside a store operation");
|
||||
}
|
||||
Ok(self.inserts.fetch_add(1, Ordering::SeqCst) as i64 + 1)
|
||||
}
|
||||
|
||||
fn iter_chain<'a>(
|
||||
&'a self,
|
||||
_entity_id: &str,
|
||||
) -> StoreResult<Box<dyn Iterator<Item = StoreResult<StoredRow>> + 'a>> {
|
||||
Ok(Box::new((0..self.rows as i64).map(|i| Ok(stored_row(i)))))
|
||||
}
|
||||
|
||||
fn get_tip(&self, _chain_id: &str) -> StoreResult<Option<TipRow>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn get_tip_for_entity(&self, _entity_id: &str) -> StoreResult<Option<TipRow>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn set_tip(&mut self, _tip: &TipUpsert) -> StoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn kv_cross_check(&self) -> StoreResult<KvCrossCheckReport> {
|
||||
Err(StoreError::Backend("not implemented for this test".into()))
|
||||
}
|
||||
|
||||
fn with_txn(
|
||||
&mut self,
|
||||
f: &mut dyn FnMut(&mut dyn Store) -> StoreResult<()>,
|
||||
) -> StoreResult<()> {
|
||||
let mut scratch = SlowStore::new(0);
|
||||
f(&mut scratch)
|
||||
}
|
||||
}
|
||||
|
||||
fn row_insert() -> RowInsert {
|
||||
RowInsert {
|
||||
entity_id: "e".into(),
|
||||
chain_id: "c".into(),
|
||||
event_type: "MUTATION".into(),
|
||||
sequence: 0,
|
||||
prev_hash: String::new(),
|
||||
row_hash: None,
|
||||
anchor_hash: None,
|
||||
raw_payload: None,
|
||||
detected_at: String::new(),
|
||||
received_at: String::new(),
|
||||
source_url: String::new(),
|
||||
extraction_method: "trusty".into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// **The claim.** A single-worker runtime, one slow store call, and an
|
||||
/// unrelated task that must keep running while it is in flight.
|
||||
///
|
||||
/// Single-threaded on purpose: with more than one worker the test would
|
||||
/// pass even if the call blocked, because another worker would pick the
|
||||
/// second task up. One worker is the only configuration where blocking
|
||||
/// is fatal and therefore the only one that proves anything.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn a_slow_store_call_does_not_park_the_runtime() {
|
||||
let store = BlockingStore::new(SlowStore::new(300));
|
||||
let ticks = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let counter = Arc::clone(&ticks);
|
||||
let ticker = tokio::spawn(async move {
|
||||
for _ in 0..20 {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
});
|
||||
|
||||
store.insert_row(row_insert()).await.expect("insert");
|
||||
let progressed = ticks.load(Ordering::SeqCst);
|
||||
ticker.abort();
|
||||
|
||||
assert!(
|
||||
progressed > 5,
|
||||
"the other task must have kept running while the store was busy. \
|
||||
It ticked {progressed} times; blocking the worker would have left \
|
||||
it at or near zero, which is exactly the runtime starvation this \
|
||||
facade exists to prevent"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn concurrent_callers_serialise_rather_than_race() {
|
||||
let store = BlockingStore::new(SlowStore::new(20));
|
||||
let mut handles = Vec::new();
|
||||
for _ in 0..8 {
|
||||
let s = store.clone();
|
||||
handles.push(tokio::spawn(async move { s.insert_row(row_insert()).await }));
|
||||
}
|
||||
let mut ids = Vec::new();
|
||||
for h in handles {
|
||||
ids.push(h.await.expect("join").expect("insert"));
|
||||
}
|
||||
ids.sort_unstable();
|
||||
|
||||
assert_eq!(
|
||||
ids,
|
||||
(1..=8).collect::<Vec<i64>>(),
|
||||
"every caller must get its own row and none may be lost. The \
|
||||
backend serialises writes; that is its property, made visible \
|
||||
here rather than hidden"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_panic_in_the_store_becomes_an_error_not_a_dead_task() {
|
||||
let mut slow = SlowStore::new(0);
|
||||
slow.panic_on_insert = true;
|
||||
let store = BlockingStore::new(slow);
|
||||
|
||||
let result = store.insert_row(row_insert()).await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(StoreError::Backend(_))),
|
||||
"a panic inside a store operation is a bug, but one that unwinds \
|
||||
into an unrelated task is far harder to find than one arriving as \
|
||||
an error at the call site that caused it"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_bounded_read_says_when_it_left_something_behind() {
|
||||
let store = BlockingStore::new(SlowStore::new(0).with_rows(10));
|
||||
|
||||
let (rows, more) = store.chain_rows_bounded("e", 4).await.expect("bounded");
|
||||
assert_eq!(rows.len(), 4);
|
||||
assert!(
|
||||
more,
|
||||
"a truncated history that looks complete is how a verifier \
|
||||
concludes a chain is intact having seen only the part that is"
|
||||
);
|
||||
|
||||
let (all, more) = store.chain_rows_bounded("e", 50).await.expect("bounded");
|
||||
assert_eq!(all.len(), 10);
|
||||
assert!(!more, "with nothing left behind it must not claim there is");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_whole_chain_can_still_be_read_when_that_is_wanted() {
|
||||
let store = BlockingStore::new(SlowStore::new(0).with_rows(5));
|
||||
let rows = store.chain_rows("e").await.expect("chain");
|
||||
assert_eq!(
|
||||
rows.len(),
|
||||
5,
|
||||
"iter_chain cannot cross a thread boundary, so this materialises \
|
||||
instead of streaming -- but it must still return everything"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_transaction_carries_its_result_out_by_value() {
|
||||
let store = BlockingStore::new(SlowStore::new(0));
|
||||
|
||||
let answer = store
|
||||
.with_txn_owned(|txn| {
|
||||
txn.insert_row(&row_insert())?;
|
||||
Ok("committed".to_string())
|
||||
})
|
||||
.await
|
||||
.expect("txn");
|
||||
|
||||
assert_eq!(
|
||||
answer, "committed",
|
||||
"the blocking with_txn captures results through outer state \
|
||||
because its closure cannot return one; an owned closure can, \
|
||||
which removes the chance of reading a captured slot on a path \
|
||||
where the transaction rolled back"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failing_transaction_reports_the_failure_not_a_missing_value() {
|
||||
let store = BlockingStore::new(SlowStore::new(0));
|
||||
|
||||
let result: StoreResult<String> = store
|
||||
.with_txn_owned(|_txn| Err(StoreError::Backend("deliberate".into())))
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Err(StoreError::Backend(msg)) => assert_eq!(
|
||||
msg, "deliberate",
|
||||
"the body's own error must survive, not be replaced by a \
|
||||
complaint that no value was produced"
|
||||
),
|
||||
other => panic!("expected the body's error, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clones_share_one_store() {
|
||||
let store = BlockingStore::new(SlowStore::new(0));
|
||||
let clone = store.clone();
|
||||
|
||||
store.insert_row(row_insert()).await.expect("first");
|
||||
let second = clone.insert_row(row_insert()).await.expect("second");
|
||||
|
||||
assert_eq!(
|
||||
second, 2,
|
||||
"a clone must talk to the same store, or two halves of an \
|
||||
application would be appending to different chains while \
|
||||
believing they shared one"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user