pub use futures::future::BoxFuture; pub use std::future::ready; use super::ActorId; use super::schemas::internal_protocol::*; use super::schemas::sturdy; use super::error::Error; use super::error::error; use super::rewrite::CaveatError; use super::rewrite::CheckedCaveat; use preserves::value::Domain; use preserves::value::IOValue; use preserves::value::Map; use preserves::value::NestedValue; use std::boxed::Box; use std::collections::hash_map::HashMap; use std::convert::TryInto; use std::sync::Arc; use std::sync::RwLock; use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize, Ordering}; use tokio::select; use tokio::sync::mpsc::{unbounded_channel, UnboundedSender, UnboundedReceiver}; // use tokio::sync::Notify; use tokio_util::sync::CancellationToken; use tracing; use tracing::Instrument; pub use super::schemas::internal_protocol::_Any; pub use super::schemas::internal_protocol::Handle; pub use super::schemas::internal_protocol::Oid; pub type ActorResult = Result<(), Error>; pub type ActorHandle = tokio::task::JoinHandle; pub trait Entity: Send { fn assert(&mut self, _t: &mut Activation, _a: _Any, _h: Handle) -> ActorResult { Ok(()) } fn retract(&mut self, _t: &mut Activation, _h: Handle) -> ActorResult { Ok(()) } fn message(&mut self, _t: &mut Activation, _m: _Any) -> ActorResult { Ok(()) } fn sync(&mut self, t: &mut Activation, peer: Arc) -> ActorResult { t.message(&peer, _Any::new(true)); Ok(()) } fn turn_end(&mut self, _t: &mut Activation) -> ActorResult { Ok(()) } fn exit_hook(&mut self, _t: &mut Activation, _exit_status: &ActorResult) -> BoxFuture { Box::pin(ready(Ok(()))) } } enum Destination { ImmediateSelf(Oid), Remote(Arc), } type OutboundAssertions = Map; type PendingEventQueue = Vec<(Arc, Event)>; // This is what other implementations call a "Turn", renamed here to // avoid conflicts with schemas::internal_protocol::Turn. pub struct Activation<'activation> { pub actor: &'activation mut Actor, pub debtor: Arc, queues: HashMap, immediate_self: Vec, } #[derive(Debug)] pub struct Debtor { id: u64, debt: Arc, // notify: Notify, } #[derive(Debug)] pub struct LoanedItem { pub debtor: Arc, pub cost: usize, pub item: T, } #[derive(Debug)] enum SystemMessage { Release, ReleaseOid(Oid), Turn(LoanedItem), Crash(Error), } pub struct Mailbox { pub actor_id: ActorId, pub mailbox_id: u64, tx: UnboundedSender, mailbox_count: Arc, } pub struct Actor { actor_id: ActorId, tx: UnboundedSender, rx: Option>, mailbox_count: Arc, outbound_assertions: OutboundAssertions, oid_map: Map>, next_task_id: u64, linked_tasks: Map, exit_hooks: Vec>, } #[derive(PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct ObjectAddress { pub mailbox: Mailbox, pub oid: Oid, } #[derive(PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct Ref { pub addr: Arc, pub attenuation: Vec, } //--------------------------------------------------------------------------- static NEXT_DEBTOR_ID: AtomicU64 = AtomicU64::new(4); preserves_schema::support::lazy_static! { pub static ref INERT_REF: Arc = { struct InertEntity; impl crate::actor::Entity for InertEntity {} let mut ac = Actor::new(); let e = ac.create(InertEntity); ac.boot(tracing::info_span!(parent: None, "INERT_REF"), |t| Box::pin(ready(Ok(t.actor.shutdown())))); e }; pub static ref SYNDICATE_CREDIT: i64 = { let credit = std::env::var("SYNDICATE_CREDIT").unwrap_or("100".to_owned()) .parse::().expect("Valid SYNDICATE_CREDIT environment variable"); tracing::info!("Configured SYNDICATE_CREDIT = {}", credit); credit }; pub static ref DEBTORS: RwLock)>> = RwLock::new(Map::new()); } pub fn start_debt_reporter() { Actor::new().boot(crate::name!("debt-reporter"), |t| Box::pin(async move { t.actor.linked_task(crate::name!("tick"), async move { let mut timer = tokio::time::interval(core::time::Duration::from_secs(1)); loop { timer.tick().await; for (id, (name, debt)) in DEBTORS.read().unwrap().iter() { let _enter = name.enter(); tracing::info!(id, debt = debug(debt.load(Ordering::Relaxed))); } } }); Ok(()) })); } impl<'activation> Activation<'activation> { pub fn new(actor: &'activation mut Actor, debtor: Arc) -> Self { Activation { actor, debtor, queues: HashMap::new(), immediate_self: Vec::new(), } } fn immediate_oid(&self, r: &Arc) -> Oid { if r.addr.mailbox.actor_id != self.actor.actor_id { panic!("Cannot use immediate_self to send to remote peers"); } r.addr.oid.clone() } pub fn assert(&mut self, r: &Arc, a: M) -> Handle where M: Into<_Any> { let handle = crate::next_handle(); if let Some(assertion) = r.rewrite(a.into()) { self.queue_for(r).push((Arc::clone(r), Event::Assert(Box::new( Assert { assertion, handle: handle.clone() })))); self.actor.outbound_assertions.insert(handle.clone(), Destination::Remote(Arc::clone(r))); } handle } pub fn assert_immediate_self(&mut self, r: &Arc, a: M) -> Handle where M: Into<_Any> { let oid = self.immediate_oid(r); let handle = crate::next_handle(); if let Some(assertion) = r.rewrite(a.into()) { self.immediate_self.push(TurnEvent { oid: oid.clone(), event: Event::Assert(Box::new( Assert { assertion, handle: handle.clone() })), }); self.actor.outbound_assertions.insert(handle.clone(), Destination::ImmediateSelf(oid)); } handle } pub fn retract(&mut self, handle: Handle) { if let Some(d) = self.actor.outbound_assertions.remove(&handle) { self.retract_known_ref(d, handle) } } fn retract_known_ref(&mut self, d: Destination, handle: Handle) { match d { Destination::Remote(r) => self.queue_for(&r).push((r, Event::Retract(Box::new(Retract { handle })))), Destination::ImmediateSelf(oid) => self.immediate_self.push(TurnEvent { oid, event: Event::Retract(Box::new(Retract { handle })), }), } } pub fn message(&mut self, r: &Arc, m: M) where M: Into<_Any> { if let Some(body) = r.rewrite(m.into()) { self.queue_for(r).push((Arc::clone(r), Event::Message(Box::new( Message { body })))) } } pub fn message_immediate_self(&mut self, r: &Arc, m: M) where M: Into<_Any> { if let Some(body) = r.rewrite(m.into()) { self.immediate_self.push(TurnEvent { oid: self.immediate_oid(r), event: Event::Message(Box::new(Message { body })), }) } } pub fn sync(&mut self, r: &Arc, peer: Arc) { self.queue_for(r).push((Arc::clone(r), Event::Sync(Box::new(Sync { peer })))); } fn queue_for(&mut self, r: &Arc) -> &mut PendingEventQueue { self.queues.entry(r.addr.mailbox.actor_id).or_default() } fn deliver(&mut self) { if !self.immediate_self.is_empty() { panic!("Unprocessed immediate_self events remain at deliver() time"); } for (_actor_id, turn) in std::mem::take(&mut self.queues).into_iter() { if turn.len() == 0 { continue; } let first_ref = Arc::clone(&turn[0].0); let target = &first_ref.addr.mailbox; let _ = target.send( &self.debtor, Turn(turn.into_iter().map( |(r, e)| TurnEvent { oid: r.addr.oid.clone(), event: e }).collect())); } } fn with_oid R, Fs: FnOnce(&mut Self, &mut Box) -> R>( &mut self, oid: &Oid, kf: Ff, ks: Fs, ) -> R { match self.actor.oid_map.remove_entry(&oid) { None => kf(self), Some((k, mut e)) => { let result = ks(self, &mut e); self.actor.oid_map.insert(k, e); result } } } } impl<'activation> Drop for Activation<'activation> { fn drop(&mut self) { self.deliver() } } impl Debtor { pub fn new(name: tracing::Span) -> Arc { let id = NEXT_DEBTOR_ID.fetch_add(1, Ordering::Relaxed); let debt = Arc::new(AtomicI64::new(0)); DEBTORS.write().unwrap().insert(id, (name, Arc::clone(&debt))); Arc::new(Debtor { id, debt, // notify: Notify::new(), }) } pub fn balance(&self) -> i64 { self.debt.load(Ordering::Relaxed) } pub fn borrow(&self, token_count: usize) { let token_count: i64 = token_count.try_into().expect("manageable token count"); self.debt.fetch_add(token_count, Ordering::Relaxed); } pub fn repay(&self, token_count: usize) { let token_count: i64 = token_count.try_into().expect("manageable token count"); let _old_debt = self.debt.fetch_sub(token_count, Ordering::Relaxed); // if _old_debt - token_count <= *SYNDICATE_CREDIT { // self.notify.notify_one(); // } } pub async fn ensure_clear_funds(&self) { let limit = *SYNDICATE_CREDIT; tokio::task::yield_now().await; while self.balance() > limit { tokio::task::yield_now().await; // self.notify.notified().await; } } } impl Drop for Debtor { fn drop(&mut self) { DEBTORS.write().unwrap().remove(&self.id); } } impl LoanedItem { pub fn new(debtor: &Arc, cost: usize, item: T) -> Self { debtor.borrow(cost); LoanedItem { debtor: Arc::clone(debtor), cost, item } } } impl Drop for LoanedItem { fn drop(&mut self) { self.debtor.repay(self.cost); } } impl Mailbox { #[must_use] pub fn send(&self, debtor: &Arc, t: Turn) -> ActorResult { let token_count = t.0.len(); self.tx.send(SystemMessage::Turn(LoanedItem::new(debtor, token_count, t))) .map_err(|_| error("Target actor not running", _Any::new(false))) } } impl std::fmt::Debug for Mailbox { fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { write!(f, "#", self.actor_id, self.mailbox_id) } } impl std::hash::Hash for Mailbox { fn hash(&self, state: &mut H) { self.mailbox_id.hash(state) } } impl Eq for Mailbox {} impl PartialEq for Mailbox { fn eq(&self, other: &Mailbox) -> bool { self.mailbox_id == other.mailbox_id } } impl Ord for Mailbox { fn cmp(&self, other: &Mailbox) -> std::cmp::Ordering { return self.mailbox_id.cmp(&other.mailbox_id) } } impl PartialOrd for Mailbox { fn partial_cmp(&self, other: &Mailbox) -> Option { return Some(self.cmp(&other)) } } impl Clone for Mailbox { fn clone(&self) -> Self { let Mailbox { actor_id, tx, mailbox_count, .. } = self; let _old_refcount = mailbox_count.fetch_add(1, Ordering::SeqCst); let new_mailbox = Mailbox { actor_id: *actor_id, mailbox_id: crate::next_mailbox_id(), tx: tx.clone(), mailbox_count: Arc::clone(mailbox_count), }; // tracing::trace!(old_mailbox = debug(&self), // new_mailbox = debug(&new_mailbox), // new_mailbox_refcount = debug(_old_refcount + 1)); new_mailbox } } impl Drop for Mailbox { fn drop(&mut self) { let old_mailbox_refcount = self.mailbox_count.fetch_sub(1, Ordering::SeqCst); let new_mailbox_refcount = old_mailbox_refcount - 1; // tracing::trace!(mailbox = debug(&self), // new_mailbox_refcount); if new_mailbox_refcount == 0 { let _ = self.tx.send(SystemMessage::Release); () } } } impl Actor { pub fn new() -> Self { let (tx, rx) = unbounded_channel(); let actor_id = crate::next_actor_id(); // tracing::trace!(id = actor_id, "Actor::new"); Actor { actor_id, tx, rx: Some(rx), mailbox_count: Arc::new(AtomicUsize::new(0)), outbound_assertions: Map::new(), oid_map: Map::new(), next_task_id: 0, linked_tasks: Map::new(), exit_hooks: Vec::new(), } } pub fn create_and_start(name: tracing::Span, e: E) -> Arc { Self::create_and_start_rec(name, e, |_, _, _| ()) } pub fn create_and_start_rec) -> ()>( name: tracing::Span, e: E, f: F, ) -> Arc { let mut ac = Self::new(); let r = ac.create_rec(e, f); ac.start(name); r } pub fn id(&self) -> ActorId { self.actor_id } fn mailbox(&mut self) -> Mailbox { let _old_refcount = self.mailbox_count.fetch_add(1, Ordering::SeqCst); let new_mailbox = Mailbox { actor_id: self.actor_id, mailbox_id: crate::next_mailbox_id(), tx: self.tx.clone(), mailbox_count: Arc::clone(&self.mailbox_count), }; // tracing::trace!(new_mailbox = debug(&new_mailbox), // new_mailbox_refcount = debug(_old_refcount + 1)); new_mailbox } pub fn shutdown(&mut self) { let _ = self.tx.send(SystemMessage::Release); () } pub fn create(&mut self, e: E) -> Arc { self.create_rec(e, |_, _, _| ()) } pub fn create_rec) -> ()>( &mut self, mut e: E, f: F, ) -> Arc { let oid = crate::next_oid(); let r = Arc::new(Ref { addr: Arc::new(ObjectAddress { mailbox: self.mailbox(), oid: oid.clone(), }), attenuation: Vec::new(), }); f(self, &mut e, &r); self.oid_map.insert(oid, Box::new(e)); r } pub fn boot FnOnce(&'a mut Activation) -> BoxFuture<'a, ActorResult>>( mut self, name: tracing::Span, boot: F, ) -> ActorHandle { name.record("actor_id", &self.id()); tokio::spawn(async move { tracing::trace!("start"); let result = self.run(boot).await; { let mut t = Activation::new(&mut self, Debtor::new(crate::name!("shutdown"))); for r in std::mem::take(&mut t.actor.exit_hooks) { match t.actor.oid_map.remove_entry(&r.addr.oid) { None => (), Some((k, mut e)) => { if let Err(err) = e.exit_hook(&mut t, &result).await { tracing::error!(err = debug(err), r = debug(&r), "error in exit hook"); } t.actor.oid_map.insert(k, e); } } } } match &result { Ok(()) => { tracing::trace!("normal stop"); () } Err(e) => tracing::error!("error stop: {}", e), } result }.instrument(name)) } pub fn start(self, name: tracing::Span) -> ActorHandle { self.boot(name, |_ac| Box::pin(ready(Ok(())))) } async fn run FnOnce(&'a mut Activation) -> BoxFuture<'a, ActorResult>>( &mut self, boot: F, ) -> ActorResult { let _id = self.id(); // tracing::trace!(_id, "boot"); boot(&mut Activation::new(self, Debtor::new(crate::name!("boot")))).await?; // tracing::trace!(_id, "run"); loop { match self.rx.as_mut().expect("present rx channel half").recv().await { None => Err(error("Unexpected channel close", _Any::new(false)))?, Some(m) => { let should_stop = self.handle(m).await?; if should_stop { return Ok(()); } } } } } pub fn add_exit_hook(&mut self, r: &Arc) { self.exit_hooks.push(Arc::clone(r)) } async fn handle(&mut self, m: SystemMessage) -> Result { match m { SystemMessage::Release => { tracing::trace!("SystemMessage::Release"); Ok(true) } SystemMessage::ReleaseOid(oid) => { tracing::trace!("SystemMessage::ReleaseOid({:?})", &oid); self.oid_map.remove(&oid); Ok(false) } SystemMessage::Turn(mut loaned_item) => { let mut events = std::mem::take(&mut loaned_item.item.0); let mut t = Activation::new(self, Arc::clone(&loaned_item.debtor)); loop { for TurnEvent { oid, event } in events.into_iter() { t.with_oid(&oid, |_| Ok(()), |t, e| match event { Event::Assert(b) => { let Assert { assertion: Assertion(assertion), handle } = *b; e.assert(t, assertion, handle) } Event::Retract(b) => { let Retract { handle } = *b; e.retract(t, handle) } Event::Message(b) => { let Message { body: Assertion(body) } = *b; e.message(t, body) } Event::Sync(b) => { let Sync { peer } = *b; e.sync(t, peer) } })?; } events = std::mem::take(&mut t.immediate_self); if events.is_empty() { break; } } Ok(false) } SystemMessage::Crash(e) => { tracing::trace!("SystemMessage::Crash({:?})", &e); Err(e)? } } } pub fn linked_task + Send + 'static>( &mut self, name: tracing::Span, boot: F, ) { let mailbox = self.mailbox(); let token = CancellationToken::new(); let task_id = self.next_task_id; self.next_task_id += 1; name.record("task_id", &task_id); { let token = token.clone(); tokio::spawn(async move { tracing::trace!(task_id, "linked task start"); select! { _ = token.cancelled() => { tracing::trace!(task_id, "linked task cancelled"); Ok(()) } result = boot => { match &result { Ok(()) => { tracing::trace!(task_id, "linked task normal stop"); () } Err(e) => { tracing::error!(task_id, "linked task error: {}", e); let _ = mailbox.tx.send(SystemMessage::Crash(e.clone())); () } } result } } }.instrument(name)); } self.linked_tasks.insert(task_id, token); } } impl Drop for Actor { fn drop(&mut self) { let mut rx = self.rx.take().expect("present rx channel half during drop"); rx.close(); for (_task_id, token) in std::mem::take(&mut self.linked_tasks).into_iter() { token.cancel(); } let to_clear = std::mem::take(&mut self.outbound_assertions); { let mut t = Activation::new(self, Debtor::new(crate::name!("drop"))); for (handle, r) in to_clear.into_iter() { tracing::trace!(h = debug(&handle), "retract on termination"); t.retract_known_ref(r, handle); } } tracing::trace!("Actor::drop"); } } impl Ref { #[must_use] pub async fn external_event(&self, debtor: &Arc, event: Event) -> ActorResult { self.addr.mailbox.send(debtor, Turn(vec![TurnEvent { oid: self.addr.oid.clone(), event }])) } #[must_use] pub async fn external_events(&self, debtor: &Arc, events: Vec) -> ActorResult { self.addr.mailbox.send(debtor, Turn(events.into_iter().map(|event| TurnEvent { oid: self.addr.oid.clone(), event, }).collect())) } pub fn attenuate(&self, attenuation: &sturdy::Attenuation) -> Result, CaveatError> { let mut r = Ref { addr: Arc::clone(&self.addr), attenuation: self.attenuation.clone(), }; r.attenuation.extend(attenuation.check()?); Ok(Arc::new(r)) } pub fn rewrite(&self, mut a: _Any) -> Option { for c in &self.attenuation { match c.rewrite(&a) { Some(v) => a = v, None => return None, } } Some(Assertion(a)) } } impl std::fmt::Debug for Ref { fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { if self.attenuation.is_empty() { write!(f, "⌜{}:{}⌝", self.addr.mailbox.actor_id, self.addr.oid.0) } else { write!(f, "⌜{}:{}\\{:?}⌝", self.addr.mailbox.actor_id, self.addr.oid.0, self.attenuation) } } } impl Drop for ObjectAddress { fn drop(&mut self) { let _ = self.mailbox.tx.send(SystemMessage::ReleaseOid(self.oid.clone())); () } } impl Domain for Ref {} impl std::convert::TryFrom<&IOValue> for Ref { type Error = preserves_schema::support::ParseError; fn try_from(_v: &IOValue) -> Result { panic!("Attempted to serialize Ref via IOValue"); } } impl std::convert::From<&Ref> for IOValue { fn from(_v: &Ref) -> IOValue { panic!("Attempted to deserialize Ref via IOValue"); } } #[macro_export] macro_rules! name { () => {tracing::info_span!(actor_id = tracing::field::Empty, task_id = tracing::field::Empty, oid = tracing::field::Empty)}; ($($item:tt)*) => {tracing::info_span!($($item)*, actor_id = tracing::field::Empty, task_id = tracing::field::Empty, oid = tracing::field::Empty)} }