(Failed?) Experiment: alternate approach to reporting Facet::activate failures

This commit is contained in:
Tony Garnock-Jones 2022-01-15 23:24:34 +01:00
parent a37a2739a0
commit b2ee8daf4b
7 changed files with 122 additions and 56 deletions

View File

@ -43,7 +43,7 @@ pub fn run_connection(
initial_ref: Arc<Cap>, initial_ref: Arc<Cap>,
) -> ActorResult { ) -> ActorResult {
facet.activate(Account::new(syndicate::name!("start-session")), facet.activate(Account::new(syndicate::name!("start-session")),
|t| run_io_relay(t, i, o, initial_ref)) |t| run_io_relay(t, i, o, initial_ref)).into()
} }
pub async fn detect_protocol( pub async fn detect_protocol(

View File

@ -181,14 +181,16 @@ fn run(
let mut path_state: Map<PathBuf, FacetId> = Map::new(); let mut path_state: Map<PathBuf, FacetId> = Map::new();
{ let initial_scan_result = facet.activate(
facet.activate(Account::new(syndicate::name!("initial_scan")), |t| { Account::new(syndicate::name!("initial_scan")), |t| {
initial_scan(t, &mut path_state, &config_ds, env.clone()); initial_scan(t, &mut path_state, &config_ds, env.clone());
config_ds.assert(t, language(), &lifecycle::ready(&spec)); config_ds.assert(t, language(), &lifecycle::ready(&spec));
Ok(()) Ok(())
}).unwrap(); });
tracing::trace!("initial_scan complete"); if initial_scan_result.is_already_terminated() {
return;
} }
tracing::trace!("initial_scan complete");
let mut rescan = |paths: Vec<PathBuf>| { let mut rescan = |paths: Vec<PathBuf>| {
facet.activate(Account::new(syndicate::name!("rescan")), |t| { facet.activate(Account::new(syndicate::name!("rescan")), |t| {
@ -209,24 +211,30 @@ fn run(
t.stop_facet(facet_id); t.stop_facet(facet_id);
} }
Ok(()) Ok(())
}).unwrap() }).as_result()
}; };
while let Ok(event) = rx.recv() { while let Ok(event) = rx.recv() {
tracing::trace!("notification: {:?}", &event); tracing::trace!("notification: {:?}", &event);
match event { if
DebouncedEvent::NoticeWrite(_p) | match event {
DebouncedEvent::NoticeRemove(_p) => DebouncedEvent::NoticeWrite(_p) |
(), DebouncedEvent::NoticeRemove(_p) =>
DebouncedEvent::Create(p) | Ok(()),
DebouncedEvent::Write(p) | DebouncedEvent::Create(p) |
DebouncedEvent::Chmod(p) | DebouncedEvent::Write(p) |
DebouncedEvent::Remove(p) => DebouncedEvent::Chmod(p) |
rescan(vec![p]), DebouncedEvent::Remove(p) =>
DebouncedEvent::Rename(p, q) => rescan(vec![p]),
rescan(vec![p, q]), DebouncedEvent::Rename(p, q) =>
_ => rescan(vec![p, q]),
tracing::info!("{:?}", event), _ => {
tracing::info!("{:?}", event);
Ok(())
}
}.is_err()
{
return;
} }
} }

View File

@ -249,7 +249,7 @@ impl DaemonInstance {
Err(_) => AnyValue::bytestring(buf), Err(_) => AnyValue::bytestring(buf),
}; };
let now = AnyValue::new(chrono::Utc::now().to_rfc3339()); let now = AnyValue::new(chrono::Utc::now().to_rfc3339());
if facet.activate( if !facet.activate(
Account::new(tracing::Span::current()), Account::new(tracing::Span::current()),
enclose!((pid, service, kind) |t| { enclose!((pid, service, kind) |t| {
log_ds.message(t, &(), &syndicate_macros::template!( log_ds.message(t, &(), &syndicate_macros::template!(
@ -260,7 +260,7 @@ impl DaemonInstance {
line: =buf, line: =buf,
}>")); }>"));
Ok(()) Ok(())
})).is_err() })).is_success()
{ {
break; break;
} }
@ -313,7 +313,7 @@ impl DaemonInstance {
facet.activate(Account::new(syndicate::name!("instance-terminated")), |t| { facet.activate(Account::new(syndicate::name!("instance-terminated")), |t| {
let m = if status.success() { None } else { Some(format!("{}", status)) }; let m = if status.success() { None } else { Some(format!("{}", status)) };
self.handle_exit(t, m) self.handle_exit(t, m)
})?; }).ignore_termination()?;
Ok(LinkedTaskTermination::Normal) Ok(LinkedTaskTermination::Normal)
})); }));
Ok(()) Ok(())
@ -443,7 +443,7 @@ fn run(
facet.activate(Account::new(syndicate::name!("instance-startup")), |t| { facet.activate(Account::new(syndicate::name!("instance-startup")), |t| {
daemon_instance.start(t) daemon_instance.start(t)
})?; }).ignore_termination()?;
Ok(LinkedTaskTermination::KeepFacet) Ok(LinkedTaskTermination::KeepFacet)
}); });
Ok(()) Ok(())

View File

@ -36,16 +36,16 @@ fn run(t: &mut Activation, ds: Arc<Cap>, spec: TcpRelayListener) -> ActorResult
t.linked_task(syndicate::name!("listener"), async move { t.linked_task(syndicate::name!("listener"), async move {
let listen_addr = format!("{}:{}", host, port); let listen_addr = format!("{}:{}", host, port);
let listener = TcpListener::bind(listen_addr).await?; let listener = TcpListener::bind(listen_addr).await?;
facet.activate(Account::new(syndicate::name!("readiness")), |t| { if !facet.activate(Account::new(syndicate::name!("readiness")), |t| {
tracing::info!("listening"); tracing::info!("listening");
ds.assert(t, language(), &lifecycle::ready(&spec)); ds.assert(t, language(), &lifecycle::ready(&spec));
Ok(()) Ok(())
})?; }).is_success() { return Ok(LinkedTaskTermination::Normal); }
loop { loop {
let (stream, addr) = listener.accept().await?; let (stream, addr) = listener.accept().await?;
let gatekeeper = spec.gatekeeper.clone(); let gatekeeper = spec.gatekeeper.clone();
let name = syndicate::name!(parent: parent_span.clone(), "conn"); let name = syndicate::name!(parent: parent_span.clone(), "conn");
facet.activate(Account::new(name.clone()), move |t| { if !facet.activate(Account::new(name.clone()), move |t| {
t.spawn(name, move |t| { t.spawn(name, move |t| {
Ok(t.linked_task(tracing::Span::current(), { Ok(t.linked_task(tracing::Span::current(), {
let facet = t.facet.clone(); let facet = t.facet.clone();
@ -56,7 +56,7 @@ fn run(t: &mut Activation, ds: Arc<Cap>, spec: TcpRelayListener) -> ActorResult
})) }))
}); });
Ok(()) Ok(())
})?; }).is_success() { return Ok(LinkedTaskTermination::Normal); }
} }
}); });
Ok(()) Ok(())

View File

@ -38,11 +38,11 @@ fn run(t: &mut Activation, ds: Arc<Cap>, spec: UnixRelayListener) -> ActorResult
let facet = t.facet.clone(); let facet = t.facet.clone();
t.linked_task(syndicate::name!("listener"), async move { t.linked_task(syndicate::name!("listener"), async move {
let listener = bind_unix_listener(&PathBuf::from(path_str)).await?; let listener = bind_unix_listener(&PathBuf::from(path_str)).await?;
facet.activate(Account::new(syndicate::name!("readiness")), |t| { if !facet.activate(Account::new(syndicate::name!("readiness")), |t| {
tracing::info!("listening"); tracing::info!("listening");
ds.assert(t, language(), &lifecycle::ready(&spec)); ds.assert(t, language(), &lifecycle::ready(&spec));
Ok(()) Ok(())
})?; }).is_success() { return Ok(LinkedTaskTermination::Normal); }
loop { loop {
let (stream, _addr) = listener.accept().await?; let (stream, _addr) = listener.accept().await?;
let peer = stream.peer_cred()?; let peer = stream.peer_cred()?;
@ -50,7 +50,7 @@ fn run(t: &mut Activation, ds: Arc<Cap>, spec: UnixRelayListener) -> ActorResult
let name = syndicate::name!(parent: parent_span.clone(), "conn", let name = syndicate::name!(parent: parent_span.clone(), "conn",
pid = ?peer.pid().unwrap_or(-1), pid = ?peer.pid().unwrap_or(-1),
uid = peer.uid()); uid = peer.uid());
facet.activate(Account::new(name.clone()), move |t| { if !facet.activate(Account::new(name.clone()), move |t| {
t.spawn(name, |t| { t.spawn(name, |t| {
Ok(t.linked_task(tracing::Span::current(), { Ok(t.linked_task(tracing::Span::current(), {
let facet = t.facet.clone(); let facet = t.facet.clone();
@ -66,7 +66,7 @@ fn run(t: &mut Activation, ds: Arc<Cap>, spec: UnixRelayListener) -> ActorResult
})) }))
}); });
Ok(()) Ok(())
})?; }).is_success() { return Ok(LinkedTaskTermination::Normal); }
} }
}); });
Ok(()) Ok(())

View File

@ -7,7 +7,6 @@
use super::dataflow::Graph; use super::dataflow::Graph;
use super::error::Error; use super::error::Error;
use super::error::encode_error;
use super::error::error; use super::error::error;
use super::rewrite::CaveatError; use super::rewrite::CaveatError;
use super::rewrite::CheckedCaveat; use super::rewrite::CheckedCaveat;
@ -455,6 +454,18 @@ pub enum RunDisposition {
Terminate(ActorResult), Terminate(ActorResult),
} }
/// Returned from [`Facet::activate`] and [`Facet::activate_exit`].
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[must_use]
pub enum ActivationResult {
/// The actor to be activated had already terminated by the time of the activation attempt.
AlreadyTerminated,
/// The activation succeeded.
Success,
/// The activation failed,
Failure(Error),
}
/// [Linked tasks][Activation::linked_task] terminate yielding values of this type. /// [Linked tasks][Activation::linked_task] terminate yielding values of this type.
pub enum LinkedTaskTermination { pub enum LinkedTaskTermination {
/// Causes the task's associated [Facet] to be [stop][Activation::stop]ped when the task /// Causes the task's associated [Facet] to be [stop][Activation::stop]ped when the task
@ -561,7 +572,7 @@ impl FacetRef {
&self, &self,
account: Arc<Account>, account: Arc<Account>,
f: F, f: F,
) -> ActorResult where ) -> ActivationResult where
F: FnOnce(&mut Activation) -> ActorResult, F: FnOnce(&mut Activation) -> ActorResult,
{ {
self.activate_exit(account, |t| f(t).into()) self.activate_exit(account, |t| f(t).into())
@ -577,14 +588,15 @@ impl FacetRef {
&self, &self,
account: Arc<Account>, account: Arc<Account>,
f: F, f: F,
) -> ActorResult where ) -> ActivationResult where
F: FnOnce(&mut Activation) -> RunDisposition, F: FnOnce(&mut Activation) -> RunDisposition,
{ {
let mut g = self.actor.state.lock(); let mut g = self.actor.state.lock();
match &mut *g { match &mut *g {
ActorState::Terminated { exit_status } => ActorState::Terminated { exit_status } => {
Err(error("Could not activate terminated actor", tracing::debug!(?exit_status, "Could not activate terminated actor");
encode_error((**exit_status).clone()))), ActivationResult::AlreadyTerminated
}
ActorState::Running(state) => { ActorState::Running(state) => {
tracing::trace!(actor_id=?self.actor.actor_id, "activate"); tracing::trace!(actor_id=?self.actor.actor_id, "activate");
let mut activation = Activation::make(self, account, state); let mut activation = Activation::make(self, account, state);
@ -605,7 +617,10 @@ impl FacetRef {
} }
}; };
tracing::trace!(actor_id=?self.actor.actor_id, "deactivate"); tracing::trace!(actor_id=?self.actor.actor_id, "deactivate");
result match result {
Ok(()) => ActivationResult::Success,
Err(e) => ActivationResult::Failure(e),
}
} }
} }
} }
@ -976,7 +991,9 @@ impl<'activation> Activation<'activation> {
loop { loop {
timer.tick().await; timer.tick().await;
let _entry = span.enter(); let _entry = span.enter();
facet.activate(Arc::clone(&account), |t| a(t))?; if facet.activate(Arc::clone(&account), |t| a(t)).as_result().is_err() {
return Ok(LinkedTaskTermination::Normal);
}
} }
}); });
Ok(()) Ok(())
@ -996,7 +1013,7 @@ impl<'activation> Activation<'activation> {
self.linked_task(crate::name!(parent: None, "Activation::at"), async move { self.linked_task(crate::name!(parent: None, "Activation::at"), async move {
tokio::time::sleep_until(instant.into()).await; tokio::time::sleep_until(instant.into()).await;
let _entry = span.enter(); let _entry = span.enter();
facet.activate(account, a)?; facet.activate(account, a).ignore_termination()?;
Ok(LinkedTaskTermination::KeepFacet) Ok(LinkedTaskTermination::KeepFacet)
}); });
} }
@ -1031,7 +1048,7 @@ impl<'activation> Activation<'activation> {
let facet_id = self.facet.facet_id; let facet_id = self.facet.facet_id;
self.pending.for_myself.push(Box::new(move |t| { self.pending.for_myself.push(Box::new(move |t| {
t.with_facet(true, facet_id, move |t| { t.with_facet(true, facet_id, move |t| {
ac.link(t).boot(name, boot); ac.link(t)?.boot(name, boot);
Ok(()) Ok(())
}) })
})); }));
@ -1526,7 +1543,7 @@ impl Actor {
Actor { rx, ac_ref } Actor { rx, ac_ref }
} }
fn link(self, t_parent: &mut Activation) -> Self { fn link(self, t_parent: &mut Activation) -> Result<Self, Error> {
if t_parent.active_facet().is_none() { if t_parent.active_facet().is_none() {
panic!("No active facet when calling spawn_link"); panic!("No active facet when calling spawn_link");
} }
@ -1534,8 +1551,8 @@ impl Actor {
t_parent.half_link(t_child); t_parent.half_link(t_child);
t_child.half_link(t_parent); t_child.half_link(t_parent);
Ok(()) Ok(())
}).expect("Failed during link"); }).as_result()?;
self Ok(self)
} }
/// Start the actor's mainloop. Takes ownership of `self`. The /// Start the actor's mainloop. Takes ownership of `self`. The
@ -1575,7 +1592,7 @@ impl Actor {
|_| RunDisposition::Terminate(result)); |_| RunDisposition::Terminate(result));
}; };
if root_facet_ref.activate(Account::new(crate::name!("boot")), boot).is_err() { if !root_facet_ref.activate(Account::new(crate::name!("boot")), boot).is_success() {
return; return;
} }
@ -1604,7 +1621,7 @@ impl Actor {
for action in actions.into_iter() { action(t)? } for action in actions.into_iter() { action(t)? }
Ok(()) Ok(())
}); });
if r.is_err() { return; } if !r.is_success() { return; }
} }
SystemMessage::Crash(e) => { SystemMessage::Crash(e) => {
tracing::trace!(actor_id = ?self.ac_ref.actor_id, tracing::trace!(actor_id = ?self.ac_ref.actor_id,
@ -2069,6 +2086,41 @@ impl<M> Entity<M> for StopOnRetract {
} }
} }
impl ActivationResult {
pub fn is_success(&self) -> bool {
self == &ActivationResult::Success
}
pub fn as_result(self) -> ActorResult {
self.into()
}
pub fn ignore_termination(self) -> ActorResult {
match self {
ActivationResult::AlreadyTerminated => Ok(()),
ActivationResult::Failure(e) => Err(e),
ActivationResult::Success => Ok(()),
}
}
pub fn is_already_terminated(&self) -> bool {
self == &ActivationResult::AlreadyTerminated
}
}
impl From<ActivationResult> for ActorResult {
fn from(r: ActivationResult) -> ActorResult {
match r {
ActivationResult::AlreadyTerminated =>
Err(error("New actor crashed unexpectedly in spawn_link", AnyValue::new(false))),
ActivationResult::Failure(e) =>
Err(e),
ActivationResult::Success =>
Ok(()),
}
}
}
impl<F: Send + FnMut(&mut Activation) -> ActorResult> Entity<Synced> for F { impl<F: Send + FnMut(&mut Activation) -> ActorResult> Entity<Synced> for F {
fn message(&mut self, t: &mut Activation, _m: Synced) -> ActorResult { fn message(&mut self, t: &mut Activation, _m: Synced) -> ActorResult {
self(t) self(t)

View File

@ -625,11 +625,14 @@ async fn input_loop(
account.ensure_clear_funds().await; account.ensure_clear_funds().await;
match src.next().await { match src.next().await {
None => return Ok(LinkedTaskTermination::Normal), None => return Ok(LinkedTaskTermination::Normal),
Some(bs) => facet.activate(Arc::clone(&account), |t| { Some(bs) => {
let mut g = relay.lock(); let r = facet.activate(Arc::clone(&account), |t| {
let tr = g.as_mut().expect("initialized"); let mut g = relay.lock();
tr.handle_inbound_datagram(t, &bs?) let tr = g.as_mut().expect("initialized");
})?, tr.handle_inbound_datagram(t, &bs?)
});
if !r.is_success() { return Ok(LinkedTaskTermination::Normal); }
}
} }
} }
} }
@ -650,11 +653,14 @@ async fn input_loop(
}; };
match n { match n {
0 => return Ok(LinkedTaskTermination::Normal), 0 => return Ok(LinkedTaskTermination::Normal),
_ => facet.activate(Arc::clone(&account), |t| { _ => {
let mut g = relay.lock(); let r = facet.activate(Arc::clone(&account), |t| {
let tr = g.as_mut().expect("initialized"); let mut g = relay.lock();
tr.handle_inbound_stream(t, &mut buf) let tr = g.as_mut().expect("initialized");
})?, tr.handle_inbound_stream(t, &mut buf)
});
if !r.is_success() { return Ok(LinkedTaskTermination::Normal); }
}
} }
} }
} }