syndicate-lang
/
syndicate-rs
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Supervisor RestartPolicy

This commit is contained in:
Tony Garnock-Jones 2021-09-01 17:31:01 +02:00
parent 74ca267cef
commit e90fe2c41e
6 changed files with 89 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
syndicate-server/src/services/config_watcher.rs
View File

@ -46,7 +46,7 @@ pub fn on_demand(t: &mut Activation, ds: Arc<Cap>) {
observer: monitor,
});
Ok(())
})
});
}
fn convert_notify_error(e: notify::Error) -> syndicate::error::Error {

2
syndicate-server/src/services/debt_reporter.rs
View File

@ -25,7 +25,7 @@ pub fn on_demand(t: &mut Activation, ds: Arc<Cap>) {
observer: monitor,
});
Ok(())
})
});
}
fn run(t: &mut Activation, ds: Arc<Cap>) -> ActorResult {

2
syndicate-server/src/services/tcp_relay_listener.rs
View File

@ -35,7 +35,7 @@ pub fn on_demand(t: &mut Activation, ds: Arc<Cap>, gateway: Arc<Cap>) {
observer: monitor,
});
Ok(())
})
});
}
fn run(

2
syndicate-server/src/services/unix_relay_listener.rs
View File

@ -36,7 +36,7 @@ pub fn on_demand(t: &mut Activation, ds: Arc<Cap>, gateway: Arc<Cap>) {
observer: monitor,
});
Ok(())
})
});
}
fn run(

14
syndicate/src/actor.rs
View File

@ -859,11 +859,14 @@ impl<'activation> Activation<'activation> {
&mut self,
name: tracing::Span,
boot: F,
) {
) -> ActorRef {
let ac = Actor::new();
let ac_ref = ac.ac_ref.clone();
self.enqueue_for_myself_at_commit(Box::new(move |_| {
Actor::new().boot(name, boot);
ac.boot(name, boot);
Ok(())
}));
ac_ref
}
/// Schedule the creation of a new actor when the Activation commits.
@ -875,14 +878,17 @@ impl<'activation> Activation<'activation> {
&mut self,
name: tracing::Span,
boot: F,
) {
) -> ActorRef {
let ac = Actor::new();
let ac_ref = ac.ac_ref.clone();
let facet_id = self.facet.facet_id;
self.enqueue_for_myself_at_commit(Box::new(move |t| {
t.with_facet(true, facet_id, move |t| {
Actor::new().link(t).boot(name, boot);
ac.link(t).boot(name, boot);
Ok(())
})
}));
ac_ref
}
/// Create a new subfacet of the currently-active facet. Runs `boot` in the new facet's

112
syndicate/src/supervise.rs
View File

@ -15,12 +15,19 @@ enum Protocol<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> {
Retry, // message
}
#[derive(Debug)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RestartPolicy {
Always,
OnErrorOnly,
}
#[derive(Debug, Clone)]
pub struct SupervisorConfiguration {
pub intensity: usize,
pub period: Duration,
pub pause_time: Duration,
pub sleep_time: Duration,
pub restart_policy: RestartPolicy,
}
pub struct Supervisor<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> {
@ -30,6 +37,7 @@ pub struct Supervisor<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResu
boot_fn: Option<Boot>,
restarts: VecDeque<Instant>,
supervisee: Supervisee,
ac_ref: Option<ActorRef>,
}
#[derive(Debug, PartialEq, Eq)]
@ -57,6 +65,7 @@ impl Default for SupervisorConfiguration {
period: Duration::from_secs(5),
pause_time: Duration::from_millis(200),
sleep_time: Duration::from_secs(10),
restart_policy: RestartPolicy::Always,
}
}
}
@ -73,27 +82,43 @@ impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult>
}
fn retract(&mut self, t: &mut Activation, _h: Handle) -> ActorResult {
self.enter_state(Supervisee::Recovering);
let now = Instant::now();
let oldest_to_keep = now - self.config.period;
self.restarts.push_back(now);
while let Some(stamp) = self.restarts.front() {
if stamp < &oldest_to_keep {
self.restarts.pop_front();
} else {
break;
}
let _name = self.name.clone();
let _entry = _name.enter();
let exit_status =
self.ac_ref.take().expect("valid supervisee ActorRef")
.exit_status()
.expect("supervisee to have terminated");
tracing::debug!(?exit_status);
match exit_status {
Ok(()) if self.config.restart_policy == RestartPolicy::OnErrorOnly => {
tracing::trace!("Not restarting: normal exit, restart_policy is OnErrorOnly");
self.enter_state(Supervisee::NotRunning);
},
_ => {
tracing::trace!("Restarting: restart_policy is Always or exit was abnormal");
self.enter_state(Supervisee::Recovering);
let now = Instant::now();
let oldest_to_keep = now - self.config.period;
self.restarts.push_back(now);
while let Some(stamp) = self.restarts.front() {
if stamp < &oldest_to_keep {
self.restarts.pop_front();
} else {
break;
}
}
let self_ref = Arc::clone(&self.self_ref);
let wait_time = if self.restarts.len() > self.config.intensity {
self.config.sleep_time
} else {
self.config.pause_time
};
t.after(wait_time, Box::new(move |t| {
t.message(&self_ref, Protocol::Retry);
Ok(())
}));
},
}
let self_ref = Arc::clone(&self.self_ref);
let wait_time = if self.restarts.len() > self.config.intensity {
self.config.sleep_time
} else {
self.config.pause_time
};
t.after(wait_time, Box::new(move |t| {
t.message(&self_ref, Protocol::Retry);
Ok(())
}));
Ok(())
}
@ -111,6 +136,12 @@ impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult>
_ => Ok(())
}
}
fn stop(&mut self, _t: &mut Activation) -> ActorResult {
let _entry = self.name.enter();
tracing::info!("Supervisor terminating");
Ok(())
}
}
impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> Supervisor<Boot> {
@ -120,18 +151,21 @@ impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> Supervisor<Bo
config: SupervisorConfiguration,
boot_fn: Boot,
) {
let _entry = name.enter();
tracing::trace!(?config);
let self_ref = t.create_inert();
let mut supervisor = Supervisor {
self_ref: Arc::clone(&self_ref),
name,
name: name.clone(),
config,
boot_fn: Some(boot_fn),
restarts: VecDeque::new(),
supervisee: Supervisee::NotRunning,
ac_ref: None,
};
supervisor.ensure_started(t).unwrap();
self_ref.become_entity(supervisor);
t.on_stop(&self_ref);
}
fn enter_state(&mut self, supervisee: Supervisee) {
@ -142,26 +176,30 @@ impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> Supervisor<Bo
fn ensure_started(&mut self, t: &mut Activation) -> ActorResult {
match self.boot_fn.take() {
None =>
tracing::error!("Cannot restart supervisee, because it panicked at startup"),
None => {
let _entry = self.name.enter();
tracing::error!("Cannot restart supervisee, because it panicked at startup")
}
Some(mut boot_fn) => {
let self_ref = Arc::clone(&self.self_ref);
t.facet(|t: &mut Activation| {
t.assert(&self.self_ref, Protocol::SuperviseeStarted);
t.spawn_link(crate::name!(parent: &self.name, "supervisee"), move |t| {
match boot_fn(t) {
Ok(()) => {
t.message(&self_ref, Protocol::BootFunction(boot_fn));
Ok(())
self.ac_ref = Some(t.spawn_link(
crate::name!(parent: &self.name, "supervisee"),
move |t| {
match boot_fn(t) {
Ok(()) => {
t.message(&self_ref, Protocol::BootFunction(boot_fn));
Ok(())
}
Err(e) => {
t.clear();
t.message(&self_ref, Protocol::BootFunction(boot_fn));
t.deliver();
Err(e)
}
}
Err(e) => {
t.clear();
t.message(&self_ref, Protocol::BootFunction(boot_fn));
t.deliver();
Err(e)
}
}
});
}));
Ok(())
})?;
}