Supervisor RestartPolicy
This commit is contained in:
parent
74ca267cef
commit
e90fe2c41e
|
@ -46,7 +46,7 @@ pub fn on_demand(t: &mut Activation, ds: Arc<Cap>) {
|
||||||
observer: monitor,
|
observer: monitor,
|
||||||
});
|
});
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
fn convert_notify_error(e: notify::Error) -> syndicate::error::Error {
|
fn convert_notify_error(e: notify::Error) -> syndicate::error::Error {
|
||||||
|
|
|
@ -25,7 +25,7 @@ pub fn on_demand(t: &mut Activation, ds: Arc<Cap>) {
|
||||||
observer: monitor,
|
observer: monitor,
|
||||||
});
|
});
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run(t: &mut Activation, ds: Arc<Cap>) -> ActorResult {
|
fn run(t: &mut Activation, ds: Arc<Cap>) -> ActorResult {
|
||||||
|
|
|
@ -35,7 +35,7 @@ pub fn on_demand(t: &mut Activation, ds: Arc<Cap>, gateway: Arc<Cap>) {
|
||||||
observer: monitor,
|
observer: monitor,
|
||||||
});
|
});
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run(
|
fn run(
|
||||||
|
|
|
@ -36,7 +36,7 @@ pub fn on_demand(t: &mut Activation, ds: Arc<Cap>, gateway: Arc<Cap>) {
|
||||||
observer: monitor,
|
observer: monitor,
|
||||||
});
|
});
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run(
|
fn run(
|
||||||
|
|
|
@ -859,11 +859,14 @@ impl<'activation> Activation<'activation> {
|
||||||
&mut self,
|
&mut self,
|
||||||
name: tracing::Span,
|
name: tracing::Span,
|
||||||
boot: F,
|
boot: F,
|
||||||
) {
|
) -> ActorRef {
|
||||||
|
let ac = Actor::new();
|
||||||
|
let ac_ref = ac.ac_ref.clone();
|
||||||
self.enqueue_for_myself_at_commit(Box::new(move |_| {
|
self.enqueue_for_myself_at_commit(Box::new(move |_| {
|
||||||
Actor::new().boot(name, boot);
|
ac.boot(name, boot);
|
||||||
Ok(())
|
Ok(())
|
||||||
}));
|
}));
|
||||||
|
ac_ref
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Schedule the creation of a new actor when the Activation commits.
|
/// Schedule the creation of a new actor when the Activation commits.
|
||||||
|
@ -875,14 +878,17 @@ impl<'activation> Activation<'activation> {
|
||||||
&mut self,
|
&mut self,
|
||||||
name: tracing::Span,
|
name: tracing::Span,
|
||||||
boot: F,
|
boot: F,
|
||||||
) {
|
) -> ActorRef {
|
||||||
|
let ac = Actor::new();
|
||||||
|
let ac_ref = ac.ac_ref.clone();
|
||||||
let facet_id = self.facet.facet_id;
|
let facet_id = self.facet.facet_id;
|
||||||
self.enqueue_for_myself_at_commit(Box::new(move |t| {
|
self.enqueue_for_myself_at_commit(Box::new(move |t| {
|
||||||
t.with_facet(true, facet_id, move |t| {
|
t.with_facet(true, facet_id, move |t| {
|
||||||
Actor::new().link(t).boot(name, boot);
|
ac.link(t).boot(name, boot);
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
}));
|
}));
|
||||||
|
ac_ref
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a new subfacet of the currently-active facet. Runs `boot` in the new facet's
|
/// Create a new subfacet of the currently-active facet. Runs `boot` in the new facet's
|
||||||
|
|
|
@ -15,12 +15,19 @@ enum Protocol<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> {
|
||||||
Retry, // message
|
Retry, // message
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum RestartPolicy {
|
||||||
|
Always,
|
||||||
|
OnErrorOnly,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
pub struct SupervisorConfiguration {
|
pub struct SupervisorConfiguration {
|
||||||
pub intensity: usize,
|
pub intensity: usize,
|
||||||
pub period: Duration,
|
pub period: Duration,
|
||||||
pub pause_time: Duration,
|
pub pause_time: Duration,
|
||||||
pub sleep_time: Duration,
|
pub sleep_time: Duration,
|
||||||
|
pub restart_policy: RestartPolicy,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Supervisor<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> {
|
pub struct Supervisor<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> {
|
||||||
|
@ -30,6 +37,7 @@ pub struct Supervisor<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResu
|
||||||
boot_fn: Option<Boot>,
|
boot_fn: Option<Boot>,
|
||||||
restarts: VecDeque<Instant>,
|
restarts: VecDeque<Instant>,
|
||||||
supervisee: Supervisee,
|
supervisee: Supervisee,
|
||||||
|
ac_ref: Option<ActorRef>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq)]
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
@ -57,6 +65,7 @@ impl Default for SupervisorConfiguration {
|
||||||
period: Duration::from_secs(5),
|
period: Duration::from_secs(5),
|
||||||
pause_time: Duration::from_millis(200),
|
pause_time: Duration::from_millis(200),
|
||||||
sleep_time: Duration::from_secs(10),
|
sleep_time: Duration::from_secs(10),
|
||||||
|
restart_policy: RestartPolicy::Always,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -73,27 +82,43 @@ impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult>
|
||||||
}
|
}
|
||||||
|
|
||||||
fn retract(&mut self, t: &mut Activation, _h: Handle) -> ActorResult {
|
fn retract(&mut self, t: &mut Activation, _h: Handle) -> ActorResult {
|
||||||
self.enter_state(Supervisee::Recovering);
|
let _name = self.name.clone();
|
||||||
let now = Instant::now();
|
let _entry = _name.enter();
|
||||||
let oldest_to_keep = now - self.config.period;
|
let exit_status =
|
||||||
self.restarts.push_back(now);
|
self.ac_ref.take().expect("valid supervisee ActorRef")
|
||||||
while let Some(stamp) = self.restarts.front() {
|
.exit_status()
|
||||||
if stamp < &oldest_to_keep {
|
.expect("supervisee to have terminated");
|
||||||
self.restarts.pop_front();
|
tracing::debug!(?exit_status);
|
||||||
} else {
|
match exit_status {
|
||||||
break;
|
Ok(()) if self.config.restart_policy == RestartPolicy::OnErrorOnly => {
|
||||||
}
|
tracing::trace!("Not restarting: normal exit, restart_policy is OnErrorOnly");
|
||||||
|
self.enter_state(Supervisee::NotRunning);
|
||||||
|
},
|
||||||
|
_ => {
|
||||||
|
tracing::trace!("Restarting: restart_policy is Always or exit was abnormal");
|
||||||
|
self.enter_state(Supervisee::Recovering);
|
||||||
|
let now = Instant::now();
|
||||||
|
let oldest_to_keep = now - self.config.period;
|
||||||
|
self.restarts.push_back(now);
|
||||||
|
while let Some(stamp) = self.restarts.front() {
|
||||||
|
if stamp < &oldest_to_keep {
|
||||||
|
self.restarts.pop_front();
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let self_ref = Arc::clone(&self.self_ref);
|
||||||
|
let wait_time = if self.restarts.len() > self.config.intensity {
|
||||||
|
self.config.sleep_time
|
||||||
|
} else {
|
||||||
|
self.config.pause_time
|
||||||
|
};
|
||||||
|
t.after(wait_time, Box::new(move |t| {
|
||||||
|
t.message(&self_ref, Protocol::Retry);
|
||||||
|
Ok(())
|
||||||
|
}));
|
||||||
|
},
|
||||||
}
|
}
|
||||||
let self_ref = Arc::clone(&self.self_ref);
|
|
||||||
let wait_time = if self.restarts.len() > self.config.intensity {
|
|
||||||
self.config.sleep_time
|
|
||||||
} else {
|
|
||||||
self.config.pause_time
|
|
||||||
};
|
|
||||||
t.after(wait_time, Box::new(move |t| {
|
|
||||||
t.message(&self_ref, Protocol::Retry);
|
|
||||||
Ok(())
|
|
||||||
}));
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -111,6 +136,12 @@ impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult>
|
||||||
_ => Ok(())
|
_ => Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn stop(&mut self, _t: &mut Activation) -> ActorResult {
|
||||||
|
let _entry = self.name.enter();
|
||||||
|
tracing::info!("Supervisor terminating");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> Supervisor<Boot> {
|
impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> Supervisor<Boot> {
|
||||||
|
@ -120,18 +151,21 @@ impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> Supervisor<Bo
|
||||||
config: SupervisorConfiguration,
|
config: SupervisorConfiguration,
|
||||||
boot_fn: Boot,
|
boot_fn: Boot,
|
||||||
) {
|
) {
|
||||||
|
let _entry = name.enter();
|
||||||
tracing::trace!(?config);
|
tracing::trace!(?config);
|
||||||
let self_ref = t.create_inert();
|
let self_ref = t.create_inert();
|
||||||
let mut supervisor = Supervisor {
|
let mut supervisor = Supervisor {
|
||||||
self_ref: Arc::clone(&self_ref),
|
self_ref: Arc::clone(&self_ref),
|
||||||
name,
|
name: name.clone(),
|
||||||
config,
|
config,
|
||||||
boot_fn: Some(boot_fn),
|
boot_fn: Some(boot_fn),
|
||||||
restarts: VecDeque::new(),
|
restarts: VecDeque::new(),
|
||||||
supervisee: Supervisee::NotRunning,
|
supervisee: Supervisee::NotRunning,
|
||||||
|
ac_ref: None,
|
||||||
};
|
};
|
||||||
supervisor.ensure_started(t).unwrap();
|
supervisor.ensure_started(t).unwrap();
|
||||||
self_ref.become_entity(supervisor);
|
self_ref.become_entity(supervisor);
|
||||||
|
t.on_stop(&self_ref);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn enter_state(&mut self, supervisee: Supervisee) {
|
fn enter_state(&mut self, supervisee: Supervisee) {
|
||||||
|
@ -142,26 +176,30 @@ impl<Boot: 'static + Send + FnMut(&mut Activation) -> ActorResult> Supervisor<Bo
|
||||||
|
|
||||||
fn ensure_started(&mut self, t: &mut Activation) -> ActorResult {
|
fn ensure_started(&mut self, t: &mut Activation) -> ActorResult {
|
||||||
match self.boot_fn.take() {
|
match self.boot_fn.take() {
|
||||||
None =>
|
None => {
|
||||||
tracing::error!("Cannot restart supervisee, because it panicked at startup"),
|
let _entry = self.name.enter();
|
||||||
|
tracing::error!("Cannot restart supervisee, because it panicked at startup")
|
||||||
|
}
|
||||||
Some(mut boot_fn) => {
|
Some(mut boot_fn) => {
|
||||||
let self_ref = Arc::clone(&self.self_ref);
|
let self_ref = Arc::clone(&self.self_ref);
|
||||||
t.facet(|t: &mut Activation| {
|
t.facet(|t: &mut Activation| {
|
||||||
t.assert(&self.self_ref, Protocol::SuperviseeStarted);
|
t.assert(&self.self_ref, Protocol::SuperviseeStarted);
|
||||||
t.spawn_link(crate::name!(parent: &self.name, "supervisee"), move |t| {
|
self.ac_ref = Some(t.spawn_link(
|
||||||
match boot_fn(t) {
|
crate::name!(parent: &self.name, "supervisee"),
|
||||||
Ok(()) => {
|
move |t| {
|
||||||
t.message(&self_ref, Protocol::BootFunction(boot_fn));
|
match boot_fn(t) {
|
||||||
Ok(())
|
Ok(()) => {
|
||||||
|
t.message(&self_ref, Protocol::BootFunction(boot_fn));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
t.clear();
|
||||||
|
t.message(&self_ref, Protocol::BootFunction(boot_fn));
|
||||||
|
t.deliver();
|
||||||
|
Err(e)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
}));
|
||||||
t.clear();
|
|
||||||
t.message(&self_ref, Protocol::BootFunction(boot_fn));
|
|
||||||
t.deliver();
|
|
||||||
Err(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})?;
|
})?;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue