2021-07-15 07:13:31 +00:00
|
|
|
use std::sync::Arc;
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
use std::sync::Mutex;
|
2021-07-15 07:13:31 +00:00
|
|
|
use std::time::SystemTime;
|
2020-06-10 08:41:12 +00:00
|
|
|
|
|
|
|
use structopt::StructOpt;
|
2021-07-15 07:13:31 +00:00
|
|
|
|
|
|
|
use syndicate::actor::*;
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
use syndicate::enclose;
|
2021-09-19 14:53:37 +00:00
|
|
|
use syndicate::language;
|
2021-07-15 07:13:31 +00:00
|
|
|
use syndicate::relay;
|
|
|
|
use syndicate::schemas::dataspace::Observe;
|
|
|
|
use syndicate::sturdy;
|
|
|
|
use syndicate::value::NestedValue;
|
|
|
|
use syndicate::value::Value;
|
|
|
|
|
2020-06-10 08:41:12 +00:00
|
|
|
use tokio::net::TcpStream;
|
|
|
|
|
2021-07-15 07:13:31 +00:00
|
|
|
use core::time::Duration;
|
2020-06-10 08:41:12 +00:00
|
|
|
|
|
|
|
#[derive(Clone, Debug, StructOpt)]
|
2020-06-12 19:30:15 +00:00
|
|
|
pub struct PingConfig {
|
|
|
|
#[structopt(short = "t", default_value = "1")]
|
|
|
|
turn_count: u32,
|
|
|
|
|
|
|
|
#[structopt(short = "a", default_value = "1")]
|
|
|
|
action_count: u32,
|
2020-06-10 08:41:12 +00:00
|
|
|
|
2020-06-12 19:30:15 +00:00
|
|
|
#[structopt(short = "l", default_value = "0")]
|
|
|
|
report_latency_every: usize,
|
2020-06-15 20:13:21 +00:00
|
|
|
|
|
|
|
#[structopt(short = "b", default_value = "0")]
|
|
|
|
bytes_padding: usize,
|
2020-06-12 19:30:15 +00:00
|
|
|
}
|
2020-06-10 08:41:12 +00:00
|
|
|
|
2020-06-12 19:30:15 +00:00
|
|
|
#[derive(Clone, Debug, StructOpt)]
|
|
|
|
pub enum PingPongMode {
|
|
|
|
Ping(PingConfig),
|
2020-06-10 08:41:12 +00:00
|
|
|
Pong,
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone, Debug, StructOpt)]
|
|
|
|
pub struct Config {
|
|
|
|
#[structopt(subcommand)]
|
|
|
|
mode: PingPongMode,
|
|
|
|
|
2023-02-08 23:17:12 +00:00
|
|
|
#[structopt(short = "d", default_value = "b4b303726566b7b3036f6964b10973796e646963617465b303736967b21069ca300c1dbfa08fba692102dd82311a8484")]
|
2020-06-10 08:41:12 +00:00
|
|
|
dataspace: String,
|
|
|
|
}
|
|
|
|
|
2021-07-15 07:13:31 +00:00
|
|
|
fn now() -> u64 {
|
|
|
|
SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).expect("time after epoch").as_nanos() as u64
|
2020-06-12 19:30:15 +00:00
|
|
|
}
|
|
|
|
|
2021-08-11 21:16:01 +00:00
|
|
|
fn simple_record2(label: &str, v1: AnyValue, v2: AnyValue) -> AnyValue {
|
2020-06-15 20:13:21 +00:00
|
|
|
let mut r = Value::simple_record(label, 2);
|
|
|
|
r.fields_vec_mut().push(v1);
|
|
|
|
r.fields_vec_mut().push(v2);
|
|
|
|
r.finish().wrap()
|
|
|
|
}
|
|
|
|
|
2020-06-12 19:30:15 +00:00
|
|
|
fn report_latencies(rtt_ns_samples: &Vec<u64>) {
|
|
|
|
let n = rtt_ns_samples.len();
|
|
|
|
let rtt_0 = rtt_ns_samples[0];
|
|
|
|
let rtt_50 = rtt_ns_samples[n * 1 / 2];
|
|
|
|
let rtt_90 = rtt_ns_samples[n * 90 / 100];
|
|
|
|
let rtt_95 = rtt_ns_samples[n * 95 / 100];
|
|
|
|
let rtt_99 = rtt_ns_samples[n * 99 / 100];
|
|
|
|
let rtt_99_9 = rtt_ns_samples[n * 999 / 1000];
|
|
|
|
let rtt_99_99 = rtt_ns_samples[n * 9999 / 10000];
|
|
|
|
let rtt_max = rtt_ns_samples[n - 1];
|
|
|
|
println!("rtt: 0% {:05.5}ms, 50% {:05.5}ms, 90% {:05.5}ms, 95% {:05.5}ms, 99% {:05.5}ms, 99.9% {:05.5}ms, 99.99% {:05.5}ms, max {:05.5}ms",
|
|
|
|
rtt_0 as f64 / 1000000.0,
|
|
|
|
rtt_50 as f64 / 1000000.0,
|
|
|
|
rtt_90 as f64 / 1000000.0,
|
|
|
|
rtt_95 as f64 / 1000000.0,
|
|
|
|
rtt_99 as f64 / 1000000.0,
|
|
|
|
rtt_99_9 as f64 / 1000000.0,
|
|
|
|
rtt_99_99 as f64 / 1000000.0,
|
|
|
|
rtt_max as f64 / 1000000.0);
|
|
|
|
println!("msg: 0% {:05.5}ms, 50% {:05.5}ms, 90% {:05.5}ms, 95% {:05.5}ms, 99% {:05.5}ms, 99.9% {:05.5}ms, 99.99% {:05.5}ms, max {:05.5}ms",
|
|
|
|
rtt_0 as f64 / 2000000.0,
|
|
|
|
rtt_50 as f64 / 2000000.0,
|
|
|
|
rtt_90 as f64 / 2000000.0,
|
|
|
|
rtt_95 as f64 / 2000000.0,
|
|
|
|
rtt_99 as f64 / 2000000.0,
|
|
|
|
rtt_99_9 as f64 / 2000000.0,
|
|
|
|
rtt_99_99 as f64 / 2000000.0,
|
|
|
|
rtt_max as f64 / 2000000.0);
|
|
|
|
}
|
|
|
|
|
2020-06-10 08:41:12 +00:00
|
|
|
#[tokio::main]
|
2023-01-30 13:25:58 +00:00
|
|
|
async fn main() -> ActorResult {
|
2021-07-15 07:13:31 +00:00
|
|
|
syndicate::convenient_logging()?;
|
2021-09-28 11:00:48 +00:00
|
|
|
let config = Config::from_args();
|
|
|
|
let sturdyref = sturdy::SturdyRef::from_hex(&config.dataspace)?;
|
2023-02-06 16:09:17 +00:00
|
|
|
let (i, o) = TcpStream::connect("127.0.0.1:9001").await?.into_split();
|
2022-01-19 13:40:50 +00:00
|
|
|
Actor::top(None, |t| {
|
2021-10-05 19:11:16 +00:00
|
|
|
relay::connect_stream(t, i, o, false, sturdyref, (), move |_state, t, ds| {
|
2021-09-28 11:00:48 +00:00
|
|
|
|
|
|
|
let (send_label, recv_label, report_latency_every, should_echo, bytes_padding) =
|
|
|
|
match config.mode {
|
|
|
|
PingPongMode::Ping(ref c) =>
|
|
|
|
("Ping", "Pong", c.report_latency_every, false, c.bytes_padding),
|
|
|
|
PingPongMode::Pong =>
|
|
|
|
("Pong", "Ping", 0, true, 0),
|
|
|
|
};
|
|
|
|
|
|
|
|
let consumer = {
|
|
|
|
let ds = Arc::clone(&ds);
|
|
|
|
let mut turn_counter: u64 = 0;
|
|
|
|
let mut event_counter: u64 = 0;
|
|
|
|
let mut rtt_ns_samples: Vec<u64> = vec![0; report_latency_every];
|
|
|
|
let mut rtt_batch_count: usize = 0;
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
let current_reply = Arc::new(Mutex::new(None));
|
|
|
|
Cap::new(&t.create(
|
|
|
|
syndicate::entity(())
|
|
|
|
.on_message(move |(), t, m: AnyValue| {
|
2021-09-28 11:00:48 +00:00
|
|
|
match m.value().as_boolean() {
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
Some(_) => {
|
2021-09-28 11:00:48 +00:00
|
|
|
tracing::info!("{:?} turns, {:?} events in the last second",
|
|
|
|
turn_counter,
|
|
|
|
event_counter);
|
|
|
|
turn_counter = 0;
|
|
|
|
event_counter = 0;
|
|
|
|
}
|
|
|
|
None => {
|
|
|
|
event_counter += 1;
|
|
|
|
let bindings = m.value().to_sequence()?;
|
|
|
|
let timestamp = &bindings[0];
|
|
|
|
let padding = &bindings[1];
|
|
|
|
|
|
|
|
if should_echo || (report_latency_every == 0) {
|
|
|
|
ds.message(t, &(), &simple_record2(&send_label,
|
|
|
|
timestamp.clone(),
|
|
|
|
padding.clone()));
|
|
|
|
} else {
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
let mut g = current_reply.lock().expect("unpoisoned");
|
|
|
|
if let None = *g {
|
2021-09-28 11:00:48 +00:00
|
|
|
turn_counter += 1;
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
t.pre_commit(enclose!((current_reply) move |_| {
|
|
|
|
*current_reply.lock().expect("unpoisoned") = None;
|
|
|
|
Ok(())
|
|
|
|
}));
|
2021-09-28 11:00:48 +00:00
|
|
|
let rtt_ns = now() - timestamp.value().to_u64()?;
|
|
|
|
rtt_ns_samples[rtt_batch_count] = rtt_ns;
|
|
|
|
rtt_batch_count += 1;
|
|
|
|
|
|
|
|
if rtt_batch_count == report_latency_every {
|
|
|
|
rtt_ns_samples.sort();
|
|
|
|
report_latencies(&rtt_ns_samples);
|
|
|
|
rtt_batch_count = 0;
|
2021-07-22 14:53:56 +00:00
|
|
|
}
|
2021-09-28 11:00:48 +00:00
|
|
|
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
*g = Some(simple_record2(&send_label,
|
|
|
|
Value::from(now()).wrap(),
|
|
|
|
padding.clone()));
|
2020-06-12 19:30:15 +00:00
|
|
|
}
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
ds.message(t, &(), g.as_ref().expect("some reply"));
|
2020-06-12 19:30:15 +00:00
|
|
|
}
|
|
|
|
}
|
2020-06-10 08:41:12 +00:00
|
|
|
}
|
2021-09-28 11:00:48 +00:00
|
|
|
Ok(())
|
MAJOR REFACTORING OF CORE ASSERTION-TRACKING STRUCTURES. Little impact on API. Read on for details.
2022-02-01 15:22:30 Two problems.
- If a stop action panics (in `_terminate_facet`), the Facet is dropped before its outbound
handles are removed. With the code as it stands, this leaks assertions (!!).
- The logic for removing an outbound handle seems to be running in the wrong facet context???
(See `f.outbound_handles.remove(&handle)` in the cleanup actions
- I think I need to remove the for_myself mechanism
- and add some callbacks to run only on successful commit
2022-02-02 12:12:33 This is hard.
Here's the current implementation:
- assert
- inserts into outbound_handles of active facet
- adds cleanup action describing how to do the retraction
- enqueues the assert action, which
- calls e.assert()
- retract
- looks up & removes the cleanup action, which
- enqueues the retract action, which
- removes from outbound_handles of the WRONG facet in the WRONG actor
- calls e.retract()
- _terminate_facet
- uses outbound_handles to retract the facet's assertions
- doesn't directly touch cleanup actions, relying on retract to do that
- if one of a facet's stop actions panics, will drop the facet, leaking its assertions
- actually, even if a stop action yields `Err`, it will drop the facet and leak assertions
- yikes
- facet drop
- panics if outbound_handles is nonempty
- actor cleanup
- relies on facet tree to find assertions to retract
Revised plan:
- ✓ revise Activation/PendingEvents structures
- rename `cleanup_actions` to `outbound_assertions`
- remove `for_myself` queues and `final_actions`
- add `pre_commit_actions`, `rollback_actions` and `commit_actions`
- ✓ assert
- as before
- but on rollback, removes from `outbound_handles` (if the facet still exists) and
`outbound_assertions` (always)
- marks the new assertion as "established" on commit
- ✓ retract
- lookup in `outbound_assertions` by handle, using presence as indication it hasn't been
scheduled in this turn
- on rollback, put it back in `outbound_assertions` ONLY IF IT IS MARKED ESTABLISHED -
otherwise it is a retraction of an `assert` that has *also* been rolled back in this turn
- on commit, remove it from `outbound_handles`
- enqueue the retract action, which just calls e.retract()
- ✓ _terminate_facet
- revised quite a bit now we rely on `RunningActor::cleanup` to use `outbound_assertions`
rather than the facet tree.
- still drops Facets on panic, but this is now mostly harmless (reorders retractions a bit)
- handles `Err` from a stop action more gracefully
- slightly cleverer tracking of what needs doing based on a `TerminationDirection`
- now ONLY applies to ORDERLY cleanup of the facet tree. Disorderly cleanup ignores the
facet tree and just retracts the assertions willy-nilly.
- ✓ facet drop
- warn if outbound_handles is nonempty, but don't do anything about it
- ✓ actor cleanup
- doesn't use the facet tree at all.
- cleanly shutting down is done elsewhere
- uses the remaining entries in `outbound_assertions` (previously `cleanup_actions`) to
deal with retractions for dropped facets as well as any other facets that haven't been
cleanly shut down
- ✓ activate
- now has a panic_guard::PanicGuard RAII for conveying a crash to an actor in case the
activation is happening from a linked task or another thread (this wasn't the case in the
examples that provoked this work, though)
- simplified
- explicit commit/rollback decision
- ✓ Actor::run
- no longer uses the same path for crash-termination and success-termination
- instead, for success-termination, takes a turn that calls Activation::stop_root
- this cleans up the facet tree using _terminate_facet
- when the turn ends, it notices that the root facet is gone and shuts down the actor
- so in principle there will be nothing for actor cleanup to do
2022-02-04 13:52:34 This took days. :-(
2022-02-04 12:59:37 +00:00
|
|
|
})))
|
2021-09-28 11:00:48 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
ds.assert(t, language(), &Observe {
|
|
|
|
pattern: {
|
|
|
|
let recv_label = AnyValue::symbol(recv_label);
|
|
|
|
syndicate_macros::pattern!{<#(recv_label) $ $>}
|
|
|
|
},
|
|
|
|
observer: Arc::clone(&consumer),
|
|
|
|
});
|
|
|
|
|
2022-01-20 09:08:49 +00:00
|
|
|
t.every(Duration::from_secs(1), move |t| {
|
|
|
|
consumer.message(t, &(), &AnyValue::new(true));
|
|
|
|
Ok(())
|
|
|
|
})?;
|
2021-09-28 11:00:48 +00:00
|
|
|
|
|
|
|
if let PingPongMode::Ping(c) = &config.mode {
|
2022-01-20 09:08:49 +00:00
|
|
|
let facet = t.facet.clone();
|
2021-09-28 11:00:48 +00:00
|
|
|
let turn_count = c.turn_count;
|
|
|
|
let action_count = c.action_count;
|
|
|
|
let account = Arc::clone(t.account());
|
2022-01-19 13:40:50 +00:00
|
|
|
t.linked_task(Some(AnyValue::symbol("boot-ping")), async move {
|
2021-09-28 11:00:48 +00:00
|
|
|
let padding = AnyValue::bytestring(vec![0; bytes_padding]);
|
|
|
|
for _ in 0..turn_count {
|
|
|
|
let current_rec = simple_record2(send_label,
|
|
|
|
Value::from(now()).wrap(),
|
|
|
|
padding.clone());
|
2022-01-20 09:08:49 +00:00
|
|
|
facet.activate(&account, None, |t| {
|
|
|
|
for _ in 0..action_count {
|
|
|
|
ds.message(t, &(), ¤t_rec);
|
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
});
|
2021-07-15 07:13:31 +00:00
|
|
|
}
|
2021-09-28 11:00:48 +00:00
|
|
|
Ok(LinkedTaskTermination::KeepFacet)
|
2021-07-24 21:22:01 +00:00
|
|
|
});
|
2021-09-28 11:00:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Ok(None)
|
2023-02-08 21:27:41 +00:00
|
|
|
})
|
2021-07-24 21:22:01 +00:00
|
|
|
}).await??;
|
2021-07-15 07:13:31 +00:00
|
|
|
Ok(())
|
2020-06-10 08:41:12 +00:00
|
|
|
}
|