diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64aee78f..eb9ae24d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ jobs: name: Tests on OTP ${{matrix.otp}} with rebar v${{matrix.rebar3}} strategy: matrix: - otp: ['28', '27'] - rebar3: ['3.25.0'] + otp: ['29', '28', '27'] + rebar3: ['3.27.3'] steps: - uses: actions/checkout@v7 diff --git a/.gitignore b/.gitignore index 19239551..6b70fdb8 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ erl_crash.dump test/results/* !test/results/.keep doc/ +tmp/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..d56c3361 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,8 @@ +Treat documentation and code comments as timeless current-state guidance. +Never use changelog language such as "used to", "previously", "now", "no +longer", or "changed from", and never narrate an edit or compare the current +implementation with an earlier one. Git history is the place for change +history. Describe the current contract and explain its rationale when useful +(for example, "done this way so it doesn't break Z"). + +When committing, add yourself as a Co-author. diff --git a/INTERNALS.md b/INTERNALS.md index 8f271238..5e8565af 100644 --- a/INTERNALS.md +++ b/INTERNALS.md @@ -10,25 +10,65 @@ them to the other nodes. This serializes per node operations and allows keeping per node consistency. -When a node receives registrations of the same name from different nodes, its table shows the most recent one. It keeps -the others aside until their node drops them: if the one it shows goes away, it shows the most recent of them, unless it -went away because of a conflict resolution, since the node that keeps its process then tells everyone. - +### What each node has +Every node keeps a copy of the registrations and group memberships held by each of the other nodes, as that node last +told it. A node only changes its copy of another node's data when that node tells it to: with its full data when the two +nodes find each other, and with each change after that. + +### Which registration a node shows +A name can be registered on two nodes at once, for example while they can't see each other. A node that holds the name +for one of its own processes shows its own. Any other node shows the most recent of the registrations it has, the node +name breaking ties, so that every node shows the same one. It keeps the others aside: if the one it shows goes, it shows +the next one. A node never hides a registration because it expects another node to drop it. + +### Conflicts +When a node holds a name and learns that another node holds it too, it runs the conflict resolution, see +[`syn_event_handler`](syn_event_handler.html). If its own process loses, it drops the name, kills the process if the +default resolution is used, and tells every other node. If its own process wins, it does nothing: the other node drops +its own. Each of the two nodes decides on its own, so the resolution must keep the same process on both. Both nodes +pass it the two registrations in the same order, by node name, so that a resolution that only looks at what it gets +decides the same on both, given the same versions of the two registrations. While an update of one of them is on its +way, the two nodes can decide on different versions; a node that still holds its own registration decides again when +it gets the update. + +### Calls for a process on another node When the operation is called from another node, the result is sent to the scope process of the calling node -in format `{'3.0', sync_register_reply, ...}` (or `sync_unregister_reply`, `sync_join_reply`, `sync_leave_reply`), +in format `{'3.5', sync_register_reply, ...}` (or `sync_unregister_reply`, `sync_join_reply`, `sync_leave_reply`), through the same process that sends it all the other data, so that it gets processed in order with it. The calling scope process applies it like any other sync message and only then replies to the caller, so that its tables are already updated when the call returns. -If the process' node doesn't know the calling node yet, it first sends it its local data, as when it receives a -discover message, so that the calling node knows it by the time the result gets there. +If the operation makes the process' node lose a conflict, for example an update that changes what the resolution +compares, the result goes out before the node drops the name, so that the calling node doesn't show the name again +after it was dropped. + +If the process' node doesn't know the scope process of the calling node, it first sends it its local data and asks for +the calling node's data, so that the two nodes know each other by the time the result gets there. If the calling node +still doesn't know the scope process of the process' node when the result gets there, it doesn't apply it and the call +returns success: the data of that node reaches it with its snapshot. + +If the scope process of the calling node crashes before it applies the result, a result already in its mailbox is lost +with it. A result still on its way goes to whichever scope process runs under the same name when it gets there: if the +restarted one is already running, it handles the result as above and replies, and if none is running, the result is +lost. When the result is lost, the call exits after the usual 5 second timeout of a call, although the process' node +did the operation, and the restarted scope process gets the change with that node's snapshot. If the call is made from a scope process (i.e. from a callback), that process can't wait for one: the process' node replies to it right away, and the scope process of the calling node applies the result in the same way when it gets to it. -Such a call returns the result of the process' node, and never `race_condition`. +Such a call returns the result of the process' node: `race_condition` then only comes from the process' node itself, +for example for an unregister of a name it holds for another process. + +Any other caller waits for the scope process of its node. If that scope process is running a callback that waits for +the caller, directly or through other processes, neither can go on until one of the calls times out after 5 seconds; +the process' node does the operation either way, whether the caller's call then returns or exits. Callbacks therefore +must not wait for processes that call Syn in the same scope and kind (see `syn_event_handler`). -If the process' node runs a version of Syn that does not support this, it replies to the caller directly, and the caller -updates the tables of its node. That node logs the unknown call the first time, and then gets direct calls until it disconnects. +A hidden node takes no part in syncing: it isn't discovered (see Joining below), and scope processes don't sync with +it, since it is usually connected to some of the nodes only and would make them disagree with the others. A call from a +hidden node for a process on another node, or to a hidden node for one of its processes, still works: the process' +node does the operation and the call returns its result, but the calling node doesn't show the change. So an +unregistration or update of that name from the calling node, which looks the name up locally, returns +`{error, undefined}`. ## Scope Subclusters Syn implements Scopes, which are a way to create namespaced, logical overlay networks running on top of the Erlang @@ -49,34 +89,61 @@ the following happens: * `syn_registry_by_pid_users` (of type `bag`). * `syn_pg_by_name_users` (of type `ordered_set`). * `syn_pg_by_pid_users` (of type `ordered_set`). - - These tables are owned by the `syn_backbone` process, so that if the related scope processes were to crash the data is not lost. - In such case, upon respawn the scope process purges the data of all remote nodes, rebuilds the monitors - for the local processes that are still alive and removes from the tables the ones that died meanwhile. - The remote data will then get synchronized again. + + These tables hold what the node shows, and are owned by the `syn_backbone` process, so that if the related scope + processes were to crash the data is not lost. In such case, upon respawn the scope process purges the data of all + remote nodes, rebuilds the monitors for the local processes that are still alive and removes from the tables the + ones that died meanwhile. The remote data will then get synchronized again. + * The registry scope process also keeps, in a table of its own, the registrations of other nodes that it doesn't show. * The 2 newly created scope processes each join a subcluster (one for registry, one for process groups) with the other processes in the Erlang distributed cluster that handle the same Scope (which have the same name). ### Subcluster protocol +#### Messages + * Everything a scope process sends to other nodes goes through its sender process, so that another node gets it in + the order it was sent. A connection too busy to take a message, for example to a node that stopped reading from + it, doesn't hold up the others: the messages for that node wait in order until its connection takes them again, + while the other nodes, and the results of their calls, get theirs. + * Syn's scope processes don't open connections between nodes: a message to a node that isn't connected is dropped, + and that node gets the data again when it connects. There is one narrow exception: if the connection drops between + the moment a scope process checks it and the moment it starts monitoring the other node's scope process, Erlang + connects the two nodes again, as it does whenever any process sends something to a node it isn't connected to. With + the kernel parameter `dist_auto_connect` set to `once`, Erlang doesn't reconnect two nodes on its own once their + connection dropped, and this can't happen. + * A call made through the API for a process on another node connects the nodes, as any call does. With + `dist_auto_connect` set to `once`, a call for a process on a node whose connection dropped exits with `nodedown` + instead, until the nodes get connected explicitly, for example with `net_kernel:connect_node/1`. + * Every message carries the scope process that sent it. A node only applies messages from the scope process it knows + for that node: once it knows the new scope process of a node that restarted, it ignores what the old one sent. A + late message from the old one can still make the node take it for a while; the old one's DOWN then makes the node + ask that node for its data again. + * Messages are tagged with the version of the protocol, `'3.5'`, so that nodes running a version of Syn with another + protocol ignore each other. + #### Joining - * Just after initialization, every scope process tries to send a discovery message in format `{'3.0', discover, self()}` - to every process with the _same name_ running on all the nodes in the Erlang cluster. - * Scope processes monitor node events, so when a new node joins the cluster the same discovery message is sent to the - new node. - * When a scope process receives the discovery message, it: - * Replies with its local data with an ack message in format `{'3.0', ack_sync, self(), LocalData}`. - * Starts monitoring the remote scope node process. - * Adds the remote node to the list of the subcluster nodes. - * The scope process that receives the `ack_sync` message: - * If it's an unknown node, it first: - * Starts monitoring the remote scope node process. - * Adds the remote node to the list of the subcluster nodes. - * Sends it its local data with another ack message, before a conflict resolution can change that data. - * Stores the received data of the remote node. - + * Just after initialization, every scope process sends a discovery message in format `{'3.5', discover, self()}` + to every process with the _same name_ running on all the visible nodes in the Erlang cluster. + * Scope processes monitor node events, so when a new visible node joins the cluster the same discovery message is sent + to the new node. A node that never answers it, for example one that doesn't run the scope, stops being waited for + when its connection drops. + * When a scope process receives the discovery message, it replies with its local data with an ack message in format + `{'3.5', ack_sync, self(), LocalData}`. If it didn't know the scope process that sent it, it starts monitoring it, + adds its node to the list of the subcluster nodes, and asks it for its own data unless it already did. + * When a scope process receives an `ack_sync` message from a scope process it didn't know, it first starts monitoring + it, adds its node to the list of the subcluster nodes, and sends it its local data, before a conflict resolution + can change that data. + * The data in an `ack_sync` replaces everything the scope process had from that node. + * A discovery or `ack_sync` message from a node this one isn't connected to was sent before the connection dropped: + it is ignored, and the two nodes find each other again when they reconnect. + * A scope process from a node whose scope process this one already knows means the other one restarted: the data of + the old one goes. + #### Leaving - * When a scope process of a remote node dies, all the other scope processes are notified because they were monitoring it. + * When a scope process of a remote node dies, or the connection to its node drops, all the other scope processes are + notified because they were monitoring it. * The data related to the remote node that left the subcluster is removed locally on every node. + * If the node is still connected when the notification gets handled, its scope process restarted, or the notification + comes from a connection that dropped and came back: the scope process asks that node for its data again. diff --git a/Makefile b/Makefile index e93486d2..74f0e368 100755 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ PROJECT_DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) +.PHONY: all compile_test clean dialyzer run test proper killzombies bench + all: @rebar3 compile @@ -27,36 +29,29 @@ else -eval 'syn:start().' endif -test: compile_test +test: ifdef suite @# 'make test suite=syn_registry_SUITE' - ct_run -noinput -dir $(PROJECT_DIR)/test -logdir $(PROJECT_DIR)/test/results \ - -suite $(suite) \ - -pa `rebar3 as test path` \ - -erl_args -config $(PROJECT_DIR)/test/sys + rebar3 ct --sname ct --sys_config $(PROJECT_DIR)/test/sys.config --suite $(PROJECT_DIR)/test/$(suite) else - ct_run -noinput -dir $(PROJECT_DIR)/test -logdir $(PROJECT_DIR)/test/results \ - -pa `rebar3 as test path` \ - -erl_args -config $(PROJECT_DIR)/test/sys + rebar3 ct --sname ct --sys_config $(PROJECT_DIR)/test/sys.config endif -proper: compile_test +proper: ifdef suite - ct_run -noinput -dir $(PROJECT_DIR)/test/property -logdir $(PROJECT_DIR)/test/results \ - -suite $(suite) \ - -pa `rebar3 as test path` \ - -erl_args -config $(PROJECT_DIR)/test/sys + rebar3 ct --sname ct --sys_config $(PROJECT_DIR)/test/sys.config --dir $(PROJECT_DIR)/test/property \ + --suite $(PROJECT_DIR)/test/property/$(suite) else - ct_run -noinput -dir $(PROJECT_DIR)/test/property -logdir $(PROJECT_DIR)/test/results \ - -pa `rebar3 as test path` \ - -erl_args -config $(PROJECT_DIR)/test/sys + rebar3 ct --sname ct --sys_config $(PROJECT_DIR)/test/sys.config --dir $(PROJECT_DIR)/test/property endif killzombies: - @pkill -9 -f beam 2>/dev/null; true + @# only the nodes started by the tests, and the test runs themselves + @pkill -9 -f "sname (syn_slave_|syn_protocol_|syn_hidden_|syn_prop_|syn_pg_prop_)" 2>/dev/null; true + @pkill -9 -f "[r]ebar3 ct --sname ct" 2>/dev/null; true @sleep 1 @epmd -daemon - @echo "Zombie beam processes killed, epmd restarted." + @echo "Test nodes killed, epmd running." bench: compile_test @erl -pa `rebar3 as test path` \ diff --git a/OPTIONS.md b/OPTIONS.md index 8b612f99..a296d1d3 100644 --- a/OPTIONS.md +++ b/OPTIONS.md @@ -73,9 +73,9 @@ With `strict_mode` turned on: ```elixir iex> pid = spawn(fn -> receive do _ -> :ok end end) #PID<0.108.0> -iex> :syn.register(:users, "hedy", pid). +iex> :syn.register(:users, "hedy", pid) {:error, :not_self} -iex> :syn.register(:users, "hedy", self()). +iex> :syn.register(:users, "hedy", self()) :ok ``` diff --git a/README.md b/README.md index 9d8e7416..4bd533e3 100755 --- a/README.md +++ b/README.md @@ -105,6 +105,20 @@ Ensure that `syn` is started with your application, for example by adding it in ]}. ``` +### Running different versions together +Nodes sync only with nodes running the same version of Syn's protocol: a node running Syn 3.5 and a node running an +earlier version ignore each other's messages, so a cluster with both splits into one group per version, each with its +own registry and process groups: + +* a name can be registered once in each group, so two processes can hold it at the same time, until the nodes of one + of the versions stop and their registrations go with them; +* a call for a process on a node running the other version fails: on a 3.5 node it raises + `{invalid_remote_scope, Scope, Node}`; +* each node logs a warning for every message it gets from the other version. + +Upgrade all the nodes of a cluster together: stop them all and start them with the new version, or start the new +version as a separate cluster (for example with another cookie) and move the traffic to it. + ## Quickstart ### Registry diff --git a/src/syn.app.src b/src/syn.app.src index a83f96da..1798ec1b 100755 --- a/src/syn.app.src +++ b/src/syn.app.src @@ -4,8 +4,6 @@ {vsn, "3.4.2"}, {registered, [ syn_backbone, - syn_pg, - syn_registry, syn_sup ]}, {applications, [ diff --git a/src/syn.erl b/src/syn.erl index 42d69fe3..acae1b2b 100755 --- a/src/syn.erl +++ b/src/syn.erl @@ -46,7 +46,9 @@ %% %% @end %% =================================================================== @@ -171,7 +173,7 @@ subcluster_nodes(pg, Scope) -> %%

Elixir

%% ``` %% iex> :syn.set_event_handler(MyCustomEventHandler) -%% ok +%% :ok %% ''' %%

Erlang

%% ``` @@ -230,8 +232,9 @@ register(Scope, Name, Pid) -> %%
  • `taken': name is already registered with another `pid()'.
  • %%
  • `not_self': the method is being called from a process other than `self()', %% but `strict_mode' is enabled.
  • -%%
  • `race_condition': the `pid()' runs on another node, and meanwhile the name got registered with another process, -%% which the local node keeps. This is a rare occasion.
  • +%%
  • `race_condition': the `pid()' runs on another node, which registered it, but meanwhile the local node got another +%% process for the name and shows that one instead. Conflict resolution then decides which process keeps the name: the +%% `pid()' can lose it, and the default resolution kills the process that loses. This is a rare occasion.
  • %% %% %%

    Examples

    @@ -246,7 +249,7 @@ register(Scope, Name, Pid) -> %% ``` %% 1> syn:register(devices, "SN-123-456789", self(), [{meta, one}]). %% ok -%% 2> syn:lookup(devices, "SN-123-456789") +%% 2> syn:lookup(devices, "SN-123-456789"). %% {<0.105.0>,[{meta, one}]} %% ''' %% @@ -263,10 +266,10 @@ register(Scope, Name, Pid) -> %%

    Examples

    %%

    Elixir

    %% ``` -%% iex> tuple = {:via, :syn, {:devices, "SN-123-456789"}}. +%% iex> tuple = {:via, :syn, {:devices, "SN-123-456789"}} %% {:via, :syn, {:devices, "SN-123-456789"}} %% iex> GenServer.start_link(__MODULE__, [], name: tuple) -%% {ok, #PID<0.105.0>} +%% {:ok, #PID<0.105.0>} %% iex> GenServer.call(tuple, :your_message) %% :your_message %% ''' @@ -292,8 +295,13 @@ register(Scope, Name, Pid, Meta) -> %% Possible error reasons: %% %% %% Note: an error in the update fun will be raised in the calling process. @@ -492,6 +500,9 @@ is_member(Scope, GroupName, Pid) -> %% Possible error reasons: %% %% %% Note: an error in the update fun will be raised in the calling process. diff --git a/src/syn.hrl b/src/syn.hrl index 10cf88f9..7718fcb3 100755 --- a/src/syn.hrl +++ b/src/syn.hrl @@ -24,16 +24,12 @@ %% THE SOFTWARE. %% ========================================================================================================== %% types --type syn_cluster_api_version() :: { - ApiCall :: atom(), - Version :: atom() -}. -type syn_registry_entry() :: { Name :: term(), Pid :: pid(), Meta :: term(), Time :: integer(), - MRef :: undefined | reference() | caller, + MRef :: undefined | reference(), Node :: node() }. -type syn_registry_entry_by_pid() :: { @@ -41,7 +37,7 @@ Name :: term(), Meta :: term(), Time :: integer(), - MRef :: undefined | reference() | caller, + MRef :: undefined | reference(), Node :: node() }. -type syn_registry_tuple() :: { @@ -57,7 +53,7 @@ }, Meta :: term(), Time :: integer(), - MRef :: undefined | reference() | caller, + MRef :: undefined | reference(), Node :: node() }. -type syn_pg_tuple() :: { @@ -74,7 +70,8 @@ handler_log_name :: atom(), scope = undefined :: atom(), process_name :: atom(), - nodes_map = #{} :: #{node() => pid()}, + nodes_map = #{} :: #{node() => {ScopePid :: pid(), MRef :: reference()}}, + asked = [] :: [node()], sender_pid :: pid(), table_by_name :: atom(), table_by_pid :: atom() diff --git a/src/syn_backbone.erl b/src/syn_backbone.erl index cc3fb2a6..38d413c3 100755 --- a/src/syn_backbone.erl +++ b/src/syn_backbone.erl @@ -32,7 +32,6 @@ -export([create_tables_for_scope/1]). -export([get_table_name/2]). -export([save_process_name/2, get_process_name/1]). --export([save_legacy_node/1, is_legacy_node/1]). -export([is_strict_mode/0]). %% gen_server callbacks @@ -72,20 +71,6 @@ get_process_name(Key) -> [] -> undefined end. -%% a node running a version of Syn that doesn't know the ordered calls, until it disconnects -%% (the table is missing when this code got loaded without restarting syn) --spec save_legacy_node(Node :: node()) -> boolean(). -save_legacy_node(Node) -> - try ets:insert(syn_legacy_nodes, {Node}) - catch error:badarg -> false - end. - --spec is_legacy_node(Node :: node()) -> boolean(). -is_legacy_node(Node) -> - try ets:member(syn_legacy_nodes, Node) - catch error:badarg -> false - end. - -spec get_table_name(TableId :: atom(), Scope :: atom()) -> TableName :: atom() | undefined. get_table_name(TableId, Scope) -> case ets:lookup(syn_table_names, {TableId, Scope}) of @@ -113,9 +98,6 @@ init([]) -> %% create table names table ets:new(syn_table_names, [set, public, named_table, {read_concurrency, true}] ++ ?ETS_OPTIMIZATIONS), ets:new(syn_process_names, [set, public, named_table, {read_concurrency, true}] ++ ?ETS_OPTIMIZATIONS), - ets:new(syn_legacy_nodes, [set, public, named_table, {read_concurrency, true}] ++ ?ETS_OPTIMIZATIONS), - %% monitor nodes, to forget legacy ones when they disconnect - ok = net_kernel:monitor_nodes(true), %% init {ok, #{}}. @@ -159,13 +141,6 @@ handle_cast(Msg, State) -> {noreply, State :: map()} | {noreply, State :: map(), Timeout :: non_neg_integer()} | {stop, Reason :: term(), State :: map()}. -handle_info({nodedown, Node}, State) -> - true = ets:delete(syn_legacy_nodes, Node), - {noreply, State}; - -handle_info({nodeup, _Node}, State) -> - {noreply, State}; - handle_info(Info, State) -> ?LOG_WARNING(#{event => unknown_info, msg => Info}), {noreply, State}. diff --git a/src/syn_event_handler.erl b/src/syn_event_handler.erl index 500ad51e..622423be 100644 --- a/src/syn_event_handler.erl +++ b/src/syn_event_handler.erl @@ -36,6 +36,20 @@ %% %% All of the callbacks, except for `resolve_registry_conflict/4', are called on all the nodes of the cluster. %% This allows you to receive events for the processes running on nodes that get shut down, or in case of net splits. +%% They run in the scope process of the node they are called on, which handles nothing else meanwhile: keep them short. +%% A registry callback runs in the registry scope process of its scope, and a process group callback in the process +%% group one. A call to Syn that needs that same scope process can't complete while the callback runs: +%% +%% To act on an event with Syn, send a message to a process or spawn one, and don't wait for it. %% %% The argument `Reason' in the callbacks can be: %% +%% When a name is registered on more than one node, for example after a net split, a node shows one of the +%% registrations and keeps the others aside. If the one it shows goes, the next one takes its place, and +%% `on_process_registered/5' for it gets the same reason as `on_process_unregistered/5' for the one that went: +%% a crash reason, `{syn_remote_scope_node_down, Scope, Node}', and so on. %% While all callbacks do not have a direct effect on Syn (their return value is ignored), a special case is the callback %% `resolve_registry_conflict/4'. If specified, this is the method that will be used to resolve registry conflicts when detected. %% @@ -75,7 +93,26 @@ %% %% Important Note: the conflict resolution method will be called on the two nodes where the conflicting processes are running on. %% Therefore, this method MUST be defined in the same way across all nodes of the cluster and have the same effect -%% regardless of the node it is run on, or you will experience unexpected results. +%% regardless of the node it is run on, or you will experience unexpected results. Each of the two nodes decides on its +%% own, possibly more than once while the conflict settles. Both pass the two registrations in the same order: first the +%% one of the process running on the node whose name sorts first. A method that decides only from its arguments therefore +%% keeps the same process on both nodes when they decide on the same versions of the two registrations, even if it +%% depends on their order, for example to break a tie. While an update of one of them is on its way, the two nodes can +%% decide on different versions; a node that still holds its own registration decides again when it gets the update. +%% The next paragraph says what that can cause. +%% The method must not depend on anything that differs between nodes, such as `node()', the local clock or random +%% numbers: if it does, both nodes can end up keeping their own process. +%% +%% A process that wins survives if the method always puts the processes in the same order, without three processes +%% each beating another, if what it compares doesn't change during the conflict, and if the process +%% isn't dropped for another reason, for example by losing a conflict over another name it is registered under. Syn's +%% default rule compares registration times, which only fails if a clock moves backwards; comparing a start time stored +%% in the metadata when registering works too. Otherwise, for example after a metadata update with a rule based on the +%% metadata, both nodes can end up dropping their process, and the name is then free on every node. +%% +%% While a conflict settles, a node that runs neither process shows the most recent registration, and switches to the +%% kept one once the other node drops its own. When the method keeps neither process, a node can briefly show one of them +%% before both are dropped, and its callbacks report that. %% %%

    Examples

    %% The following callback module implements the `on_process_unregistered/5' and the `on_process_left/5' callbacks. @@ -219,7 +256,7 @@ ensure_event_handler_loaded() -> %% get handler CustomEventHandler = get_custom_event_handler(), %% ensure that is it loaded (not using code:ensure_loaded/1 to support embedded mode) - catch CustomEventHandler:module_info(exports), + try CustomEventHandler:module_info(exports) catch _:_ -> ok end, ok. %% @private diff --git a/src/syn_gen_scope.erl b/src/syn_gen_scope.erl index c7d6aafc..cb67dd0c 100644 --- a/src/syn_gen_scope.erl +++ b/src/syn_gen_scope.erl @@ -31,15 +31,14 @@ -export([ start_link/3, subcluster_nodes/2, - call/3, call/4, + call/3, call_ordered/4, - reply_ordered/2, - send_to_local_scope/3 + reply_ordered/2 ]). -export([ broadcast/2, broadcast/3, - send_to_node/3, + is_current_scope/2, discover_requester/2, send_ordered_result/5 ]). @@ -65,7 +64,6 @@ %% types -type requester() :: {node(), ScopePid :: pid() | undefined, ReplyWhen :: applied | handled}. --export_type([requester/0]). %% callbacks -callback init(#state{}) -> @@ -83,7 +81,7 @@ {noreply, #state{}, timeout() | hibernate | {continue, term()}} | {stop, Reason :: term(), #state{}}. -callback save_remote_data(RemoteNode :: node(), RemoteData :: term(), #state{}) -> any(). --callback get_local_data(#state{}) -> {ok, Data :: term()} | undefined. +-callback get_local_data(#state{}) -> {ok, Data :: term()}. -callback purge_local_data_for_node(Node :: node(), #state{}) -> any(). %% =================================================================== @@ -105,7 +103,7 @@ start_link(Handler, HandlerLogName, Scope) when is_atom(Scope) -> subcluster_nodes(Handler, Scope) -> case get_process_name_for_scope(Handler, Scope) of undefined -> error({invalid_scope, Scope}); - ProcessName -> gen_server:call(ProcessName, {'3.0', subcluster_nodes}) + ProcessName -> gen_server:call(ProcessName, {'3.5', subcluster_nodes}) end. -spec call(Handler :: module(), Scope :: atom(), Message :: term()) -> Response :: term(). @@ -123,8 +121,8 @@ call(Handler, Node, Scope, Message) -> end end. -%% a call for a process on another node, whose result the local scope process applies in order with everything else -%% that node sends it, see send_ordered_result/5: undefined if that node runs a version that doesn't know the call +%% a call for a process on another node: that node sends the result to the local scope process, after everything +%% it sent it before, and the local scope process applies it and then replies (see send_ordered_result/5) -spec call_ordered(Handler :: module(), Node :: atom(), Scope :: atom(), Message :: tuple()) -> Response :: term(). call_ordered(Handler, Node, Scope, Message) -> case get_process_name_for_scope(Handler, Scope) of @@ -132,41 +130,20 @@ call_ordered(Handler, Node, Scope, Message) -> error({invalid_scope, Scope}); ProcessName -> - case syn_backbone:is_legacy_node(Node) of - true -> - undefined; - - false -> - %% a scope process can't wait for one, itself included: it gets the reply right away, - %% and the local scope process applies the result when it gets to it - ReplyWhen = case is_scope_process() of - true -> handled; - false -> applied - end, - Requester = {node(), whereis(ProcessName), ReplyWhen}, - case call(Handler, Node, Scope, erlang:append_element(Message, Requester)) of - undefined -> - %% that node logs the unknown call, ask it only once - syn_backbone:save_legacy_node(Node), - undefined; - - Response -> - Response - end - end - end. - -%% sends Message to the local scope process, if it's running --spec send_to_local_scope(Handler :: module(), Scope :: atom(), Message :: term()) -> any(). -send_to_local_scope(Handler, Scope, Message) -> - case get_process_name_for_scope(Handler, Scope) of - undefined -> - ok; - - ProcessName -> - case whereis(ProcessName) of - undefined -> ok; - ScopePid -> ScopePid ! Message + %% a scope process can't wait for one, itself included: it gets the reply right away, + %% and the local scope process applies the result when it gets to it + ReplyWhen = case is_scope_process() of + true -> handled; + false -> applied + end, + Requester = {node(), whereis(ProcessName), ReplyWhen}, + case call(Handler, Node, Scope, erlang:append_element(Message, Requester)) of + undefined -> + %% that node runs a version of Syn that this one doesn't sync with + error({invalid_remote_scope, Scope, Node}); + + Response -> + Response end end. @@ -184,29 +161,41 @@ broadcast(Message, State) -> -spec broadcast(Message :: term(), ExcludedNodes :: [node()], #state{}) -> any(). broadcast(Message, ExcludedNodes, #state{sender_pid = SenderPid, process_name = ProcessName, nodes_map = NodesMap}) -> - SenderPid ! {broadcast, Message, ExcludedNodes, ProcessName, NodesMap}. + SenderPid ! {broadcast, Message, ExcludedNodes, ProcessName, maps:keys(NodesMap)}. -spec send_to_node(RemoteNode :: node(), Message :: term(), #state{}) -> any(). send_to_node(RemoteNode, Message, #state{sender_pid = SenderPid, process_name = ProcessName}) -> SenderPid ! {send_single, RemoteNode, Message, ProcessName}. -%% an ordered call from a node this scope process doesn't know yet: that node gets the local data first, as for -%% a discover, so that it knows this node when the result reaches it +%% ScopePid is the scope process this one knows for its node: only what that process sends gets applied +-spec is_current_scope(ScopePid :: pid(), #state{}) -> boolean(). +is_current_scope(ScopePid, #state{nodes_map = NodesMap}) -> + case maps:find(node(ScopePid), NodesMap) of + {ok, {ScopePid, _MRef}} -> true; + _ -> false + end. + +%% an ordered call from a scope process this one doesn't know: that node gets the local data first, so that it knows +%% this node when the result reaches it, and is asked for its own -spec discover_requester(Requester :: requester(), #state{}) -> {RequesterNode :: node(), #state{}}. -discover_requester({RequesterNode, RequesterScopePid, _}, #state{ - handler = Handler, - nodes_map = NodesMap -} = State) -> - case is_pid(RequesterScopePid) andalso not maps:is_key(RequesterNode, NodesMap) of +discover_requester({RequesterNode, RequesterScopePid, _}, State) when is_pid(RequesterScopePid) -> + case is_current_scope(RequesterScopePid, State) orelse not is_connected(RequesterNode) of true -> - {ok, LocalData} = Handler:get_local_data(State), - send_to_node(RequesterNode, {'3.0', ack_sync, self(), LocalData}, State), - _MRef = monitor(process, RequesterScopePid), - {RequesterNode, State#state{nodes_map = NodesMap#{RequesterNode => RequesterScopePid}}}; + %% known, or gone since the call: the result can't reach it anyway, and it syncs again once it's back + {RequesterNode, State}; false -> - {RequesterNode, State} - end. + case adopt(RequesterScopePid, State) of + {ok, State1} -> + send_local_data(RequesterNode, State1), + {RequesterNode, ask(RequesterNode, State1)}; + + disconnected -> + {RequesterNode, State} + end + end; +discover_requester({RequesterNode, undefined, _}, State) -> + {RequesterNode, State}. %% the result of an ordered call goes to the scope process of the requester after everything sent to it before, %% with From appended: that process applies it and then replies, unless the call came from a scope process, @@ -270,7 +259,7 @@ init([Handler, HandlerLogName, Scope, ProcessName]) -> {noreply, #state{}, timeout() | hibernate | {continue, term()}} | {stop, Reason :: term(), Reply :: term(), #state{}} | {stop, Reason :: term(), #state{}}. -handle_call({'3.0', subcluster_nodes}, _From, #state{ +handle_call({'3.5', subcluster_nodes}, _From, #state{ nodes_map = NodesMap } = State) -> Nodes = maps:keys(NodesMap), @@ -297,59 +286,59 @@ handle_cast(Msg, #state{handler_log_name = HandlerLogName, scope = Scope} = Stat {noreply, #state{}} | {noreply, #state{}, timeout() | hibernate | {continue, term()}} | {stop, Reason :: term(), #state{}}. -handle_info({'3.0', discover, RemoteScopePid}, #state{ - handler = Handler, +handle_info({'3.5', discover, RemoteScopePid}, #state{ handler_log_name = HandlerLogName, - scope = Scope, - nodes_map = NodesMap + scope = Scope } = State) -> RemoteScopeNode = node(RemoteScopePid), ?LOG_NOTICE(#{handler => HandlerLogName, scope => Scope, event => discover_request, from => RemoteScopeNode}), - %% send local data to remote (ordered to maintain FIFO with broadcasts) - {ok, LocalData} = Handler:get_local_data(State), - send_to_node(RemoteScopeNode, {'3.0', ack_sync, self(), LocalData}, State), - %% is this a new node? - case maps:is_key(RemoteScopeNode, NodesMap) of - true -> - %% already known, ignore + case {is_connected(RemoteScopeNode), is_current_scope(RemoteScopePid, State)} of + {false, _} -> + %% sent before the connection dropped: that node asks again once it's back {noreply, State}; - false -> - %% monitor - _MRef = monitor(process, RemoteScopePid), - {noreply, State#state{nodes_map = NodesMap#{RemoteScopeNode => RemoteScopePid}}} + {true, true} -> + send_local_data(RemoteScopeNode, State), + {noreply, State}; + + {true, false} -> + %% a scope process this one doesn't know: it gets the local data, and is asked for its own + case adopt(RemoteScopePid, State) of + {ok, State1} -> + send_local_data(RemoteScopeNode, State1), + {noreply, ask(RemoteScopeNode, State1)}; + + disconnected -> + {noreply, State} + end end; -handle_info({'3.0', ack_sync, RemoteScopePid, Data}, #state{ - handler = Handler, +handle_info({'3.5', ack_sync, RemoteScopePid, Data}, #state{ handler_log_name = HandlerLogName, - nodes_map = NodesMap, scope = Scope } = State) -> RemoteScopeNode = node(RemoteScopePid), ?LOG_NOTICE(#{handler => HandlerLogName, scope => Scope, event => ack_sync, from => RemoteScopeNode, entries => length(Data)}), - %% is this a new node? - case maps:is_key(RemoteScopeNode, NodesMap) of - true -> - %% already known, save remote data - Handler:save_remote_data(RemoteScopeNode, Data, State), + case {is_connected(RemoteScopeNode), is_current_scope(RemoteScopePid, State)} of + {false, _} -> + %% sent before the connection dropped: that node sends its data again once it's back {noreply, State}; - false -> - %% monitor - _MRef = monitor(process, RemoteScopePid), - State1 = State#state{nodes_map = NodesMap#{RemoteScopeNode => RemoteScopePid}}, - %% add to subcluster and send local data to remote before saving remote data, - %% so that the snapshot keeps the entries a conflict resolution may remove - %% and the broadcasts of the resolution reach the remote node after it (FIFO) - {ok, LocalData} = Handler:get_local_data(State1), - send_to_node(RemoteScopeNode, {'3.0', ack_sync, self(), LocalData}, State1), - %% save remote data - Handler:save_remote_data(RemoteScopeNode, Data, State1), - %% return - {noreply, State1} + {true, true} -> + save_remote_data(RemoteScopeNode, Data, State); + + {true, false} -> + %% a scope process this one doesn't know: it gets the local data before a conflict resolution can change it + case adopt(RemoteScopePid, State) of + {ok, State1} -> + send_local_data(RemoteScopeNode, State1), + save_remote_data(RemoteScopeNode, Data, State1); + + disconnected -> + {noreply, State} + end end; handle_info({'DOWN', MRef, process, Pid, Reason}, #state{ @@ -358,23 +347,29 @@ handle_info({'DOWN', MRef, process, Pid, Reason}, #state{ scope = Scope, nodes_map = NodesMap } = State) when node(Pid) =/= node() -> - %% scope process down RemoteNode = node(Pid), - case maps:take(RemoteNode, NodesMap) of - {Pid, NodesMap1} -> + case maps:find(RemoteNode, NodesMap) of + {ok, {Pid, MRef}} -> ?LOG_NOTICE(#{handler => HandlerLogName, scope => Scope, event => scope_down, from => RemoteNode, reason => Reason}), - Handler:purge_local_data_for_node(RemoteNode, State), - {noreply, State#state{nodes_map = NodesMap1}}; - - error -> - %% relay to handler - Handler:handle_info({'DOWN', MRef, process, Pid, Reason}, State) + State1 = State#state{nodes_map = maps:remove(RemoteNode, NodesMap)}, + Handler:purge_local_data_for_node(RemoteNode, State1), + %% connected now, so the node didn't go: its scope process restarted, or this DOWN is from a connection + %% that dropped and came back. Whatever scope process runs there now sends its data + case is_connected(RemoteNode) of + true -> {noreply, send_discover(RemoteNode, State1)}; + false -> {noreply, State1} + end; + + _ -> + %% a monitor that has been replaced since + {noreply, State} end; -handle_info({nodedown, _Node}, State) -> - %% ignore & wait for monitor DOWN message - {noreply, State}; +handle_info({nodedown, RemoteNode}, State) -> + %% the node's data is purged on the monitor's DOWN; an answer to a request for it can't come anymore, and the + %% nodeup that follows a reconnection asks again + {noreply, forget_asked(RemoteNode, State)}; handle_info({nodeup, RemoteNode}, #state{ handler_log_name = HandlerLogName, @@ -382,8 +377,7 @@ handle_info({nodeup, RemoteNode}, #state{ } = State) -> ?LOG_NOTICE(#{handler => HandlerLogName, scope => Scope, event => nodeup, from => RemoteNode}), - send_to_node(RemoteNode, {'3.0', discover, self()}, State), - {noreply, State}; + {noreply, send_discover(RemoteNode, State)}; handle_info(Info, #state{handler = Handler} = State) -> Handler:handle_info(Info, State). @@ -397,16 +391,11 @@ handle_info(Info, #state{handler = Handler} = State) -> {stop, Reason :: term(), #state{}}. handle_continue(after_init, #state{ handler_log_name = HandlerLogName, - scope = Scope, - process_name = ProcessName + scope = Scope } = State) -> ?LOG_NOTICE(#{handler => HandlerLogName, scope => Scope, event => discovering_cluster}), - %% broadcasting is done in the scope process to avoid issues with ordering guarantees - lists:foreach(fun(RemoteNode) -> - {ProcessName, RemoteNode} ! {'3.0', discover, self()} - end, nodes()), - {noreply, State}. + {noreply, lists:foldl(fun send_discover/2, State, nodes())}. %% ---------------------------------------------------------------------------------------------------------- %% Terminate @@ -445,19 +434,142 @@ get_process_name_for_scope(Handler, Scope) -> is_scope_process() -> get('$initial_call') =:= {?MODULE, init, 1}. +%% visible nodes only: a hidden node isn't discovered, and syncing with it after one of its calls would make it share +%% data with the nodes it called and no others, so that the nodes disagree. Its calls still get their results +-spec is_connected(Node :: node()) -> boolean(). +is_connected(Node) -> + lists:member(Node, nodes()). + +%% starts treating ScopePid as the scope process of its node, and monitors it: if another one was known for that node, +%% it restarted, so the data that came from it goes, and so does the DOWN still waiting from its monitor. +%% The monitor comes right after checking the connection, before any callback runs, since monitoring a process on a node +%% that just got disconnected makes Erlang connect it again +-spec adopt(ScopePid :: pid(), #state{}) -> {ok, #state{}} | disconnected. +adopt(ScopePid, #state{handler = Handler, nodes_map = NodesMap} = State) -> + Node = node(ScopePid), + case is_connected(Node) of + true -> + MRef = monitor(process, ScopePid), + State1 = State#state{nodes_map = NodesMap#{Node => {ScopePid, MRef}}}, + case maps:find(Node, NodesMap) of + {ok, {OtherScopePid, OldMRef}} when OtherScopePid =/= ScopePid -> + erlang:demonitor(OldMRef, [flush]), + Handler:purge_local_data_for_node(Node, State1); + + _ -> + ok + end, + {ok, State1}; + + false -> + disconnected + end. + +%% applies the full data of a node, which also answers a request for it +-spec save_remote_data(RemoteNode :: node(), Data :: term(), #state{}) -> {noreply, #state{}}. +save_remote_data(RemoteNode, Data, #state{handler = Handler, asked = Asked} = State) -> + State1 = State#state{asked = lists:delete(RemoteNode, Asked)}, + Handler:save_remote_data(RemoteNode, Data, State1), + {noreply, State1}. + +-spec send_local_data(Node :: node(), #state{}) -> any(). +send_local_data(Node, #state{handler = Handler} = State) -> + {ok, LocalData} = Handler:get_local_data(State), + send_to_node(Node, {'3.5', ack_sync, self(), LocalData}, State). + +%% asks Node for its data +-spec send_discover(Node :: node(), #state{}) -> #state{}. +send_discover(Node, #state{asked = Asked} = State) -> + send_to_node(Node, {'3.5', discover, self()}, State), + State#state{asked = [Node | lists:delete(Node, Asked)]}. + +%% stops waiting for the data of Node, whose connection dropped +-spec forget_asked(Node :: node(), #state{}) -> #state{}. +forget_asked(Node, #state{asked = Asked} = State) -> + State#state{asked = lists:delete(Node, Asked)}. + +%% asks Node for its data, unless already waiting for it +-spec ask(Node :: node(), #state{}) -> #state{}. +ask(Node, #state{asked = Asked} = State) -> + case lists:member(Node, Asked) of + true -> State; + false -> send_discover(Node, State) + end. + -spec sender_loop() -> terminated. sender_loop() -> + sender_loop(#{}). + +%% messages to a node go out in the order they came, and only that order matters. A node whose connection is busy, for +%% example because it stopped reading from it, would make a plain send wait: its messages wait in its backlog instead, +%% so that the other nodes, and the results of calls made from them, don't wait for it +-spec sender_loop(Backlogs :: #{node() => queue:queue({atom(), term()})}) -> terminated. +sender_loop(Backlogs) -> + Backlogs1 = send_backlogs(Backlogs), + %% a busy connection gives no notice when it frees up: try again soon + Timeout = case map_size(Backlogs1) of + 0 -> infinity; + _ -> 1 + end, receive - {broadcast, Message, ExcludedNodes, ProcessName, NodesMap} -> - lists:foreach(fun(RemoteNode) -> - {ProcessName, RemoteNode} ! Message - end, maps:keys(NodesMap) -- ExcludedNodes), - sender_loop(); + {broadcast, Message, ExcludedNodes, ProcessName, Nodes} -> + sender_loop(lists:foldl(fun(RemoteNode, Acc) -> + send_in_order(RemoteNode, ProcessName, Message, Acc) + end, Backlogs1, Nodes -- ExcludedNodes)); {send_single, RemoteNode, Message, ProcessName} -> - {ProcessName, RemoteNode} ! Message, - sender_loop(); + sender_loop(send_in_order(RemoteNode, ProcessName, Message, Backlogs1)); terminate -> + %% what still waits for a busy connection goes with this process: that node gets the DOWN and syncs again + _ = send_backlogs(Backlogs1), terminated + after Timeout -> + sender_loop(Backlogs1) + end. + +-spec send_in_order(RemoteNode :: node(), ProcessName :: atom(), Message :: term(), + Backlogs :: #{node() => queue:queue({atom(), term()})}) -> #{node() => queue:queue({atom(), term()})}. +send_in_order(RemoteNode, ProcessName, Message, Backlogs) -> + case maps:find(RemoteNode, Backlogs) of + {ok, Queue} -> + Backlogs#{RemoteNode := queue:in({ProcessName, Message}, Queue)}; + + error -> + case send_noconnect({ProcessName, RemoteNode}, Message) of + nosuspend -> Backlogs#{RemoteNode => queue:from_list([{ProcessName, Message}])}; + _ -> Backlogs + end end. + +%% sends what each backlog can send without waiting +-spec send_backlogs(Backlogs :: #{node() => queue:queue({atom(), term()})}) -> + #{node() => queue:queue({atom(), term()})}. +send_backlogs(Backlogs) -> + maps:fold(fun(RemoteNode, Queue, Acc) -> + case send_queue(RemoteNode, Queue) of + empty -> maps:remove(RemoteNode, Acc); + Queue1 -> Acc#{RemoteNode := Queue1} + end + end, Backlogs, Backlogs). + +-spec send_queue(RemoteNode :: node(), queue:queue({atom(), term()})) -> empty | queue:queue({atom(), term()}). +send_queue(RemoteNode, Queue) -> + case queue:out(Queue) of + {empty, _} -> + empty; + + {{value, {ProcessName, Message} = Item}, Rest} -> + case send_noconnect({ProcessName, RemoteNode}, Message) of + %% back in front of Rest, which keeps the work queue:out did, so a backlog that can't move costs + %% the same to try again however long it is + nosuspend -> queue:in_r(Item, Rest); + _ -> send_queue(RemoteNode, Rest) + end + end. + +%% syn never opens a connection: a message to a node that isn't connected is dropped, and that node gets the data again +%% when it connects. A connection too busy to take the message right away gives nosuspend, and the message isn't sent +-spec send_noconnect(Dest :: {atom(), node()}, Message :: term()) -> ok | noconnect | nosuspend. +send_noconnect(Dest, Message) -> + erlang:send(Dest, Message, [noconnect, nosuspend]). diff --git a/src/syn_pg.erl b/src/syn_pg.erl index e956ca94..bd58c2d3 100644 --- a/src/syn_pg.erl +++ b/src/syn_pg.erl @@ -189,109 +189,38 @@ join_or_update(Scope, GroupName, Pid, MetaOrFun) -> _ -> Node = node(Pid), - case Node =/= node() of - true -> join_or_update_ordered(Scope, Node, GroupName, Pid, MetaOrFun); - false -> join_or_update_direct(Scope, Node, GroupName, Pid, MetaOrFun) + Response = case Node =/= node() of + true -> + %% the scope process of this node applies the result, in order with everything else the process' + %% node sends it, and replies once it has: a lookup on this node right after the call sees it, + %% except for the cases in "Calls for a process on another node" in INTERNALS.md + syn_gen_scope:call_ordered(?MODULE, Node, Scope, + {'3.5', join_or_update_on_node_ordered, GroupName, Pid, MetaOrFun}); + + false -> + syn_gen_scope:call(?MODULE, Scope, {'3.5', join_or_update_on_node, GroupName, Pid, MetaOrFun}) + end, + case Response of + {ok, {Pid, Meta}} -> {ok, {Pid, Meta}}; + {error, Reason} -> {error, Reason}; + {raise, Class, Reason, Stacktrace} -> erlang:raise(Class, Reason, Stacktrace) end end. -%% the scope process of this node applies the result, in order with everything else the process' node sends it, -%% and replies once it has: a lookup on this node right after the call sees it --spec join_or_update_ordered(Scope :: atom(), Node :: node(), GroupName :: term(), Pid :: pid(), MetaOrFun :: term() | function()) -> - {ok, {Pid :: pid(), Meta :: term()}} | {error, Reason :: term()}. -join_or_update_ordered(Scope, Node, GroupName, Pid, MetaOrFun) -> - case syn_gen_scope:call_ordered(?MODULE, Node, Scope, {'3.0', join_or_update_on_node_ordered, GroupName, Pid, MetaOrFun}) of - {ok, {Pid, Meta}} -> - {ok, {Pid, Meta}}; - - {error, Reason} -> - {error, Reason}; - - {raise, Class, Reason, Stacktrace} -> - erlang:raise(Class, Reason, Stacktrace); - - undefined -> - %% the process' node runs a version that doesn't know this call - join_or_update_direct(Scope, Node, GroupName, Pid, MetaOrFun) - end. - -%% the process' node replies to the caller directly: used for local processes, and with nodes that don't know -%% the ordered call --spec join_or_update_direct(Scope :: atom(), Node :: node(), GroupName :: term(), Pid :: pid(), MetaOrFun :: term() | function()) -> - {ok, {Pid :: pid(), Meta :: term()}} | {error, Reason :: term()}. -join_or_update_direct(Scope, Node, GroupName, Pid, MetaOrFun) -> - case syn_gen_scope:call(?MODULE, Node, Scope, {'3.0', join_or_update_on_node, node(), GroupName, Pid, MetaOrFun}) of - {ok, {CallbackMethod, PreviousMeta, Meta, Time, TableByName, TableByPid}} when Node =/= node() -> - %% update table on caller node immediately so that subsequent calls have an updated pg, - %% marked so that a snapshot sent before this change doesn't remove it - add_to_local_table(GroupName, Pid, Meta, Time, caller, TableByName, TableByPid), - %% callback - syn_event_handler:call_event_handler(CallbackMethod, [Scope, GroupName, Pid, Meta, normal]), - syn_event_handler:call_event_handler(CallbackMethod, [Scope, GroupName, Pid, PreviousMeta, Meta, normal]), - %% return - {ok, {Pid, Meta}}; - - {ok, {_, _, Meta, _, _, _}} -> - {ok, {Pid, Meta}}; - - {noop, Meta} -> - {ok, {Pid, Meta}}; - - {error, Reason} -> - {error, Reason}; - - {raise, Class, Reason, Stacktrace} -> - erlang:raise(Class, Reason, Stacktrace) - end. - -spec leave(Scope :: atom(), GroupName :: term(), pid()) -> ok | {error, Reason :: term()}. leave(Scope, GroupName, Pid) -> case syn_backbone:get_table_name(syn_pg_by_name, Scope) of undefined -> error({invalid_scope, Scope}); - TableByName -> + _ -> Node = node(Pid), case Node =/= node() of - true -> leave_ordered(Scope, Node, GroupName, Pid, TableByName); - false -> leave_direct(Scope, Node, GroupName, Pid, TableByName) + true -> syn_gen_scope:call_ordered(?MODULE, Node, Scope, {'3.5', leave_on_node_ordered, GroupName, Pid}); + false -> syn_gen_scope:call(?MODULE, Scope, {'3.5', leave_on_node, GroupName, Pid}) end end. --spec leave_ordered(Scope :: atom(), Node :: node(), GroupName :: term(), Pid :: pid(), TableByName :: atom()) -> - ok | {error, Reason :: term()}. -leave_ordered(Scope, Node, GroupName, Pid, TableByName) -> - case syn_gen_scope:call_ordered(?MODULE, Node, Scope, {'3.0', leave_on_node_ordered, GroupName, Pid}) of - ok -> - ok; - - {error, Reason} -> - {error, Reason}; - - undefined -> - %% the process' node runs a version that doesn't know this call - leave_direct(Scope, Node, GroupName, Pid, TableByName) - end. - --spec leave_direct(Scope :: atom(), Node :: node(), GroupName :: term(), Pid :: pid(), TableByName :: atom()) -> - ok | {error, Reason :: term()}. -leave_direct(Scope, Node, GroupName, Pid, TableByName) -> - case syn_gen_scope:call(?MODULE, Node, Scope, {'3.0', leave_on_node, node(), GroupName, Pid}) of - {ok, {Meta, TableByPid}} when Node =/= node() -> - %% remove table on caller node immediately so that subsequent calls have an updated process group - remove_from_local_table(GroupName, Pid, TableByName, TableByPid), - %% callback - syn_event_handler:call_event_handler(on_process_left, [Scope, GroupName, Pid, Meta, normal]), - %% return - ok; - - {ok, _} -> - ok; - - {error, Reason} -> - {error, Reason} - end. - -spec count(Scope :: atom()) -> non_neg_integer(). count(Scope) -> Set = group_names_ordset(Scope, '_'), @@ -392,50 +321,40 @@ init(#state{ {noreply, #state{}, timeout() | hibernate | {continue, term()}} | {stop, Reason :: term(), Reply :: term(), #state{}} | {stop, Reason :: term(), #state{}}. -handle_call({'3.0', join_or_update_on_node, RequesterNode, GroupName, Pid, MetaOrFun}, _From, #state{ - table_by_name = TableByName, - table_by_pid = TableByPid -} = State) -> - case join_or_update_on_node(RequesterNode, GroupName, Pid, MetaOrFun, State) of - {ok, {CallbackMethod, PreviousMeta, Meta, Time}} -> - {reply, {ok, {CallbackMethod, PreviousMeta, Meta, Time, TableByName, TableByPid}}, State}; - - {noop, Meta, _Time} -> - {reply, {noop, Meta}, State}; - - Error -> - {reply, Error, State} +handle_call({'3.5', join_or_update_on_node, GroupName, Pid, MetaOrFun}, _From, State) -> + case join_or_update_on_node(node(), GroupName, Pid, MetaOrFun, State) of + {ok, {_, _, Meta, _}} -> {reply, {ok, {Pid, Meta}}, State}; + {noop, Meta, _Time} -> {reply, {ok, {Pid, Meta}}, State}; + Error -> {reply, Error, State} end; -handle_call({'3.0', join_or_update_on_node_ordered, GroupName, Pid, MetaOrFun, Requester}, From, State) -> +handle_call({'3.5', join_or_update_on_node_ordered, GroupName, Pid, MetaOrFun, Requester}, From, State) -> {RequesterNode, State1} = syn_gen_scope:discover_requester(Requester, State), case join_or_update_on_node(RequesterNode, GroupName, Pid, MetaOrFun, State1) of {ok, {_, _, Meta, Time}} -> - Message = {'3.0', sync_join_reply, GroupName, Pid, Meta, Time}, + Message = {'3.5', sync_join_reply, self(), GroupName, Pid, Meta, Time}, syn_gen_scope:send_ordered_result(Requester, Message, {ok, {Pid, Meta}}, From, State1); {noop, Meta, Time} -> %% sent anyway, the requester may be missing the entry - Message = {'3.0', sync_join_reply, GroupName, Pid, Meta, Time}, + Message = {'3.5', sync_join_reply, self(), GroupName, Pid, Meta, Time}, syn_gen_scope:send_ordered_result(Requester, Message, {ok, {Pid, Meta}}, From, State1); Error -> {reply, Error, State1} end; -handle_call({'3.0', leave_on_node, RequesterNode, GroupName, Pid}, _From, #state{ - table_by_pid = TableByPid -} = State) -> - case leave_on_node(RequesterNode, GroupName, Pid, State) of - {ok, Meta} -> {reply, {ok, {Meta, TableByPid}}, State}; +handle_call({'3.5', leave_on_node, GroupName, Pid}, _From, State) -> + case leave_on_node(node(), GroupName, Pid, State) of + {ok, _Meta} -> {reply, ok, State}; {error, Reason} -> {reply, {error, Reason}, State} end; -handle_call({'3.0', leave_on_node_ordered, GroupName, Pid, Requester}, From, State) -> +handle_call({'3.5', leave_on_node_ordered, GroupName, Pid, Requester}, From, State) -> {RequesterNode, State1} = syn_gen_scope:discover_requester(Requester, State), case leave_on_node(RequesterNode, GroupName, Pid, State1) of {ok, Meta} -> - Message = {'3.0', sync_leave_reply, GroupName, Pid, Meta}, + Message = {'3.5', sync_leave_reply, self(), GroupName, Pid, Meta}, syn_gen_scope:send_ordered_result(Requester, Message, ok, From, State1); {error, Reason} -> @@ -454,35 +373,33 @@ handle_call(Request, From, #state{scope = Scope} = State) -> {noreply, #state{}} | {noreply, #state{}, timeout() | hibernate | {continue, term()}} | {stop, Reason :: term(), #state{}}. -handle_info({'3.0', sync_join, GroupName, Pid, Meta, Time, Reason}, #state{nodes_map = NodesMap} = State) -> - case maps:is_key(node(Pid), NodesMap) of - true -> - handle_pg_sync(GroupName, Pid, Meta, Time, Reason, State); - - false -> - %% ignore, node not yet discovered (ack_sync not yet received) - ok +handle_info({'3.5', sync_join, ScopePid, GroupName, Pid, Meta, Time, Reason}, State) -> + case is_from_owner(ScopePid, Pid, State) of + true -> handle_pg_sync(GroupName, Pid, Meta, Time, Reason, State); + false -> ok end, {noreply, State}; -handle_info({'3.0', sync_join_reply, GroupName, Pid, Meta, Time, From}, #state{nodes_map = NodesMap} = State) -> - case maps:is_key(node(Pid), NodesMap) of - true -> - handle_pg_sync(GroupName, Pid, Meta, Time, normal, State); - - false -> - %% node not yet discovered, its ack_sync brings the entry - ok +handle_info({'3.5', sync_join_reply, ScopePid, GroupName, Pid, Meta, Time, From}, State) -> + case is_from_owner(ScopePid, Pid, State) of + true -> handle_pg_sync(GroupName, Pid, Meta, Time, normal, State); + false -> ok end, syn_gen_scope:reply_ordered(From, {ok, {Pid, Meta}}), {noreply, State}; -handle_info({'3.0', sync_leave, GroupName, Pid, Meta, Reason}, State) -> - handle_pg_leave_sync(GroupName, Pid, Meta, Reason, State), +handle_info({'3.5', sync_leave, ScopePid, GroupName, Pid, Meta, Reason}, State) -> + case is_from_owner(ScopePid, Pid, State) of + true -> handle_pg_leave_sync(GroupName, Pid, Meta, Reason, State); + false -> ok + end, {noreply, State}; -handle_info({'3.0', sync_leave_reply, GroupName, Pid, Meta, From}, State) -> - handle_pg_leave_sync(GroupName, Pid, Meta, normal, State), +handle_info({'3.5', sync_leave_reply, ScopePid, GroupName, Pid, Meta, From}, State) -> + case is_from_owner(ScopePid, Pid, State) of + true -> handle_pg_leave_sync(GroupName, Pid, Meta, normal, State); + false -> ok + end, syn_gen_scope:reply_ordered(From, ok), {noreply, State}; @@ -503,7 +420,7 @@ handle_info({'DOWN', _MRef, process, Pid, Reason}, #state{ %% callback syn_event_handler:call_event_handler(on_process_left, [Scope, GroupName, Pid, Meta, Reason]), %% broadcast - syn_gen_scope:broadcast({'3.0', sync_leave, GroupName, Pid, Meta, Reason}, State) + syn_gen_scope:broadcast({'3.5', sync_leave, self(), GroupName, Pid, Meta, Reason}, State) end, Entries) end, %% return @@ -517,7 +434,7 @@ handle_info(Info, #state{scope = Scope} = State) -> %% ---------------------------------------------------------------------------------------------------------- %% Data callbacks %% ---------------------------------------------------------------------------------------------------------- --spec get_local_data(State :: term()) -> {ok, Data :: term()} | undefined. +-spec get_local_data(State :: term()) -> {ok, Data :: term()}. get_local_data(#state{table_by_name = TableByName}) -> {ok, get_pg_tuples_for_node(node(), TableByName)}. @@ -654,7 +571,7 @@ leave_on_node(RequesterNode, GroupName, Pid, #state{ %% callback syn_event_handler:call_event_handler(on_process_left, [Scope, GroupName, Pid, Meta, normal]), %% broadcast - syn_gen_scope:broadcast({'3.0', sync_leave, GroupName, Pid, Meta, normal}, [RequesterNode], State), + syn_gen_scope:broadcast({'3.5', sync_leave, self(), GroupName, Pid, Meta, normal}, [RequesterNode], State), %% return {ok, Meta} end. @@ -683,7 +600,7 @@ do_join_on_node(GroupName, Pid, PreviousMeta, Meta, MRef, Reason, RequesterNode, syn_event_handler:call_event_handler(CallbackMethod, [Scope, GroupName, Pid, Meta, Reason]), syn_event_handler:call_event_handler(CallbackMethod, [Scope, GroupName, Pid, PreviousMeta, Meta, normal]), %% broadcast - syn_gen_scope:broadcast({'3.0', sync_join, GroupName, Pid, Meta, Time, Reason}, [RequesterNode], State), + syn_gen_scope:broadcast({'3.5', sync_join, self(), GroupName, Pid, Meta, Time, Reason}, [RequesterNode], State), %% return {ok, {CallbackMethod, PreviousMeta, Meta, Time}}. @@ -748,7 +665,7 @@ maybe_demonitor(Pid, TableByPid) -> Pid :: pid(), Meta :: term(), Time :: integer(), - MRef :: undefined | reference() | caller, + MRef :: undefined | reference(), TableByName :: atom(), TableByPid :: atom() ) -> true. @@ -792,6 +709,7 @@ purge_pg_for_remote_node(Scope, Node, TableByName, TableByPid) when Node =/= nod ets:match_delete(TableByName, {{'_', '_'}, '_', '_', '_', Node}), ets:match_delete(TableByPid, {{'_', '_'}, '_', '_', '_', Node}). +%% the full data of a node replaces what this node had from it: the entries it doesn't have anymore go -spec reconcile_remote_pg_snapshot(Node :: node(), [syn_pg_tuple()], #state{}) -> ok. reconcile_remote_pg_snapshot(Node, PgTuplesOfRemoteNode, #state{ scope = Scope, @@ -799,9 +717,9 @@ reconcile_remote_pg_snapshot(Node, PgTuplesOfRemoteNode, #state{ table_by_pid = TableByPid }) -> ExistingEntries = ets:select(TableByName, [{ - {{'$1', '$2'}, '$3', '$4', '$5', Node}, + {{'$1', '$2'}, '$3', '_', '_', Node}, [], - [{{'$1', '$2', '$3', '$4', '$5'}}] + [{{'$1', '$2', '$3'}}] }]), case ExistingEntries of [] -> @@ -810,9 +728,8 @@ reconcile_remote_pg_snapshot(Node, PgTuplesOfRemoteNode, #state{ _ -> %% gb_sets compares with == like the ordered_set tables do, so 1 and 1.0 are the same group here too SnapshotKeys = gb_sets:from_list([{GroupName, Pid} || {GroupName, Pid, _Meta, _Time} <- PgTuplesOfRemoteNode]), - SnapshotTime = lists:foldl(fun({_, _, _, Time}, Acc) -> max(Time, Acc) end, 0, PgTuplesOfRemoteNode), - lists:foreach(fun({GroupName, Pid, Meta, Time, MRef}) -> - case gb_sets:is_element({GroupName, Pid}, SnapshotKeys) orelse is_newer_caller_entry(MRef, Time, SnapshotTime) of + lists:foreach(fun({GroupName, Pid, Meta}) -> + case gb_sets:is_element({GroupName, Pid}, SnapshotKeys) of true -> ok; @@ -825,12 +742,6 @@ reconcile_remote_pg_snapshot(Node, PgTuplesOfRemoteNode, #state{ end, ExistingEntries) end. -%% an entry this node wrote itself from the reply to its own call, newer than anything in the snapshot: -%% the snapshot may have been sent before the call was handled, so it can't know about it yet --spec is_newer_caller_entry(MRef :: undefined | reference() | caller, Time :: integer(), SnapshotTime :: integer()) -> boolean(). -is_newer_caller_entry(caller, Time, SnapshotTime) -> Time > SnapshotTime; -is_newer_caller_entry(_, _, _) -> false. - -spec handle_pg_sync( GroupName :: term(), Pid :: pid(), @@ -851,10 +762,6 @@ handle_pg_sync(GroupName, Pid, Meta, Time, Reason, #state{ %% callback syn_event_handler:call_event_handler(on_process_joined, [Scope, GroupName, Pid, Meta, Reason]); - {{_, _}, _, TableTime, caller, _} when Time < TableTime -> - %% older than what this node wrote from the reply to its own call: ignore - ok; - {{_, _}, TableMeta, _, MRef, _} -> %% more recent (because it comes from the same node, which means that it's sequential) %% maybe updated meta or time only @@ -892,6 +799,11 @@ handle_pg_leave_sync(GroupName, Pid, Meta, Reason, #state{ syn_event_handler:call_event_handler(on_process_left, [Scope, GroupName, Pid, Meta, Reason]) end. +%% a message about Pid comes from the scope process this node knows for Pid's node +-spec is_from_owner(ScopePid :: pid(), Pid :: pid(), #state{}) -> boolean(). +is_from_owner(ScopePid, Pid, State) -> + node(ScopePid) =:= node(Pid) andalso syn_gen_scope:is_current_scope(ScopePid, State). + -spec multi_call_and_receive( CollectorPid :: pid(), Pid :: pid(), diff --git a/src/syn_registry.erl b/src/syn_registry.erl index f8b11d67..cd923a3f 100755 --- a/src/syn_registry.erl +++ b/src/syn_registry.erl @@ -59,6 +59,14 @@ -include_lib("kernel/include/logger.hrl"). +%% types +%% this node's registration of a name, with the monitor on its process +-type own_claim() :: {Pid :: pid(), Meta :: term(), Time :: integer(), MRef :: reference()}. +%% another node's registration of a name, as that node holds it +-type claim() :: {Pid :: pid(), Meta :: term(), Time :: integer()}. +%% a registration of another node that this node doesn't show +-type aside_entry() :: {Name :: term(), Node :: node(), Pid :: pid(), Meta :: term(), Time :: integer()}. + %% =================================================================== %% API %% =================================================================== @@ -131,61 +139,24 @@ register_or_update(Scope, Name, Pid, MetaOrFun) -> _ -> Node = node(Pid), - case Node =/= node() of - true -> register_or_update_ordered(Scope, Node, Name, Pid, MetaOrFun); - false -> register_or_update_direct(Scope, Node, Name, Pid, MetaOrFun) + Response = case Node =/= node() of + true -> + %% the scope process of this node applies the result, in order with everything else the process' + %% node sends it, and replies once it has: a lookup on this node right after the call sees it, + %% except for the cases in "Calls for a process on another node" in INTERNALS.md + syn_gen_scope:call_ordered(?MODULE, Node, Scope, + {'3.5', register_or_update_on_node_ordered, Name, Pid, MetaOrFun}); + + false -> + syn_gen_scope:call(?MODULE, Scope, {'3.5', register_or_update_on_node, Name, Pid, MetaOrFun}) + end, + case Response of + {ok, {Pid, Meta}} -> {ok, {Pid, Meta}}; + {error, Reason} -> {error, Reason}; + {raise, Class, Reason, Stacktrace} -> erlang:raise(Class, Reason, Stacktrace) end end. -%% the scope process of this node applies the result, in order with everything else the process' node sends it, -%% and replies once it has: a lookup on this node right after the call sees it --spec register_or_update_ordered(Scope :: atom(), Node :: node(), Name :: term(), Pid :: pid(), MetaOrFun :: term() | function()) -> - {ok, {Pid :: pid(), Meta :: term()}} | {error, Reason :: term()}. -register_or_update_ordered(Scope, Node, Name, Pid, MetaOrFun) -> - case syn_gen_scope:call_ordered(?MODULE, Node, Scope, {'3.0', register_or_update_on_node_ordered, Name, Pid, MetaOrFun}) of - {ok, {Pid, Meta}} -> - {ok, {Pid, Meta}}; - - {error, Reason} -> - {error, Reason}; - - {raise, Class, Reason, Stacktrace} -> - erlang:raise(Class, Reason, Stacktrace); - - undefined -> - %% the process' node runs a version that doesn't know this call - register_or_update_direct(Scope, Node, Name, Pid, MetaOrFun) - end. - -%% the process' node replies to the caller directly: used for local processes, and with nodes that don't know -%% the ordered call --spec register_or_update_direct(Scope :: atom(), Node :: node(), Name :: term(), Pid :: pid(), MetaOrFun :: term() | function()) -> - {ok, {Pid :: pid(), Meta :: term()}} | {error, Reason :: term()}. -register_or_update_direct(Scope, Node, Name, Pid, MetaOrFun) -> - case syn_gen_scope:call(?MODULE, Node, Scope, {'3.0', register_or_update_on_node, node(), Name, Pid, MetaOrFun}) of - {ok, {CallbackMethod, PreviousMeta, Meta, Time, TableByName, TableByPid}} when Node =/= node() -> - %% update table on caller node immediately so that subsequent calls have an updated registry, - %% marked so that a snapshot sent before this change doesn't remove it - add_to_local_table(Name, Pid, Meta, Time, caller, TableByName, TableByPid), - %% callback - syn_event_handler:call_event_handler(CallbackMethod, [Scope, Name, Pid, Meta, normal]), - syn_event_handler:call_event_handler(CallbackMethod, [Scope, Name, Pid, PreviousMeta, Meta, normal]), - %% return - {ok, {Pid, Meta}}; - - {ok, {_, _, Meta, _, _, _}} -> - {ok, {Pid, Meta}}; - - {noop, Meta} -> - {ok, {Pid, Meta}}; - - {error, Reason} -> - {error, Reason}; - - {raise, Class, Reason, Stacktrace} -> - erlang:raise(Class, Reason, Stacktrace) - end. - -spec unregister(Scope :: atom(), Name :: term()) -> ok | {error, Reason :: term()}. unregister(Scope, Name) -> case syn_backbone:get_table_name(syn_registry_by_name, Scope) of @@ -198,51 +169,15 @@ unregister(Scope, Name) -> undefined -> {error, undefined}; - {Name, Pid, Meta, _, _, _} -> + {Name, Pid, _, _, _, _} -> Node = node(Pid), case Node =/= node() of - true -> unregister_ordered(Scope, Node, Name, Pid, Meta, TableByName); - false -> unregister_direct(Scope, Node, Name, Pid, Meta, TableByName) + true -> syn_gen_scope:call_ordered(?MODULE, Node, Scope, {'3.5', unregister_on_node_ordered, Name, Pid}); + false -> syn_gen_scope:call(?MODULE, Scope, {'3.5', unregister_on_node, Name, Pid}) end end end. --spec unregister_ordered(Scope :: atom(), Node :: node(), Name :: term(), Pid :: pid(), Meta :: term(), TableByName :: atom()) -> - ok | {error, Reason :: term()}. -unregister_ordered(Scope, Node, Name, Pid, Meta, TableByName) -> - case syn_gen_scope:call_ordered(?MODULE, Node, Scope, {'3.0', unregister_on_node_ordered, Name, Pid}) of - ok -> - ok; - - {error, Reason} -> - {error, Reason}; - - undefined -> - %% the process' node runs a version that doesn't know this call - unregister_direct(Scope, Node, Name, Pid, Meta, TableByName) - end. - --spec unregister_direct(Scope :: atom(), Node :: node(), Name :: term(), Pid :: pid(), Meta :: term(), TableByName :: atom()) -> - ok | {error, Reason :: term()}. -unregister_direct(Scope, Node, Name, Pid, Meta, TableByName) -> - case syn_gen_scope:call(?MODULE, Node, Scope, {'3.0', unregister_on_node, node(), Name, Pid}) of - {ok, TableByPid} when Node =/= node() -> - %% remove table on caller node immediately so that subsequent calls have an updated registry - remove_from_local_table(Name, Pid, TableByName, TableByPid), - %% callback - syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, Pid, Meta, normal]), - %% the process' node doesn't send it to our scope process, which may keep it aside or show it by now - syn_gen_scope:send_to_local_scope(?MODULE, Scope, {'3.0', sync_unregister, Name, Pid, Meta, normal}), - %% return - ok; - - {ok, _} -> - ok; - - {error, Reason} -> - {error, Reason} - end. - -spec count(Scope :: atom()) -> non_neg_integer(). count(Scope) -> case syn_backbone:get_table_name(syn_registry_by_name, Scope) of @@ -283,8 +218,8 @@ init(#state{ %% purge remote & rebuild purge_registry_for_remote_nodes(Scope, TableByName, TableByPid), rebuild_monitors(Scope, TableByName, TableByPid), - %% init, with the registrations of other nodes kept aside (see shadow/6) - HandlerState = #{shadowed => ets:new(syn_registry_shadowed, [bag])}, + %% init, with a table for the registrations of other nodes that this node doesn't show (see store/4) + HandlerState = #{aside => ets:new(syn_registry_aside, [bag])}, {ok, HandlerState}. %% ---------------------------------------------------------------------------------------------------------- @@ -297,50 +232,51 @@ init(#state{ {noreply, #state{}, timeout() | hibernate | {continue, term()}} | {stop, Reason :: term(), Reply :: term(), #state{}} | {stop, Reason :: term(), #state{}}. -handle_call({'3.0', register_or_update_on_node, RequesterNode, Name, Pid, MetaOrFun}, _From, #state{ - table_by_name = TableByName, - table_by_pid = TableByPid -} = State) -> - case register_or_update_on_node(RequesterNode, Name, Pid, MetaOrFun, State) of - {ok, {CallbackMethod, PreviousMeta, Meta, Time}} -> - {reply, {ok, {CallbackMethod, PreviousMeta, Meta, Time, TableByName, TableByPid}}, State}; +handle_call({'3.5', register_or_update_on_node, Name, Pid, MetaOrFun}, _From, State) -> + case register_or_update_on_node(node(), Name, Pid, MetaOrFun, State) of + {ok, Meta, _Time} -> + %% an update can make this node lose a conflict it was keeping + resolve(Name, all, State), + {reply, {ok, {Pid, Meta}}, State}; {noop, Meta, _Time} -> - {reply, {noop, Meta}, State}; + {reply, {ok, {Pid, Meta}}, State}; Error -> {reply, Error, State} end; -handle_call({'3.0', register_or_update_on_node_ordered, Name, Pid, MetaOrFun, Requester}, From, State) -> +handle_call({'3.5', register_or_update_on_node_ordered, Name, Pid, MetaOrFun, Requester}, From, State) -> {RequesterNode, State1} = syn_gen_scope:discover_requester(Requester, State), case register_or_update_on_node(RequesterNode, Name, Pid, MetaOrFun, State1) of - {ok, {_, _, Meta, Time}} -> - Message = {'3.0', sync_register_reply, Name, Pid, Meta, Time}, - syn_gen_scope:send_ordered_result(Requester, Message, {ok, {Pid, Meta}}, From, State1); + {ok, Meta, Time} -> + Message = {'3.5', sync_register_reply, self(), Name, Pid, Meta, Time}, + Result = syn_gen_scope:send_ordered_result(Requester, Message, {ok, {Pid, Meta}}, From, State1), + %% an update can make this node lose a conflict it was keeping: the withdrawal goes after the result, + %% so that the caller's node doesn't put the registration back after removing it + resolve(Name, all, State1), + Result; {noop, Meta, Time} -> %% sent anyway, the requester may be missing the entry - Message = {'3.0', sync_register_reply, Name, Pid, Meta, Time}, + Message = {'3.5', sync_register_reply, self(), Name, Pid, Meta, Time}, syn_gen_scope:send_ordered_result(Requester, Message, {ok, {Pid, Meta}}, From, State1); Error -> {reply, Error, State1} end; -handle_call({'3.0', unregister_on_node, RequesterNode, Name, Pid}, _From, #state{ - table_by_pid = TableByPid -} = State) -> - case unregister_on_node(RequesterNode, Name, Pid, State) of - {ok, _Meta} -> {reply, {ok, TableByPid}, State}; +handle_call({'3.5', unregister_on_node, Name, Pid}, _From, State) -> + case unregister_on_node(node(), Name, Pid, State) of + {ok, _Meta} -> {reply, ok, State}; {error, Reason} -> {reply, {error, Reason}, State} end; -handle_call({'3.0', unregister_on_node_ordered, Name, Pid, Requester}, From, State) -> +handle_call({'3.5', unregister_on_node_ordered, Name, Pid, Requester}, From, State) -> {RequesterNode, State1} = syn_gen_scope:discover_requester(Requester, State), case unregister_on_node(RequesterNode, Name, Pid, State1) of {ok, Meta} -> - Message = {'3.0', sync_unregister_reply, Name, Pid, Meta}, + Message = {'3.5', sync_unregister_reply, self(), Name, Pid, Meta}, syn_gen_scope:send_ordered_result(Requester, Message, ok, From, State1); {error, Reason} -> @@ -359,53 +295,53 @@ handle_call(Request, From, #state{scope = Scope} = State) -> {noreply, #state{}} | {noreply, #state{}, timeout() | hibernate | {continue, term()}} | {stop, Reason :: term(), #state{}}. -handle_info({'3.0', sync_register, Name, Pid, Meta, Time, Reason}, #state{nodes_map = NodesMap} = State) -> - case maps:is_key(node(Pid), NodesMap) of - true -> - handle_registry_sync(Name, Pid, Meta, Time, Reason, State); - - false -> - %% ignore, node not yet discovered (ack_sync not yet received) - ok +handle_info({'3.5', sync_register, ScopePid, Name, Pid, Meta, Time, Reason}, State) -> + case is_from_owner(ScopePid, Pid, State) of + true -> put_claim(Name, Pid, Meta, Time, Reason, State); + false -> ok end, {noreply, State}; -handle_info({'3.0', sync_register_reply, Name, Pid, Meta, Time, From}, #state{ - table_by_name = TableByName, - nodes_map = NodesMap +handle_info({'3.5', sync_register_reply, ScopePid, Name, Pid, Meta, Time, From}, #state{ + table_by_name = TableByName } = State) -> - Reply = case maps:is_key(node(Pid), NodesMap) of + Reply = case is_from_owner(ScopePid, Pid, State) of true -> - handle_registry_sync(Name, Pid, Meta, Time, normal, State), + put_claim(Name, Pid, Meta, Time, normal, State), case find_registry_entry_by_name(Name, TableByName) of {Name, Pid, _, _, _, _} -> {ok, {Pid, Meta}}; _ -> - %% this node keeps the name on another process: it resolved a conflict against this one, - %% or has a more recent registration of it + %% this node shows another registration of the name {error, race_condition} end; false -> - %% node not yet discovered, its ack_sync brings the entry + %% a scope process this node doesn't know (anymore): the entry comes with its full data, unless this node + %% doesn't sync with that one, as with a hidden node {ok, {Pid, Meta}} end, syn_gen_scope:reply_ordered(From, Reply), {noreply, State}; -handle_info({'3.0', sync_unregister, Name, Pid, Meta, Reason}, State) -> - handle_registry_unregister_sync(Name, Pid, Meta, Reason, State), +handle_info({'3.5', sync_unregister, ScopePid, Name, Pid, _Meta, Reason}, State) -> + case is_from_owner(ScopePid, Pid, State) of + true -> drop_claim(Name, Pid, Reason, State); + false -> ok + end, {noreply, State}; -handle_info({'3.0', sync_unregister_reply, Name, Pid, Meta, From}, State) -> - handle_registry_unregister_sync(Name, Pid, Meta, normal, State), +handle_info({'3.5', sync_unregister_reply, ScopePid, Name, Pid, _Meta, From}, State) -> + case is_from_owner(ScopePid, Pid, State) of + true -> drop_claim(Name, Pid, normal, State); + false -> ok + end, syn_gen_scope:reply_ordered(From, ok), {noreply, State}; handle_info({'DOWN', _MRef, process, Pid, Reason}, #state{ scope = Scope, - table_by_name = TableByName, table_by_pid = TableByPid } = State) -> case find_registry_entries_by_pid(Pid, TableByPid) of @@ -415,13 +351,7 @@ handle_info({'DOWN', _MRef, process, Pid, Reason}, #state{ Entries -> lists:foreach(fun({_, Name, Meta, _, _, _}) -> - %% remove from table - remove_from_local_table(Name, Pid, TableByName, TableByPid), - %% callback - syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, Pid, Meta, Reason]), - %% broadcast - syn_gen_scope:broadcast({'3.0', sync_unregister, Name, Pid, Meta, Reason}, State), - show_shadowed(Name, State) + withdraw(Name, Pid, Meta, Reason, [], State) end, Entries) end, %% return @@ -435,27 +365,34 @@ handle_info(Info, #state{scope = Scope} = State) -> %% ---------------------------------------------------------------------------------------------------------- %% Data callbacks %% ---------------------------------------------------------------------------------------------------------- --spec get_local_data(#state{}) -> {ok, Data :: term()} | undefined. +-spec get_local_data(#state{}) -> {ok, Data :: term()}. get_local_data(#state{table_by_name = TableByName}) -> + %% a node always shows its own registrations {ok, get_registry_tuples_for_node(node(), TableByName)}. +%% the full data of a node replaces what this node had from it -spec save_remote_data(RemoteNode :: node(), RemoteData :: term(), #state{}) -> any(). save_remote_data(RemoteNode, RegistryTuplesOfRemoteNode, #state{scope = Scope} = State) -> - reconcile_remote_registry_snapshot(RemoteNode, RegistryTuplesOfRemoteNode, State), - %% insert/update tuples + Reason = {syn_remote_scope_node_up, Scope, RemoteNode}, + SnapshotNames = maps:from_list([{Name, true} || {Name, _, _, _} <- RegistryTuplesOfRemoteNode]), + %% what the node doesn't hold anymore + lists:foreach(fun({Name, Pid}) -> + case maps:is_key(Name, SnapshotNames) of + true -> ok; + false -> drop_claim(Name, Pid, Reason, State) + end + end, claimed_by_node(RemoteNode, State)), + %% what it holds now lists:foreach(fun({Name, Pid, Meta, Time}) -> - handle_registry_sync(Name, Pid, Meta, Time, {syn_remote_scope_node_up, Scope, RemoteNode}, State) + put_claim(Name, Pid, Meta, Time, Reason, State) end, RegistryTuplesOfRemoteNode). -spec purge_local_data_for_node(Node :: node(), #state{}) -> any(). -purge_local_data_for_node(Node, #state{ - scope = Scope, - table_by_name = TableByName, - table_by_pid = TableByPid -} = State) -> - RegistryTuples = purge_registry_for_remote_node(Scope, Node, TableByName, TableByPid), - unshadow_node(Node, State), - lists:foreach(fun({Name, _, _, _}) -> show_shadowed(Name, State) end, RegistryTuples). +purge_local_data_for_node(Node, #state{scope = Scope} = State) -> + Reason = {syn_remote_scope_node_down, Scope, Node}, + lists:foreach(fun({Name, Pid}) -> + drop_claim(Name, Pid, Reason, State) + end, claimed_by_node(Node, State)). %% =================================================================== %% Internal @@ -503,7 +440,7 @@ do_rebuild_monitors([{Name, Pid, Meta, Time} | T], NewMRefs, Scope, TableByName, MetaOrFun :: term() | function(), #state{} ) -> - {ok, {CallbackMethod :: atom(), PreviousMeta :: term(), Meta :: term(), Time :: integer()}} | + {ok, Meta :: term(), Time :: integer()} | {noop, Meta :: term(), Time :: integer()} | {error, Reason :: term()} | {raise, Class :: atom(), Reason :: term(), Stacktrace :: list()}. @@ -523,7 +460,7 @@ register_or_update_on_node(RequesterNode, Name, Pid, MetaOrFun, #state{ undefined -> erlang:monitor(process, Pid); %% process is not monitored yet, add MRef0 -> MRef0 end, - do_register_on_node(Name, Pid, undefined, MetaOrFun, MRef, normal, RequesterNode, on_process_registered, State); + do_register_on_node(Name, Pid, MetaOrFun, MRef, RequesterNode, State); {Name, Pid, TableMeta, TableTime, MRef, _} when is_function(MetaOrFun) -> %% update with fun @@ -532,7 +469,7 @@ register_or_update_on_node(RequesterNode, Name, Pid, MetaOrFun, #state{ {noop, TableMeta, TableTime}; Meta -> - do_register_on_node(Name, Pid, TableMeta, Meta, MRef, normal, RequesterNode, on_registry_process_updated, State) + do_register_on_node(Name, Pid, Meta, MRef, RequesterNode, State) catch Class:Reason:Stacktrace -> ?LOG_ERROR(#{event => callback_error, @@ -546,9 +483,9 @@ register_or_update_on_node(RequesterNode, Name, Pid, MetaOrFun, #state{ %% same pid, same meta {noop, MetaOrFun, TableTime}; - {Name, Pid, TableMeta, _, MRef, _} -> + {Name, Pid, _, _, MRef, _} -> %% same pid, different meta - do_register_on_node(Name, Pid, TableMeta, MetaOrFun, MRef, normal, RequesterNode, on_registry_process_updated, State); + do_register_on_node(Name, Pid, MetaOrFun, MRef, RequesterNode, State); _ -> {error, taken} @@ -561,7 +498,6 @@ register_or_update_on_node(RequesterNode, Name, Pid, MetaOrFun, #state{ -spec unregister_on_node(RequesterNode :: node(), Name :: term(), Pid :: pid(), #state{}) -> {ok, Meta :: term()} | {error, Reason :: term()}. unregister_on_node(RequesterNode, Name, Pid, #state{ - scope = Scope, table_by_name = TableByName, table_by_pid = TableByPid } = State) -> @@ -572,14 +508,7 @@ unregister_on_node(RequesterNode, Name, Pid, #state{ {Name, Pid, Meta, _, _, _} -> %% demonitor if the process is not registered under other names maybe_demonitor(Pid, TableByPid), - %% remove from table - remove_from_local_table(Name, Pid, TableByName, TableByPid), - %% callback - syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, Pid, Meta, normal]), - %% broadcast - syn_gen_scope:broadcast({'3.0', sync_unregister, Name, Pid, Meta, normal}, [RequesterNode], State), - show_shadowed(Name, State), - %% return + withdraw(Name, Pid, Meta, normal, [RequesterNode], State), {ok, Meta}; _ -> @@ -590,30 +519,262 @@ unregister_on_node(RequesterNode, Name, Pid, #state{ -spec do_register_on_node( Name :: term(), Pid :: pid(), - PreviousMeta :: term(), Meta :: term(), - MRef :: reference() | undefined, - Reason :: term(), + MRef :: reference(), RequesterNode :: node(), - CallbackMethod :: atom(), #state{} -) -> - {ok, {CallbackMethod :: atom(), PreviousMeta :: term(), Meta :: term(), Time :: integer()}}. -do_register_on_node(Name, Pid, PreviousMeta, Meta, MRef, Reason, RequesterNode, CallbackMethod, #state{ +) -> {ok, Meta :: term(), Time :: integer()}. +do_register_on_node(Name, Pid, Meta, MRef, RequesterNode, State) -> + Time = erlang:system_time(), + case read(Name, State) of + {Before, []} -> + %% no other node holds the name: what this node shows is its own registration + show(Name, Before, {Pid, Meta, Time, MRef}, normal, State); + + Stored -> + {_, Others} = claims(Stored), + store(Name, {{Pid, Meta, Time, MRef}, Others}, normal, Stored, State) + end, + syn_gen_scope:broadcast({'3.5', sync_register, self(), Name, Pid, Meta, Time, normal}, [RequesterNode], State), + {ok, Meta, Time}. + +%% this node drops its own registration of Pid under Name and tells the other nodes, but ExcludedNodes +-spec withdraw(Name :: term(), Pid :: pid(), Meta :: term(), Reason :: term(), ExcludedNodes :: [node()], #state{}) -> + any(). +withdraw(Name, Pid, Meta, Reason, ExcludedNodes, State) -> + case read(Name, State) of + {Before, []} -> + %% no other node holds the name: nothing to show instead + show(Name, Before, undefined, Reason, State); + + Stored -> + {_, Others} = claims(Stored), + store(Name, {undefined, Others}, Reason, Stored, State) + end, + syn_gen_scope:broadcast({'3.5', sync_unregister, self(), Name, Pid, Meta, Reason}, ExcludedNodes, State). + +%% Pid's node holds Pid under Name, with Meta and Time +-spec put_claim(Name :: term(), Pid :: pid(), Meta :: term(), Time :: integer(), Reason :: term(), #state{}) -> any(). +put_claim(Name, Pid, Meta, Time, Reason, State) -> + Node = node(Pid), + case read(Name, State) of + {{_, Pid, Meta, Time, _, _}, _} -> + %% nothing changed + ok; + + {Before, []} when Before =:= undefined; element(6, Before) =:= Node -> + %% no other node holds the name: what this node shows is this registration + show(Name, Before, {Pid, Meta, Time, undefined}, Reason, State); + + Stored -> + {Own, Others} = claims(Stored), + Claim = {Pid, Meta, Time}, + case maps:find(Node, Others) of + {ok, Claim} -> + %% nothing changed + ok; + + _ -> + store(Name, {Own, Others#{Node => Claim}}, Reason, Stored, State), + case Own of + undefined -> ok; + _ -> resolve(Name, [Node], State) + end + end + end. + +%% Pid's node doesn't hold Pid under Name anymore +-spec drop_claim(Name :: term(), Pid :: pid(), Reason :: term(), #state{}) -> any(). +drop_claim(Name, Pid, Reason, State) -> + case read(Name, State) of + {{_, Pid, _, _, _, _} = Before, []} -> + %% no other node holds the name: nothing to show instead + show(Name, Before, undefined, Reason, State); + + Stored -> + {Own, Others} = claims(Stored), + Node = node(Pid), + case maps:find(Node, Others) of + {ok, {Pid, _, _}} -> store(Name, {Own, maps:remove(Node, Others)}, Reason, Stored, State); + _ -> ok + end + end. + +%% what this node stores for Name: the entry it shows, and the registrations of other nodes it keeps aside +-spec read(Name :: term(), #state{}) -> {Shown :: syn_registry_entry() | undefined, Aside :: [aside_entry()]}. +read(Name, #state{table_by_name = TableByName} = State) -> + {find_registry_entry_by_name(Name, TableByName), ets:lookup(aside_table(State), Name)}. + +%% the registrations of Name that this node has: its own, if it holds one, and those of other nodes, by node +-spec claims({Shown :: syn_registry_entry() | undefined, Aside :: [aside_entry()]}) -> + {Own :: own_claim() | undefined, Others :: #{node() => claim()}}. +claims({Shown, Aside}) -> + Others = maps:from_list([{Node, {Pid, Meta, Time}} || {_, Node, Pid, Meta, Time} <- Aside]), + LocalNode = node(), + case Shown of + undefined -> {undefined, Others}; + {_, Pid, Meta, Time, MRef, LocalNode} -> {{Pid, Meta, Time, MRef}, Others}; + {_, Pid, Meta, Time, _, Node} -> {undefined, Others#{Node => {Pid, Meta, Time}}} + end. + +%% the names a node holds, as this node has them +-spec claimed_by_node(Node :: node(), #state{}) -> [{Name :: term(), Pid :: pid()}]. +claimed_by_node(Node, #state{table_by_name = TableByName} = State) -> + Shown = ets:select(TableByName, [{{'$1', '$2', '_', '_', '_', Node}, [], [{{'$1', '$2'}}]}]), + Aside = ets:select(aside_table(State), [{{'$1', Node, '$2', '_', '_'}, [], [{{'$1', '$2'}}]}]), + Shown ++ Aside. + +%% stores the registrations of Name that this node has, where it had Stored: the one it shows goes in the tables, the +%% others aside. A node shows its own registration if it holds one, and otherwise the most recent one, the node name +%% breaking ties, so that every node shows the same one. The callbacks report the change in what this node shows. +-spec store( + Name :: term(), + {Own :: own_claim() | undefined, Others :: #{node() => claim()}}, + Reason :: term(), + Stored :: {Shown :: syn_registry_entry() | undefined, Aside :: [aside_entry()]}, + #state{} +) -> any(). +store(Name, {Own, Others}, Reason, {Before, BeforeAside}, State) -> + Shown = case Own of + undefined -> newest(Others); + _ -> Own + end, + %% aside: only what changed, most names have no registration aside + ShownNode = case Shown of + undefined -> undefined; + {ShownPid, _, _, _} -> node(ShownPid) + end, + Aside = [{Name, Node, Pid, Meta, Time} || {Node, {Pid, Meta, Time}} <- maps:to_list(Others), Node =/= ShownNode], + case Aside =:= BeforeAside of + true -> + ok; + + false -> + AsideTable = aside_table(State), + AsideSet = maps:from_keys(Aside, true), + BeforeAsideSet = maps:from_keys(BeforeAside, true), + [true = ets:delete_object(AsideTable, Entry) || Entry <- BeforeAside, not maps:is_key(Entry, AsideSet)], + [true = ets:insert(AsideTable, Entry) || Entry <- Aside, not maps:is_key(Entry, BeforeAsideSet)] + end, + show(Name, Before, Shown, Reason, State). + +%% puts Shown in the tables in place of Before, the entry this node showed for Name, and calls the callbacks for the change +-spec show( + Name :: term(), + Before :: syn_registry_entry() | undefined, + Shown :: {Pid :: pid(), Meta :: term(), Time :: integer(), MRef :: reference() | undefined} | undefined, + Reason :: term(), + #state{} +) -> any(). +show(Name, Before, Shown, Reason, #state{ scope = Scope, table_by_name = TableByName, table_by_pid = TableByPid -} = State) -> - %% add to local table - Time = erlang:system_time(), - add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid), - %% callback - syn_event_handler:call_event_handler(CallbackMethod, [Scope, Name, Pid, Meta, Reason]), - syn_event_handler:call_event_handler(CallbackMethod, [Scope, Name, Pid, PreviousMeta, Meta, normal]), - %% broadcast - syn_gen_scope:broadcast({'3.0', sync_register, Name, Pid, Meta, Time, Reason}, [RequesterNode], State), - %% return - {ok, {CallbackMethod, PreviousMeta, Meta, Time}}. +}) -> + case {Before, Shown} of + {undefined, undefined} -> + ok; + + {undefined, {Pid, Meta, Time, MRef}} -> + add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid), + syn_event_handler:call_event_handler(on_process_registered, [Scope, Name, Pid, Meta, Reason]); + + {{_, TablePid, TableMeta, _, _, _}, undefined} -> + remove_from_local_table(Name, TablePid, TableByName, TableByPid), + syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, TablePid, TableMeta, Reason]); + + {{_, Pid, Meta, Time, MRef, _}, {Pid, Meta, Time, MRef}} -> + ok; + + {{_, Pid, TableMeta, _, _, _}, {Pid, Meta, Time, MRef}} -> + add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid), + case TableMeta =/= Meta of + true -> + syn_event_handler:call_event_handler(on_registry_process_updated, [Scope, Name, Pid, Meta, Reason]), + syn_event_handler:call_event_handler(on_registry_process_updated, [Scope, Name, Pid, TableMeta, Meta, Reason]); + + false -> + ok + end; + + {{_, TablePid, TableMeta, _, _, _}, {Pid, Meta, Time, MRef}} -> + remove_from_local_table(Name, TablePid, TableByName, TableByPid), + add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid), + syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, TablePid, TableMeta, Reason]), + syn_event_handler:call_event_handler(on_process_registered, [Scope, Name, Pid, Meta, Reason]) + end. + +%% the most recent of the other nodes' registrations, the node name breaking ties, or undefined +-spec newest(Others :: #{node() => claim()}) -> + {Pid :: pid(), Meta :: term(), Time :: integer(), MRef :: undefined} | undefined. +newest(Others) -> + Newest = maps:fold(fun + (Node, {_, _, Time} = Claim, {NewestTime, NewestNode, _}) when {Time, Node} > {NewestTime, NewestNode} -> + {Time, Node, Claim}; + (_, _, {_, _, _} = Acc) -> + Acc; + (Node, {_, _, Time} = Claim, undefined) -> + {Time, Node, Claim} + end, undefined, Others), + case Newest of + undefined -> undefined; + {_, _, {Pid, Meta, Time}} -> {Pid, Meta, Time, undefined} + end. + +%% this node holds Name and has other nodes' registrations of it: the resolver decides between this node's process and +%% the registrations of Nodes (all of them when this node's own registration changed, otherwise the ones that changed). +%% This node keeps its own only if it wins every time, and otherwise drops it and tells everyone; the node that keeps +%% its own does nothing, since the others drop theirs +-spec resolve(Name :: term(), Nodes :: [node()] | all, #state{}) -> any(). +resolve(Name, Nodes, #state{scope = Scope, table_by_pid = TableByPid} = State) -> + case claims(read(Name, State)) of + {{TablePid, TableMeta, TableTime, _}, Others} when map_size(Others) > 0 -> + Contenders = case Nodes of + all -> maps:values(Others); + _ -> [Claim || {Node, Claim} <- maps:to_list(Others), lists:member(Node, Nodes)] + end, + Outcomes = lists:map(fun({Pid, Meta, _Time} = Claim) -> + %% both nodes pass the two registrations in the same order, the one of the node whose name sorts + %% first being the first, so that a resolver that only looks at its arguments decides the same on both + {First, Second} = case node(Pid) < node(TablePid) of + true -> {Claim, {TablePid, TableMeta, TableTime}}; + false -> {{TablePid, TableMeta, TableTime}, Claim} + end, + {PidToKeep, KillOtherPid} = syn_event_handler:do_resolve_registry_conflict(Scope, Name, First, Second), + Keep = case PidToKeep of + TablePid -> local; + Pid -> remote; + Invalid -> {none, Invalid} + end, + ?LOG_NOTICE(#{handler => ?MODULE_LOG_NAME, scope => Scope, + event => registry_conflict, name => Name, + remote => {Pid, Meta}, local => {TablePid, TableMeta}, keep => Keep}), + {Keep, KillOtherPid} + end, Contenders), + case [KillOtherPid || {Keep, KillOtherPid} <- Outcomes, Keep =/= local] of + [] -> + ok; + + Kills -> + maybe_demonitor(TablePid, TableByPid), + case lists:member(true, Kills) of + true -> exit(TablePid, {syn_resolve_kill, Name, TableMeta}); + false -> ok + end, + withdraw(Name, TablePid, TableMeta, syn_conflict_resolution, [], State) + end; + + _ -> + ok + end. + +%% a message about Pid comes from the scope process this node knows for Pid's node +-spec is_from_owner(ScopePid :: pid(), Pid :: pid(), #state{}) -> boolean(). +is_from_owner(ScopePid, Pid, State) -> + node(ScopePid) =:= node(Pid) andalso syn_gen_scope:is_current_scope(ScopePid, State). + +-spec aside_table(#state{}) -> ets:table(). +aside_table(#state{handler_state = #{aside := Aside}}) -> Aside. -spec get_registry_tuples_for_node(Node :: node(), TableByName :: atom()) -> [syn_registry_tuple()]. get_registry_tuples_for_node(Node, TableByName) -> @@ -672,7 +833,7 @@ maybe_demonitor(Pid, TableByPid) -> Pid :: pid(), Meta :: term(), Time :: integer(), - MRef :: undefined | reference() | caller, + MRef :: undefined | reference(), TableByName :: atom(), TableByPid :: atom() ) -> true. @@ -693,23 +854,6 @@ remove_from_local_table(Name, Pid, TableByName, TableByPid) -> true = ets:match_delete(TableByName, {Name, Pid, '_', '_', '_', '_'}), true = ets:match_delete(TableByPid, {Pid, Name, '_', '_', '_', '_'}). --spec update_local_table( - Name :: term(), - PreviousPid :: pid(), - { - Pid :: pid(), - Meta :: term(), - Time :: integer(), - MRef :: undefined | reference() - }, - TableByName :: atom(), - TableByPid :: atom() -) -> true. -update_local_table(Name, PreviousPid, {Pid, Meta, Time, MRef}, TableByName, TableByPid) -> - maybe_demonitor(PreviousPid, TableByPid), - remove_from_local_table(Name, PreviousPid, TableByName, TableByPid), - add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid). - -spec purge_registry_for_remote_nodes(Scope :: atom(), TableByName :: atom(), TableByPid :: atom()) -> any(). purge_registry_for_remote_nodes(Scope, TableByName, TableByPid) -> LocalNode = node(), @@ -723,8 +867,7 @@ purge_registry_for_remote_nodes(Scope, TableByName, TableByPid) -> purge_registry_for_remote_node(Scope, RemoteNode, TableByName, TableByPid) end, undefined, RemoteNodes). --spec purge_registry_for_remote_node(Scope :: atom(), Node :: atom(), TableByName :: atom(), TableByPid :: atom()) -> - [syn_registry_tuple()]. +-spec purge_registry_for_remote_node(Scope :: atom(), Node :: atom(), TableByName :: atom(), TableByPid :: atom()) -> true. purge_registry_for_remote_node(Scope, Node, TableByName, TableByPid) when Node =/= node() -> %% loop elements for callback RegistryTuples = get_registry_tuples_for_node(Node, TableByName), @@ -735,295 +878,4 @@ purge_registry_for_remote_node(Scope, Node, TableByName, TableByPid) when Node = end, RegistryTuples), %% remove all from pid table true = ets:match_delete(TableByName, {'_', '_', '_', '_', '_', Node}), - true = ets:match_delete(TableByPid, {'_', '_', '_', '_', '_', Node}), - RegistryTuples. - --spec reconcile_remote_registry_snapshot(Node :: node(), [syn_registry_tuple()], #state{}) -> ok. -reconcile_remote_registry_snapshot(Node, RegistryTuplesOfRemoteNode, #state{ - scope = Scope, - table_by_name = TableByName, - table_by_pid = TableByPid -} = State) -> - ExistingEntries = ets:select(TableByName, [{ - {'$1', '$2', '$3', '$4', '$5', Node}, - [], - [{{'$1', '$2', '$3', '$4', '$5'}}] - }]), - Shadows = shadows_of_node(Node, State), - case ExistingEntries =:= [] andalso Shadows =:= [] of - true -> - ok; - - false -> - %% compare pids too: an entry whose name the snapshot has with another pid is stale, whatever its time - SnapshotPids = maps:from_list([{Name, Pid} || {Name, Pid, _Meta, _Time} <- RegistryTuplesOfRemoteNode]), - %% registrations kept aside that the node doesn't have anymore - lists:foreach(fun({Name, _, Pid, _, _, _}) -> - case maps:get(Name, SnapshotPids, undefined) =:= Pid of - true -> ok; - false -> unshadow(Name, Pid, State) - end - end, Shadows), - SnapshotTime = lists:foldl(fun({_, _, _, Time}, Acc) -> max(Time, Acc) end, 0, RegistryTuplesOfRemoteNode), - lists:foreach(fun({Name, Pid, Meta, Time, MRef}) -> - case maps:get(Name, SnapshotPids, undefined) =:= Pid orelse is_newer_caller_entry(MRef, Time, SnapshotTime) of - true -> - ok; - - false -> - remove_from_local_table(Name, Pid, TableByName, TableByPid), - syn_event_handler:call_event_handler(on_process_unregistered, - [Scope, Name, Pid, Meta, {syn_remote_scope_node_up, Scope, Node}] - ), - show_shadowed(Name, State) - end - end, ExistingEntries) - end. - -%% an entry this node wrote itself from the reply to its own call, newer than anything in the snapshot: -%% the snapshot may have been sent before the call was handled, so it can't know about it yet --spec is_newer_caller_entry(MRef :: undefined | reference() | caller, Time :: integer(), SnapshotTime :: integer()) -> boolean(). -is_newer_caller_entry(caller, Time, SnapshotTime) -> Time > SnapshotTime; -is_newer_caller_entry(_, _, _) -> false. - --spec handle_registry_sync( - Name :: term(), - Pid :: pid(), - Meta :: term(), - Time :: non_neg_integer(), - Reason :: term(), - #state{} -) -> any(). -handle_registry_sync(Name, Pid, Meta, Time, Reason, #state{ - scope = Scope, - table_by_name = TableByName, - table_by_pid = TableByPid -} = State) -> - case find_registry_entry_by_name(Name, TableByName) of - undefined -> - %% no conflict - add_to_local_table(Name, Pid, Meta, Time, undefined, TableByName, TableByPid), - unshadow(Name, node(Pid), State), - %% callback - syn_event_handler:call_event_handler(on_process_registered, [Scope, Name, Pid, Meta, Reason]); - - {_, Pid, _, TableTime, caller, _} when Time < TableTime -> - %% same pid, older than what this node wrote from the reply to its own call: ignore - ok; - - {_, Pid, TableMeta, _, MRef, _} -> - %% same pid, more recent (because it comes from the same node, which means that it's sequential) - %% maybe updated meta or time only - add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid), - unshadow(Name, node(Pid), State), - %% callback (call only if meta update) - case TableMeta =/= Meta of - true -> - syn_event_handler:call_event_handler(on_registry_process_updated, [Scope, Name, Pid, Meta, Reason]), - syn_event_handler:call_event_handler(on_registry_process_updated, [Scope, Name, Pid, TableMeta, Meta, Reason]); - _ -> ok - end; - - {_, TablePid, TableMeta, TableTime, TableMRef, TableNode} when TableNode =:= node() -> - %% current node runs a conflicting process -> resolve - %% * the conflict is resolved by the two nodes that own the conflicting processes - %% * when a process is chosen, the time is updated - %% * the node that runs the process that is kept sends the sync_register message - %% * recipients check that the time is more recent that what they have to ensure that there are no race conditions - resolve_conflict(Scope, Name, {Pid, Meta, Time}, {TablePid, TableMeta, TableTime, TableMRef}, State); - - {_, TablePid, TableMeta, TableTime, _, TableNode} when TableTime < Time -> - %% current node does not own any of the conflicting processes, update - update_local_table(Name, TablePid, {Pid, Meta, Time, undefined}, TableByName, TableByPid), - unshadow(Name, node(Pid), State), - %% the replaced one is still registered on its node, unless it's the same node - case TableNode =/= node(Pid) of - true -> shadow(Name, TablePid, TableMeta, TableTime, normal, State); - false -> ok - end, - %% callbacks - syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, TablePid, TableMeta, Reason]), - syn_event_handler:call_event_handler(on_process_registered, [Scope, Name, Pid, Meta, Reason]); - - {_, _, _, _, _, TableNode} when TableNode =/= node(Pid) -> - %% current node does not own any of the conflicting processes, incoming data is older or as old: - %% keep it aside, it's still registered on its node - shadow(Name, Pid, Meta, Time, Reason, State); - - _ -> - %% race condition: incoming data is older, ignore - ok - end. - --spec handle_registry_unregister_sync( - Name :: term(), - Pid :: pid(), - Meta :: term(), - Reason :: term(), - #state{} -) -> any(). -handle_registry_unregister_sync(Name, Pid, Meta, Reason, #state{ - scope = Scope, - table_by_name = TableByName, - table_by_pid = TableByPid -} = State) -> - unshadow(Name, Pid, State), - case find_registry_entry_by_name(Name, TableByName) of - {_, Pid, _, _, _, _} -> - remove_from_local_table(Name, Pid, TableByName, TableByPid), - %% callback - syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, Pid, Meta, Reason]), - %% after a conflict resolution, the node that keeps its process tells everyone - case Reason of - syn_conflict_resolution -> ok; - _ -> show_shadowed(Name, State) - end; - - _ -> - %% not in table, nothing to do - ok - end. - -%% registrations of a name by other nodes that the table doesn't show, because it shows one at least as recent: -%% kept aside until their node drops them, so that the most recent one gets shown if the table's one goes first --spec shadow(Name :: term(), Pid :: pid(), Meta :: term(), Time :: integer(), Reason :: term(), #state{}) -> any(). -shadow(Name, Pid, Meta, Time, Reason, State) -> - case shadowed_table(State) of - undefined -> - ok; - - Shadowed -> - %% one per node, the last one it sent - unshadow(Name, node(Pid), State), - true = ets:insert(Shadowed, {Name, node(Pid), Pid, Meta, Time, Reason}) - end. - -%% drops the registration of Name kept aside for a node, or with a pid --spec unshadow(Name :: term(), NodeOrPid :: node() | pid(), #state{}) -> any(). -unshadow(Name, NodeOrPid, State) -> - case shadowed_table(State) of - undefined -> - ok; - - Shadowed -> - lists:foreach(fun - ({_, Node, Pid, _, _, _} = Shadow) when Node =:= NodeOrPid; Pid =:= NodeOrPid -> - true = ets:delete_object(Shadowed, Shadow); - (_) -> - ok - end, ets:lookup(Shadowed, Name)) - end. - -%% drops all the registrations kept aside for a node --spec unshadow_node(Node :: node(), #state{}) -> any(). -unshadow_node(Node, State) -> - case shadowed_table(State) of - undefined -> ok; - Shadowed -> true = ets:match_delete(Shadowed, {'_', Node, '_', '_', '_', '_'}) - end. - --spec shadows_of_node(Node :: node(), #state{}) -> [tuple()]. -shadows_of_node(Node, State) -> - case shadowed_table(State) of - undefined -> []; - Shadowed -> ets:match_object(Shadowed, {'_', Node, '_', '_', '_', '_'}) - end. - -%% the table doesn't have Name anymore: show the most recent registration kept aside, if any --spec show_shadowed(Name :: term(), #state{}) -> any(). -show_shadowed(Name, #state{ - scope = Scope, - table_by_name = TableByName, - table_by_pid = TableByPid -} = State) -> - case shadowed_table(State) of - undefined -> - ok; - - Shadowed -> - case {ets:lookup(Shadowed, Name), find_registry_entry_by_name(Name, TableByName)} of - {[_ | _] = Shadows, undefined} -> - %% most recent, the node breaking ties - {_, _, Pid, Meta, Time, Reason} = Shadow = lists:last(lists:keysort(5, lists:keysort(2, Shadows))), - true = ets:delete_object(Shadowed, Shadow), - add_to_local_table(Name, Pid, Meta, Time, undefined, TableByName, TableByPid), - syn_event_handler:call_event_handler(on_process_registered, [Scope, Name, Pid, Meta, Reason]); - - _ -> - ok - end - end. - --spec shadowed_table(#state{}) -> ets:table() | undefined. -shadowed_table(#state{handler_state = #{shadowed := Shadowed}}) -> Shadowed; -shadowed_table(_) -> undefined. - --spec resolve_conflict( - Scope :: atom(), - Name :: term(), - {Pid :: pid(), Meta :: term(), Time :: non_neg_integer()}, - {TablePid :: pid(), TableMeta :: term(), TableTime :: non_neg_integer(), TableMRef :: reference()}, - #state{} -) -> any(). -resolve_conflict(Scope, Name, {Pid, Meta, Time}, {TablePid, TableMeta, TableTime, TableMRef}, #state{ - table_by_name = TableByName, - table_by_pid = TableByPid -} = State) -> - %% call conflict resolution - {PidToKeep, KillOtherPid} = syn_event_handler:do_resolve_registry_conflict( - Scope, - Name, - {Pid, Meta, Time}, - {TablePid, TableMeta, TableTime} - ), - %% resolve - case PidToKeep of - Pid -> - %% -> we keep the remote pid - ?LOG_NOTICE(#{handler => ?MODULE_LOG_NAME, scope => Scope, - event => registry_conflict, name => Name, - remote => {Pid, Meta}, local => {TablePid, TableMeta}, keep => remote}), - %% update locally, the incoming sync_register will update with the time coming from remote node - update_local_table(Name, TablePid, {Pid, Meta, Time, undefined}, TableByName, TableByPid), - unshadow(Name, node(Pid), State), - %% kill - case KillOtherPid of - true -> exit(TablePid, {syn_resolve_kill, Name, TableMeta}); - false -> ok - end, - %% callbacks - syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, TablePid, TableMeta, syn_conflict_resolution]), - syn_event_handler:call_event_handler(on_process_registered, [Scope, Name, Pid, Meta, syn_conflict_resolution]), - %% broadcast unregister for the losing local pid so other nodes clean up stale ack_sync data - syn_gen_scope:broadcast({'3.0', sync_unregister, Name, TablePid, TableMeta, syn_conflict_resolution}, State); - - TablePid -> - %% -> we keep the local pid, remote pid will be killed by the other node in the conflict - ?LOG_NOTICE(#{handler => ?MODULE_LOG_NAME, scope => Scope, - event => registry_conflict, name => Name, - remote => {Pid, Meta}, local => {TablePid, TableMeta}, keep => local}), - %% overwrite with updated time - ResolveTime = erlang:system_time(), - add_to_local_table(Name, TablePid, TableMeta, ResolveTime, TableMRef, TableByName, TableByPid), - %% broadcast to all (including remote node to update the time) - syn_gen_scope:broadcast({'3.0', sync_register, Name, TablePid, TableMeta, ResolveTime, syn_conflict_resolution}, State); - - Invalid -> - ?LOG_NOTICE(#{handler => ?MODULE_LOG_NAME, scope => Scope, - event => registry_conflict, name => Name, - remote => {Pid, Meta}, local => {TablePid, TableMeta}, - keep => {none, Invalid}}), - %% remove - maybe_demonitor(TablePid, TableByPid), - remove_from_local_table(Name, TablePid, TableByName, TableByPid), - %% kill local, remote will be killed by other node performing the same resolve - case KillOtherPid of - true -> exit(TablePid, {syn_resolve_kill, Name, TableMeta}); - false -> ok - end, - %% callback - syn_event_handler:call_event_handler(on_process_unregistered, [Scope, Name, TablePid, TableMeta, syn_conflict_resolution]), - %% broadcast to all, remote node included: if its process died before it could see the conflict, - %% it has added ours from our snapshot and this removes it - syn_gen_scope:broadcast({'3.0', sync_unregister, Name, TablePid, TableMeta, syn_conflict_resolution}, State) - end. + true = ets:match_delete(TableByPid, {'_', '_', '_', '_', '_', Node}). diff --git a/test/property/syn_pg_cluster_SUITE.erl b/test/property/syn_pg_cluster_SUITE.erl index 307fe46e..2f8d0f52 100644 --- a/test/property/syn_pg_cluster_SUITE.erl +++ b/test/property/syn_pg_cluster_SUITE.erl @@ -786,10 +786,7 @@ start_node(#state{nodes = Nodes}) -> args => ["-connect_all", "false", "-kernel", "dist_auto_connect", "never"], connection => 0 }), - CodePath = lists:filter(fun(Path) -> - nomatch =/= string:find(Path, "/syn/") - end, code:get_path()), - ok = rpc:call(Node, code, add_pathsa, [CodePath]), + ok = rpc:call(Node, code, add_pathsa, [code:get_path()]), ExistingNodes = [N#node_st.node || N <- maps:values(Nodes)], lists:foreach(fun(EN) -> rpc:call(Node, net_kernel, connect_node, [EN]), diff --git a/test/property/syn_registry_cluster_SUITE.erl b/test/property/syn_registry_cluster_SUITE.erl index 05f2c79f..84ca55b3 100644 --- a/test/property/syn_registry_cluster_SUITE.erl +++ b/test/property/syn_registry_cluster_SUITE.erl @@ -693,10 +693,7 @@ start_node(#state{nodes = Nodes}) -> args => ["-connect_all", "false", "-kernel", "dist_auto_connect", "never"], connection => 0 }), - CodePath = lists:filter(fun(Path) -> - nomatch =/= string:find(Path, "/syn/") - end, code:get_path()), - ok = rpc:call(Node, code, add_pathsa, [CodePath]), + ok = rpc:call(Node, code, add_pathsa, [code:get_path()]), ExistingNodes = [N#node_st.node || N <- maps:values(Nodes)], lists:foreach(fun(EN) -> rpc:call(Node, net_kernel, connect_node, [EN]), diff --git a/test/syn_benchmark.erl b/test/syn_benchmark.erl index 1960e50b..43946eff 100644 --- a/test/syn_benchmark.erl +++ b/test/syn_benchmark.erl @@ -55,13 +55,15 @@ %% example run: `PROCESS_COUNT=100000 WORKERS_PER_NODE=100 NODES_COUNT=2 make bench` start() -> %% init - ProcessCount = list_to_integer(os:getenv("PROCESS_COUNT", "100000")), + RequestedProcessCount = list_to_integer(os:getenv("PROCESS_COUNT", "100000")), WorkersPerNode = list_to_integer(os:getenv("WORKERS_PER_NODE", "100")), SlavesCount = list_to_integer(os:getenv("NODES_COUNT", "2")), SkipRegistry = case os:getenv("SKIP_REGISTRY") of false -> false; _ -> true end, SkipPG = case os:getenv("SKIP_PG") of false -> false; _ -> true end, - ProcessesPerNode = round(ProcessCount / SlavesCount), + %% every slave gets the same number of processes, and the counts the benchmark waits for are the total of these + ProcessesPerNode = RequestedProcessCount div SlavesCount, + ProcessCount = ProcessesPerNode * SlavesCount, io:format("====> Starting benchmark~n"), io:format(" --> Nodes: ~w / ~w slave(s)~n", [SlavesCount + 1, SlavesCount]), @@ -287,7 +289,7 @@ start() -> register_on_node(CollectorPid, WorkersPerNode, FromName, Pids) -> %% split pids in workers - PidsPerNode = round(length(Pids) / WorkersPerNode), + PidsPerNode = length(Pids) div WorkersPerNode, {WorkerInfo, []} = lists:foldl(fun(I, {WInfo, RPids}) -> {WorkerPids, RestOfPids} = case I of WorkersPerNode -> @@ -322,7 +324,7 @@ worker_register_on_node(Name, [Pid | PidsTail]) -> unregister_on_node(CollectorPid, WorkersPerNode, FromName, ToName) -> %% split pids in workers ProcessesPerNode = ToName - FromName + 1, - ProcessesPerWorker = round(ProcessesPerNode / WorkersPerNode), + ProcessesPerWorker = ProcessesPerNode div WorkersPerNode, WorkerInfo = lists:foldl(fun(I, Acc) -> {WorkerFromName, WorkerToName} = case I of WorkersPerNode -> @@ -355,7 +357,7 @@ worker_unregister_on_node(Name, ToName) -> join_on_node(CollectorPid, WorkersPerNode, Pids) -> %% split pids in workers - PidsPerNode = round(length(Pids) / WorkersPerNode), + PidsPerNode = length(Pids) div WorkersPerNode, {PidsPerWorker, []} = lists:foldl(fun(I, {P, RPids}) -> {WPids, RestOfPids} = case I of WorkersPerNode -> @@ -388,7 +390,7 @@ worker_join_on_node([Pid | PidsTail]) -> leave_on_node(CollectorPid, WorkersPerNode, Pids) -> %% split pids in workers - PidsPerNode = round(length(Pids) / WorkersPerNode), + PidsPerNode = length(Pids) div WorkersPerNode, {PidsPerWorker, []} = lists:foldl(fun(I, {P, RPids}) -> {WPids, RestOfPids} = case I of WorkersPerNode -> diff --git a/test/syn_pg_SUITE.erl b/test/syn_pg_SUITE.erl index cc10cd10..6fd040d0 100644 --- a/test/syn_pg_SUITE.erl +++ b/test/syn_pg_SUITE.erl @@ -57,6 +57,7 @@ three_nodes_remote_join_from_scope_process/1, three_nodes_remote_join_from_another_scope_process/1, three_nodes_remote_join_with_undiscovered_node/1, + three_nodes_remote_join_from_hidden_node/1, three_nodes_clock_going_backwards/1 ]). -export([ @@ -131,6 +132,7 @@ groups() -> three_nodes_remote_join_from_scope_process, three_nodes_remote_join_from_another_scope_process, three_nodes_remote_join_with_undiscovered_node, + three_nodes_remote_join_from_hidden_node, three_nodes_clock_going_backwards ]}, {four_nodes_pg, [shuffle], [ @@ -176,7 +178,7 @@ do_init_per_scope(TestGroup, Count, Config) -> case syn_test_suite_helper:init_cluster(Count) of {error_initializing_cluster, Other} -> end_per_group(TestGroup, Config), - {skip, Other}; + ct:fail({could_not_init_cluster, Other}); NodesConfig -> NodesConfig ++ Config @@ -215,7 +217,8 @@ init_per_testcase(TestCase, Config) -> %% Config0 = Config1 = [tuple()] %% Reason = any() %% ------------------------------------------------------------------- -end_per_testcase(_, _Config) -> +end_per_testcase(_, Config) -> + syn_test_suite_helper:reconnect_cluster(Config), syn_test_suite_helper:clean_after_test(). %% =================================================================== @@ -817,7 +820,7 @@ three_nodes_join_filter_unknown_node(Config) -> %% send sync message from out of scope node InvalidPid = syn_test_suite_helper:start_process(), - {syn_pg_scope_bc, SlaveNode1} ! {'3.0', sync_join, <<"group-name">>, InvalidPid, undefined, os:system_time(millisecond), normal}, + {syn_pg_scope_bc, SlaveNode1} ! {'3.5', sync_join, self(), <<"group-name">>, InvalidPid, undefined, os:system_time(millisecond), normal}, %% check false = rpc:call(SlaveNode1, syn, is_member, [scope_bc, <<"group-name">>, InvalidPid]). @@ -1437,7 +1440,8 @@ three_nodes_multi_call(Config) -> ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]), %% start processes - RecipientLoopLate = fun() -> timer:sleep(200), recipient_loop() end, + %% a member that doesn't answer before the timeout of the calls below, however long the joins take + RecipientLoopLate = fun() -> timer:sleep(5000), recipient_loop() end, Pid = syn_test_suite_helper:start_process(fun recipient_loop/0), Pid2 = syn_test_suite_helper:start_process(RecipientLoopLate), @@ -1918,13 +1922,9 @@ four_nodes_concurrency(Config) -> spawn(SlaveNode2, WorkerFun), spawn(SlaveNode3, WorkerFun), - %% wait for workers done - syn_test_suite_helper:assert_received_messages([ - {done, LocalNode}, - {done, SlaveNode1}, - {done, SlaveNode2}, - {done, SlaveNode3} - ]), + %% wait for workers done: each runs for several seconds, longer than the usual wait for a message + [receive {done, N} -> ok after 60000 -> ct:fail({worker_not_done, N}) end + || N <- [LocalNode, SlaveNode1, SlaveNode2, SlaveNode3]], %% check results are same across network syn_test_suite_helper:assert_wait( @@ -1976,7 +1976,7 @@ three_nodes_ack_sync_reconciles_pg_snapshot(Config) -> RemoteScopePid = rpc:call(SlaveNode1, erlang, whereis, [syn_pg_scope_reconcile]), true = is_pid(RemoteScopePid), - syn_pg_scope_reconcile ! {'3.0', ack_sync, RemoteScopePid, RemoteSnapshot}, + syn_pg_scope_reconcile ! {'3.5', ack_sync, RemoteScopePid, RemoteSnapshot}, syn_test_suite_helper:assert_wait( [], @@ -2050,7 +2050,7 @@ three_nodes_ack_sync_reconciles_large_pg_snapshot(Config) -> ScopePid = whereis(syn_pg_scope_reconcile_large), {reductions, ReductionsBefore} = erlang:process_info(ScopePid, reductions), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, syn_test_suite_helper:assert_wait( [{Pid, applied_meta}], @@ -2084,7 +2084,7 @@ three_nodes_stale_snapshot_does_not_undo_remote_join(Config) -> %% hold the slave's snapshot back in this node's scope mailbox, then join {RemoteScopePid, Snapshot} = get_remote_snapshot(SlaveNode1, syn_pg_scope_stale), ok = sys:suspend(ScopePid), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, CallRef = syn_test_suite_helper:run_async(fun() -> syn:join(scope_stale, "group", Pid, meta) end), %% the slave has it, the call waits for our scope @@ -2113,7 +2113,7 @@ three_nodes_stale_snapshot_does_not_undo_remote_leave(Config) -> {RemoteScopePid, Snapshot} = get_remote_snapshot(SlaveNode1, syn_pg_scope_stale), [{"group", Pid, meta, _}] = Snapshot, ok = sys:suspend(ScopePid), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, CallRef = syn_test_suite_helper:run_async(fun() -> syn:leave(scope_stale, "group", Pid) end), wait_for_message_queue_len(ScopePid, 2), syn_test_suite_helper:assert_async_waiting(CallRef), @@ -2125,8 +2125,8 @@ three_nodes_stale_snapshot_does_not_undo_remote_leave(Config) -> [] = rpc:call(SlaveNode1, syn, members, [scope_stale, "group"]). three_nodes_late_leave_does_not_undo_remote_join(Config) -> - %% the slave removes its process from a group and we add it again right after: the leave the slave sent - %% before our call reaches our scope before the join, so it can't remove it + %% the slave removes its process from a group and we add it again right after: the slave sends the result of our + %% join after the leave it sent before, so the leave reaches our scope first and can't remove the member SlaveNode1 = proplists:get_value(syn_slave_1, Config), ScopePid = start_scope_stale(SlaveNode1), @@ -2134,14 +2134,15 @@ three_nodes_late_leave_does_not_undo_remote_join(Config) -> Pid = syn_test_suite_helper:start_process(SlaveNode1), ok = syn:join(scope_stale, "group", Pid, meta), - %% the slave's leave waits in our scope's queue while we join again - ok = sys:suspend(ScopePid), + %% the slave's sender holds its leave, then the join's broadcast to the other nodes and its result for us + SlaveScopePid = rpc:call(SlaveNode1, erlang, whereis, [syn_pg_scope_stale]), + Sender = (rpc:call(SlaveNode1, sys, get_state, [SlaveScopePid]))#state.sender_pid, + Resume = syn_test_suite_helper:pause_process(Sender), ok = rpc:call(SlaveNode1, syn, leave, [scope_stale, "group", Pid]), - wait_for_message_queue_len(ScopePid, 1), CallRef = syn_test_suite_helper:run_async(fun() -> syn:join(scope_stale, "group", Pid, new_meta) end), - wait_for_message_queue_len(ScopePid, 2), + wait_for_message_queue_len(Sender, 3), syn_test_suite_helper:assert_async_waiting(CallRef), - ok = sys:resume(ScopePid), + Resume(), ok = syn_test_suite_helper:wait_async(CallRef), _ = sys:get_state(ScopePid), @@ -2165,7 +2166,7 @@ three_nodes_scope_down_does_not_undo_remote_join(Config) -> wait_for_message_queue_len(ScopePid, 2), %% and sends us its snapshot, without the member RemoteScopePid = rpc:call(SlaveNode1, erlang, whereis, [syn_pg_scope_stale]), - RemoteScopePid ! {'3.0', discover, ScopePid}, + RemoteScopePid ! {'3.5', discover, ScopePid}, wait_for_message_queue_len(ScopePid, 3), CallRef = syn_test_suite_helper:run_async(fun() -> syn:join(scope_stale, "group", Pid, meta) end), wait_for_message_queue_len(ScopePid, 4), @@ -2207,7 +2208,7 @@ three_nodes_remote_join_from_scope_process(Config) -> %% an older snapshot of the slave waits in the queue of our scope process, then it leaves {RemoteScopePid, Snapshot} = get_remote_snapshot(SlaveNode1, syn_pg_scope_stale), ok = sys:suspend(ScopePid), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, _ = sys:replace_state(ScopePid, fun(State) -> TestPid ! {left, syn:leave(scope_stale, "group", Pid)}, State @@ -2242,6 +2243,27 @@ three_nodes_remote_join_from_another_scope_process(Config) -> ), [{Pid, meta}] = rpc:call(SlaveNode1, syn, members, [scope_stale, "group"]). +three_nodes_remote_join_from_hidden_node(_Config) -> + %% a hidden node takes no part in syncing: its call for a process of ours works, but the two nodes don't take each + %% other in, so neither has the members the other adds + + ok = syn:start(), + ok = syn:add_node_to_scopes([scope_hidden]), + {PeerPid, HiddenNode} = start_hidden_node(syn_hidden_pg, scope_hidden), + try + Pid = syn_test_suite_helper:start_process(), + ok = rpc:call(HiddenNode, syn, join, [scope_hidden, "group", Pid, meta]), + [{Pid, meta}] = syn:members(scope_hidden, "group"), + HiddenPid = syn_test_suite_helper:start_process(HiddenNode), + ok = rpc:call(HiddenNode, syn, join, [scope_hidden, "group", HiddenPid, meta]), + [] = syn:subcluster_nodes(pg, scope_hidden), + [] = rpc:call(HiddenNode, syn, subcluster_nodes, [pg, scope_hidden]), + [{HiddenPid, meta}] = rpc:call(HiddenNode, syn, members, [scope_hidden, "group"]), + [{Pid, meta}] = syn:members(scope_hidden, "group") + after + peer:stop(PeerPid) + end. + three_nodes_remote_join_with_undiscovered_node(Config) -> %% the slave gets our call before discovering us, and we haven't discovered it either: it sends us its data %% before the result, as for a discover, so we already have the member when the call returns @@ -2253,8 +2275,8 @@ three_nodes_remote_join_with_undiscovered_node(Config) -> Pid = syn_test_suite_helper:start_process(SlaveNode1), %% the scope processes forget each other - RemoteScopePid ! {'DOWN', make_ref(), process, ScopePid, test}, - ScopePid ! {'DOWN', make_ref(), process, RemoteScopePid, test}, + forget_scope(SlaveNode1, RemoteScopePid, node()), + forget_scope(node(), ScopePid, SlaveNode1), syn_test_suite_helper:assert_scope_subcluster(pg, node(), scope_stale, []), syn_test_suite_helper:assert_scope_subcluster(pg, SlaveNode1, scope_stale, []), @@ -2270,10 +2292,11 @@ three_nodes_clock_going_backwards(Config) -> SlaveNode1 = proplists:get_value(syn_slave_1, Config), ScopePid = start_scope_stale(SlaveNode1), + RemoteScopePid = rpc:call(SlaveNode1, erlang, whereis, [syn_pg_scope_stale]), Pid = syn_test_suite_helper:start_process(SlaveNode1), - ScopePid ! {'3.0', sync_join, "group", Pid, old, 200, normal}, - ScopePid ! {'3.0', sync_join, "group", Pid, new, 100, normal}, + ScopePid ! {'3.5', sync_join, RemoteScopePid, "group", Pid, old, 200, normal}, + ScopePid ! {'3.5', sync_join, RemoteScopePid, "group", Pid, new, 100, normal}, _ = sys:get_state(ScopePid), [{Pid, new}] = syn:members(scope_stale, "group"). @@ -2307,6 +2330,29 @@ get_remote_snapshot(Node, ScopeProcessName) -> {RemoteScopePid, Snapshot}. %% start syn and add scope_stale on this node and on Node, returns the scope process of this node +%% starts a hidden node with syn and Scope on it, connected to this node only +start_hidden_node(Name, Scope) -> + {ok, PeerPid, Node} = peer:start(#{ + name => Name, + args => ["-hidden", "-connect_all", "false", "-kernel", "dist_auto_connect", "never"], + connection => 0 + }), + ok = rpc:call(Node, code, add_pathsa, [code:get_path()]), + true = net_kernel:connect_node(Node), + false = lists:member(Node, nodes()), + ok = rpc:call(Node, syn, start, []), + ok = rpc:call(Node, syn, add_node_to_scopes, [[Scope]]), + {PeerPid, Node}. + +%% the scope process ScopePid of Node stops knowing the one of OtherNode, as if it had never discovered it +forget_scope(Node, ScopePid, OtherNode) -> + rpc:call(Node, sys, replace_state, [ScopePid, fun(#state{nodes_map = NodesMap} = State) -> + {_, MRef} = maps:get(OtherNode, NodesMap), + erlang:demonitor(MRef, [flush]), + State#state{nodes_map = maps:remove(OtherNode, NodesMap)} + end]), + ok. + start_scope_stale(Node) -> ok = syn:start(), ok = rpc:call(Node, syn, start, []), diff --git a/test/syn_protocol_SUITE.erl b/test/syn_protocol_SUITE.erl new file mode 100644 index 00000000..1a661ef9 --- /dev/null +++ b/test/syn_protocol_SUITE.erl @@ -0,0 +1,778 @@ +%% ========================================================================================================== +%% Syn - A global Process Registry and Process Group manager. +%% +%% The MIT License (MIT) +%% +%% Copyright (c) 2015-2026 Roberto Ostinelli and Neato Robotics, Inc. +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ========================================================================================================== +%% Tests that drive the registry through specific message orders: a sender process that sends one message per step, +%% callbacks that block until the test lets them go, and messages injected as if a peer had sent them. Each test +%% checks that the nodes still end up agreeing, or that a call returns what it promises. +-module(syn_protocol_SUITE). +-export([all/0, suite/0]). +-export([init_per_suite/1, end_per_suite/1]). +-export([init_per_testcase/2, end_per_testcase/2]). + +%% tests +-export([ + conflict_normal/1, + conflict_second_split/1, + conflict_skew/1, + conflict_refresh_livelock/1, + conflict_tie/1, + three_owners_challenger_cut_off/1, + three_owners_challenger_unregistered/1, + discovery_incarnation_registry/1, + discovery_incarnation_pg/1, + ordered_update_withdraw/1, + scope_update_withdraw/1, + local_update_withdraw/1, + late_down_after_resync/1, + snapshot_aside_and_fences/1, + restart_claims/1, + ordered_adoption_fetches_snapshot/1, + late_old_incarnation_recovers/1, + disconnected_handshakes_ignored/1, + killed_process_loses_both_names/1, + unchanged_pairs_not_resolved/1, + monitor_does_not_reconnect/1, + messages_from_another_version_ignored/1, + asked_forgets_nodes_that_go/1, + stalled_node_holds_up_only_itself/1 +]). + +%% run on other nodes, or by Syn as callbacks +-export([idle/0, hold/2, set_owner_time/2, fill/3]). +-export([install_stepped_sender/0, stepped_sender/0, restore_sender/0]). +-export([install_held_sender/0, held_sender/1, held/1, step_to/2, flush/1]). +-export([on_process_registered/5, on_process_unregistered/5]). +-export([on_registry_process_updated/5, on_registry_process_updated/6]). +-export([resolve_registry_conflict/4]). +-include_lib("syn/src/syn.hrl"). + +all() -> [conflict_normal, conflict_second_split, conflict_skew, conflict_refresh_livelock, conflict_tie, + three_owners_challenger_cut_off, three_owners_challenger_unregistered, + discovery_incarnation_registry, discovery_incarnation_pg, + ordered_update_withdraw, scope_update_withdraw, local_update_withdraw, + late_down_after_resync, snapshot_aside_and_fences, restart_claims, + ordered_adoption_fetches_snapshot, + late_old_incarnation_recovers, disconnected_handshakes_ignored, + killed_process_loses_both_names, unchanged_pairs_not_resolved, monitor_does_not_reconnect, + messages_from_another_version_ignored, asked_forgets_nodes_that_go, stalled_node_holds_up_only_itself]. +suite() -> [{timetrap, {seconds, 30}}]. + +init_per_suite(Config) -> + Peers = [begin + {ok, Peer, Node} = peer:start(#{name => Name, connection => 0, + args => ["+S", "2", "-connect_all", "false", "-kernel", + "dist_auto_connect", "never", "prevent_overlapping_partitions", "false"]}), + ok = rpc:call(Node, code, add_pathsa, [code:get_path()]), + {Peer, Node} + end || Name <- [syn_protocol_a, syn_protocol_b]], + [{peers, Peers} | Config]. +end_per_suite(Config) -> + [peer:stop(Peer) || {Peer, _} <- proplists:get_value(peers, Config)], ok. +init_per_testcase(_, Config) -> + [{_, A}, {_, B}] = proplists:get_value(peers, Config), + true = rpc:call(A, net_kernel, connect_node, [B]), + [begin + ok = rpc:call(N, syn, start, []), + ok = rpc:call(N, syn, add_node_to_scopes, [[protocol]]) + end || N <- [node(), A, B]], + [wait(fun() -> lists:sort(rpc:call(N, syn_registry, subcluster_nodes, [protocol])) + =:= lists:sort([node(), A, B] -- [N]) end) || N <- [node(), A, B]], + [{a, A}, {b, B} | Config]. +end_per_testcase(_, Config) -> + [net_kernel:connect_node(N) || {_, N} <- proplists:get_value(peers, Config)], + [begin + rpc:call(N, application, stop, [syn]), + rpc:call(N, application, unset_env, [syn, event_handler]), + rpc:call(N, application, unset_env, [syn, protocol_gate]), + rpc:call(N, application, unset_env, [syn, protocol_observer]), + rpc:call(N, application, unset_env, [syn, protocol_purge_gate]) + end || N <- [node(), proplists:get_value(a, Config), proplists:get_value(b, Config)]], ok. + +wait(Fun) -> wait(Fun, 500). +wait(Fun, 0) -> ct:fail({wait_failed, Fun()}); +wait(Fun, N) -> case catch Fun() of true -> ok; _ -> timer:sleep(10), wait(Fun, N-1) end. +idle() -> receive stop -> ok end. +pid(N) -> rpc:call(N, erlang, spawn, [?MODULE, idle, []]). +scope(N, Handler) -> rpc:call(N, erlang, whereis, [list_to_atom(atom_to_list(Handler) ++ "_protocol")]). +view(N) -> rpc:call(N, syn, lookup, [protocol, x]). +split(A, B) -> + rpc:call(A, erlang, disconnect_node, [B]), + rpc:call(B, erlang, disconnect_node, [A]), + wait(fun() -> rpc:call(A, syn_registry, subcluster_nodes, [protocol]) =:= [node()] + andalso rpc:call(B, syn_registry, subcluster_nodes, [protocol]) =:= [node()] end). +sender(N, Handler) -> (sys:get_state(scope(N, Handler)))#state.sender_pid. +pause_sender(N, Handler) -> + Sender = sender(N, Handler), + Pauser = rpc:call(N, erlang, spawn, [?MODULE, hold, [Sender, self()]]), + receive {paused, Pauser} -> Pauser after 5000 -> error(pause_timeout) end. +hold(Pid, Test) -> + true = erlang:suspend_process(Pid), Test ! {paused, self()}, + receive resume -> erlang:resume_process(Pid) after 20000 -> ok end. +resume_sender(_N, Pauser) -> Pauser ! resume, ok. +messages(Pid) -> {messages, Messages} = rpc:call(node(Pid), erlang, process_info, [Pid, messages]), Messages. +%% pretends that the clock of the node running P is ahead: changes the time of its registration and sends it to the +%% other nodes as that node would +set_owner_time(P, Time) -> + sys:replace_state(scope(node(), syn_registry), fun(S) -> + [{x, P, Meta, _, Ref, _}] = ets:lookup(S#state.table_by_name, x), + syn_registry:add_to_local_table(x, P, Meta, Time, Ref, S#state.table_by_name, S#state.table_by_pid), + syn_gen_scope:broadcast({'3.5', sync_register, self(), x, P, Meta, Time, normal}, S), S + end), ok. +future() -> erlang:system_time() + erlang:convert_time_unit(3600, second, native). +resolve_registry_conflict(_Scope, Name, {P1, M1, _}, {P2, M2, _}) -> + report({resolved, node(), Name, P1, P2}), + case application:get_env(syn, protocol_gate) of + {ok, Test} -> + application:unset_env(syn, protocol_gate), + Test ! {resolver_blocked, self()}, + receive protocol_continue -> ok after 10000 -> error(gate_timeout) end; + undefined -> ok + end, + case M1 < M2 of true -> P1; false -> P2 end. +%% issue #96 without anything going wrong: the owners resolve, the winner neither refreshes its time nor announces +%% itself again, and every node ends up with the winner +conflict_normal(Config) -> conflict(Config, false). +%% issue #96 with the loser's clock ahead: the bystander shows the winner once the loser drops its own +conflict_skew(Config) -> conflict(Config, true). +conflict(Config, Skew) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + [rpc:call(N, syn, set_event_handler, [?MODULE]) || N <- [node(), A, B]], + PA = pid(A), PB = pid(B), + ok = rpc:call(A, syn, register, [protocol, x, PA, 1]), + wait(fun() -> view(B) =:= {PA, 1} andalso view(node()) =:= {PA, 1} end), + split(A, B), + ok = rpc:call(B, syn, register, [protocol, x, PB, 2]), + wait(fun() -> view(node()) =:= {PB, 2} end), + case Skew of + true -> ok = rpc:call(B, ?MODULE, set_owner_time, [PB, future()]), + wait(fun() -> S = sys:get_state(scope(node(), syn_registry)), + [{x, PB, _, T, _, _}] = ets:lookup(S#state.table_by_name, x), + T > erlang:system_time() end); + false -> ok + end, + %% B has sent its snapshot before its resolver runs. The resolver of B waits until the test lets it go, so that A + %% handles that snapshot, keeps its process and sends nothing, before B drops its own. + ok = rpc:call(B, application, set_env, [syn, protocol_gate, self()]), + OldTime = owner_time(A), + true = rpc:call(A, net_kernel, connect_node, [B]), + Blocked = receive {resolver_blocked, SP} -> SP after 5000 -> ct:fail(no_conflict) end, + %% A keeps its process without giving it a new time or telling the other nodes again + _ = sys:get_state(scope(A, syn_registry)), + OldTime = owner_time(A), + Blocked ! protocol_continue, + wait(fun() -> lists:all(fun(N) -> view(N) =:= {PA, 1} end, [node(), A, B]) end), + quiescent([node(), A, B]), + OldTime = owner_time(A), + ct:pal("Skew=~p, converged A/B/Common: ~p", [Skew, [view(N) || N <- [A,B,node()]]]). +owner_time(N) -> + S = sys:get_state(scope(N, syn_registry)), + [{x, _, _, T, _, _}] = rpc:call(N, ets, lookup, [S#state.table_by_name, x]), T. + +%% a sender that sends its next message only when the test says so, to the same nodes and in the same order as the +%% real one would +install_stepped_sender() -> + S = sys:replace_state(scope(node(), syn_registry), fun(State) -> + State#state.sender_pid ! terminate, + State#state{sender_pid = spawn_link(?MODULE, stepped_sender, [])} + end), S#state.sender_pid. +stepped_sender() -> + receive + {step, Test, Ref} -> + receive + {broadcast, Message, Excluded, Name, Nodes} -> + [erlang:send({Name, N}, Message, [noconnect]) || N <- Nodes -- Excluded]; + {send_single, N, Message, Name} -> erlang:send({Name, N}, Message, [noconnect]) + end, + Test ! {stepped, Ref}, stepped_sender(); + {sync, Test, Ref, Nodes} -> + [sys:get_state({syn_registry_protocol, N}) || N <- Nodes], + Test ! {synced, Ref}, stepped_sender(); + terminate -> ok + end. +restore_sender() -> + sys:replace_state(scope(node(), syn_registry), fun(S) -> + S#state.sender_pid ! terminate, + S#state{sender_pid = spawn_link(syn_gen_scope, sender_loop, [])} + end), ok. +step(Sender) -> + Ref = make_ref(), Sender ! {step, self(), Ref}, + receive {stepped, Ref} -> ok after 3000 -> error(step_timeout) end. + +%% a sender that holds what it sends per destination node, and sends the oldest message for a node when the test says +%% so: messages keep their order towards each node, while messages towards different nodes arrive in the order the +%% test picks, as Erlang allows +install_held_sender() -> + S = sys:replace_state(scope(node(), syn_registry), fun(State) -> + State#state.sender_pid ! terminate, + State#state{sender_pid = spawn_link(?MODULE, held_sender, [[]])} + end), S#state.sender_pid. +held_sender(Held) -> + receive + {broadcast, Message, Excluded, Name, Nodes} -> + held_sender(Held ++ [{N, Message, Name} || N <- Nodes -- Excluded]); + {send_single, N, Message, Name} -> held_sender(Held ++ [{N, Message, Name}]); + {held, Test, Ref} -> Test ! {held, Ref, [{N, element(2, M)} || {N, M, _} <- Held]}, held_sender(Held); + {step_to, N, Test, Ref} -> + case lists:keytake(N, 1, Held) of + {value, {N, Message, Name}, Rest} -> + erlang:send({Name, N}, Message, [noconnect]), + Test ! {stepped, Ref, element(2, Message)}, held_sender(Rest); + false -> Test ! {stepped, Ref, nothing}, held_sender(Held) + end; + {flush, Test, Ref} -> + [erlang:send({Name, N}, Message, [noconnect]) || {N, Message, Name} <- Held], + Test ! {flushed, Ref}, held_sender([]); + terminate -> ok + end. +%% the destination node and the kind of each message held, oldest first +held(Sender) -> + Ref = make_ref(), Sender ! {held, self(), Ref}, + receive {held, Ref, Held} -> Held after 3000 -> error(held_timeout) end. +%% sends the oldest message held for N, and returns its kind, or nothing +step_to(Sender, N) -> + Ref = make_ref(), Sender ! {step_to, N, self(), Ref}, + receive {stepped, Ref, Kind} -> Kind after 3000 -> error(step_timeout) end. +%% sends everything held; what is held for a node that isn't connected is lost, as on a dropped connection +flush(Sender) -> + Ref = make_ref(), Sender ! {flush, self(), Ref}, + receive {flushed, Ref} -> ok after 3000 -> error(flush_timeout) end. +%% runs a function on a node; a peer runs it through its control connection, which doesn't connect it to this node +on(_Config, N, M, F, Args) when N =:= node() -> apply(M, F, Args); +on(Config, N, M, F, Args) -> + {Peer, N} = lists:keyfind(N, 2, proplists:get_value(peers, Config)), + peer:call(Peer, M, F, Args). +%% splits any two nodes, this one included, without sending anything that would connect them again +cut(Config, X, Y) -> + on(Config, X, erlang, disconnect_node, [Y]), + on(Config, Y, erlang, disconnect_node, [X]), + wait(fun() -> not lists:member(Y, on(Config, X, syn_registry, subcluster_nodes, [protocol])) + andalso not lists:member(X, on(Config, Y, syn_registry, subcluster_nodes, [protocol])) end). +%% issue #96 as seen on a cluster: the loser's snapshot never reaches the winner, as the connection drops again, and +%% the bystander still ends up with the winner +conflict_second_split(Config) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + [rpc:call(N, syn, set_event_handler, [?MODULE]) || N <- [node(), A, B]], + PA = pid(A), PB = pid(B), + ok = rpc:call(A, syn, register, [protocol, x, PA, 1]), + wait(fun() -> view(B) =:= {PA, 1} andalso view(node()) =:= {PA, 1} end), + split(A, B), + ok = rpc:call(B, syn, register, [protocol, x, PB, 2]), + wait(fun() -> view(node()) =:= {PB, 2} end), + SB = rpc:call(B, ?MODULE, install_stepped_sender, []), + true = rpc:call(A, net_kernel, connect_node, [B]), + %% B's answer to the discover of A waits in B's sender, while B gets the snapshot of A + wait(fun() -> length(messages(SB)) =:= 2 end), + step(SB), + wait(fun() -> view(B) =:= {PA, 1} end), + %% B has already dropped its process, but its snapshot hasn't reached A: a second cut loses it. Common stays connected + %% to both + split(A, B), + step(SB), step(SB), + wait(fun() -> view(node()) =:= {PA, 1} end), + ok = rpc:call(B, ?MODULE, restore_sender, []), + true = rpc:call(A, net_kernel, connect_node, [B]), + wait(fun() -> lists:sort(rpc:call(B, syn_registry, subcluster_nodes, [protocol])) + =:= lists:sort([node(), A]) andalso view(B) =:= {PA, 1} end), + wait(fun() -> lists:all(fun(N) -> messages(scope(N, syn_registry)) =:= [] + andalso messages(sender(N, syn_registry)) =:= [] end, [node(), A, B]) end), + {PA, 1} = view(A), {PA, 1} = view(B), {PA, 1} = view(node()), + ct:pal("Second split, no clock skew, A/B/Common: ~p", [[view(N) || N <- [A,B,node()]]]). +%% both owners update their registration while their snapshots are on the way: the conflict still ends, after a +%% bounded number of messages, and nobody refreshes its time +conflict_refresh_livelock(Config) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + split(A, B), PA = pid(A), PB = pid(B), + ok = rpc:call(A, syn, register, [protocol, x, PA, initial_a]), + ok = rpc:call(B, syn, register, [protocol, x, PB, initial_b]), + wait(fun() -> messages(sender(A, syn_registry)) =:= [] + andalso messages(sender(B, syn_registry)) =:= [] end), + SA = rpc:call(A, ?MODULE, install_stepped_sender, []), + SB = rpc:call(B, ?MODULE, install_stepped_sender, []), + true = rpc:call(A, net_kernel, connect_node, [B]), + step(SA), step(SB), + wait(fun() -> lists:any(fun(M) -> element(1,M) =:= send_single andalso + element(2,element(3,M)) =:= ack_sync end, messages(SA)) andalso + lists:any(fun(M) -> element(1,M) =:= send_single andalso + element(2,element(3,M)) =:= ack_sync end, messages(SB)) end), + %% both snapshots wait in the senders, and both nodes update their registration before getting the other's + ok = rpc:call(A, syn, register, [protocol, x, PA, updated_a]), + ok = rpc:call(B, syn, register, [protocol, x, PB, updated_b]), + TimeA = owner_time(A), TimeB = owner_time(B), + %% both nodes get the other's older snapshot first, and keep their own without giving it a new time + step(SA), step(SB), + _ = sys:get_state(scope(A, syn_registry)), _ = sys:get_state(scope(B, syn_registry)), + TimeA = owner_time(A), TimeB = owner_time(B), + Deliveries = pump([SA, SB], [node(), A, B], 0, 60), + Expected = case TimeA > TimeB of true -> {PA, updated_a}; false -> {PB, updated_b} end, + wait(fun() -> lists:all(fun(N) -> view(N) =:= Expected end, [node(), A, B]) end), + ct:pal("Converged after ~p further FIFO sender instructions, no timestamp refresh", [Deliveries]), + [] = messages(SA), [] = messages(SB), ok. + +%% both owners register the name with the same metadata while apart, and this suite's resolver keeps the second +%% registration it gets on a tie: both nodes pass the two in the same order, so they keep the same process +conflict_tie(Config) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + [rpc:call(N, syn, set_event_handler, [?MODULE]) || N <- [node(), A, B]], + split(A, B), PA = pid(A), PB = pid(B), + ok = rpc:call(A, syn, register, [protocol, x, PA, same]), + ok = rpc:call(B, syn, register, [protocol, x, PB, same]), + true = rpc:call(A, net_kernel, connect_node, [B]), + %% the registration of the node whose name sorts last comes second + Kept = case A < B of true -> PB; false -> PA end, + wait(fun() -> lists:all(fun(N) -> view(N) =:= {Kept, same} end, [node(), A, B]) end), + quiescent([node(), A, B]), + {Kept, same} = view(A), {Kept, same} = view(B), {Kept, same} = view(node()). + +%% issue #96 on a larger cluster, with three owners and this suite's resolver, which keeps the older process. This +%% node, Common, loses its own conflict to A's process, then shows B's newer process while B is cut from A, and B's +%% process loses on B to Common's process, which Common has already dropped. Once B's process leaves Common, through a +%% cut between B and Common or through B's message, Common shows A's process again, and so does every node +three_owners_challenger_cut_off(Config) -> three_owners(Config, cut). +three_owners_challenger_unregistered(Config) -> three_owners(Config, unregister). +three_owners(Config, How) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), Common = node(), + On = fun(N, M, F, Args) -> on(Config, N, M, F, Args) end, + View = fun(N) -> On(N, syn, lookup, [protocol, x]) end, + [ok = On(N, syn, set_event_handler, [?MODULE]) || N <- [Common, A, B]], + PA = On(A, erlang, spawn, [?MODULE, idle, []]), PB = On(B, erlang, spawn, [?MODULE, idle, []]), PC = pid(Common), + cut(Config, A, B), cut(Config, A, Common), + SC = install_held_sender(), SB = On(B, ?MODULE, install_held_sender, []), + ok = On(A, syn, register, [protocol, x, PA, 1]), + ok = syn:register(protocol, x, PC, 2), + [{B, sync_register}] = held(SC), + %% A and Common connect again: Common's messages to A go through until Common has lost to A's process, while + %% those to B wait + true = On(A, net_kernel, connect_node, [Common]), + ToA = fun Loop(0) -> ct:fail({no_conflict, View(Common), held(SC)}); + Loop(N) -> case View(Common) of + {PA, 1} -> ok; + _ -> step_to(SC, A), timer:sleep(20), Loop(N - 1) + end end, + ok = ToA(200), + wait(fun() -> step_to(SC, A) =:= nothing andalso On(A, erlang, process_info, + [On(A, erlang, whereis, [syn_registry_protocol]), message_queue_len]) =:= {message_queue_len, 0} end), + {PA, 1} = View(A), + %% B registers its process, then gets Common's registration of a process that Common has already dropped, and + %% B's process loses + undefined = View(B), + ok = On(B, syn, register, [protocol, x, PB, 3]), + sync_register = step_to(SC, B), + wait(fun() -> View(B) =:= {PC, 2} end), + wait(fun() -> On(B, ?MODULE, held, [SB]) =:= [{Common, sync_register}, {Common, sync_unregister}] end), + %% B's registration reaches Common, which shows it as the newer one + sync_register = On(B, ?MODULE, step_to, [SB, Common]), + wait(fun() -> View(Common) =:= {PB, 3} end), + case How of + cut -> cut(Config, B, Common); + unregister -> sync_unregister = On(B, ?MODULE, step_to, [SB, Common]) + end, + wait(fun() -> View(Common) =:= {PA, 1} end), + %% what is still held goes to the nodes still connected, then every node connects again + ok = flush(SC), ok = On(B, ?MODULE, flush, [SB]), + ok = restore_sender(), ok = On(B, ?MODULE, restore_sender, []), + true = net_kernel:connect_node(B), + true = On(A, net_kernel, connect_node, [B]), + wait(fun() -> lists:all(fun(N) -> lists:sort(On(N, syn_registry, subcluster_nodes, [protocol])) + =:= lists:sort([Common, A, B] -- [N]) end, [Common, A, B]) end), + wait(fun() -> lists:all(fun(N) -> View(N) =:= {PA, 1} end, [Common, A, B]) end), + quiescent([Common, A, B]), + {PA, 1} = View(A), {PA, 1} = View(B), {PA, 1} = View(Common). + +%% a new scope process' handshake arrives before the old one's DOWN, which Erlang allows: the messages are injected to +%% get this order every time. The node keeps the new one, then finds the live peer again +discovery_incarnation_registry(Config) -> incarnation(Config, syn_registry). +discovery_incarnation_pg(Config) -> incarnation(Config, syn_pg). +incarnation(Config, Handler) -> + A = proplists:get_value(a, Config), Local = scope(node(), Handler), Old = scope(A, Handler), + {Old, OldRef} = maps:get(A, (sys:get_state(Local))#state.nodes_map), + Held = pause_sender(A, Handler), FakeNew = pid(A), + Local ! {'3.5', discover, FakeNew}, + Local ! {'3.5', ack_sync, FakeNew, []}, + Local ! {'DOWN', OldRef, process, Old, killed}, + S = sys:get_state(Local), + {FakeNew, _} = maps:get(A, S#state.nodes_map), + exit(FakeNew, kill), + resume_sender(A, Held), + wait(fun() -> case maps:get(A, (sys:get_state(Local))#state.nodes_map, undefined) of + {Old, _} -> true; _ -> false end end), + ct:pal("~p retained the new pid on old DOWN, then rediscovered the live peer", [Handler]). + +quiescent(Ns) -> + [sys:get_state(scope(N, syn_registry)) || N <- Ns], + wait(fun() -> lists:all(fun(N) -> messages(scope(N, syn_registry)) =:= [] + andalso messages(sender(N, syn_registry)) =:= [] end, Ns) end). +%% steps the stepped senders until they have nothing left to send, even once the scope processes of Nodes have handled +%% everything the senders sent them; returns how many messages were sent, and fails past Max +pump(Senders, Nodes, Count, Max) -> + case [S || S <- Senders, messages(S) =/= []] of + [] -> + [sync(S, Nodes) || S <- Senders], + case [S || S <- Senders, messages(S) =/= []] of + [] -> Count; + _ -> pump(Senders, Nodes, Count, Max) + end; + + _ when Count >= Max -> + ct:fail({traffic_did_not_stop, Count}); + + Ready -> + [step(S) || S <- Ready], + pump(Senders, Nodes, Count + length(Ready), Max) + end. + +%% returns once the scope processes of Nodes have handled everything Sender sent them: the sender asks each of them, +%% and its questions arrive after its messages +sync(Sender, Nodes) -> + Ref = make_ref(), Sender ! {sync, self(), Ref, Nodes}, + receive {synced, Ref} -> ok after 5000 -> error(sync_timeout) end. +report(Event) -> + case application:get_env(syn, protocol_observer) of + {ok, Test} -> Test ! Event; + _ -> ok + end. +on_process_registered(Scope, Name, Pid, Meta, Reason) -> + report({registered, node(), Scope, Name, Pid, Meta, Reason}). +on_process_unregistered(Scope, Name, Pid, Meta, Reason) -> + report({unregistered, node(), Scope, Name, Pid, Meta, Reason}), + case application:get_env(syn, protocol_purge_gate) of + {ok, Test} -> + application:unset_env(syn, protocol_purge_gate), + Test ! {purge_blocked, self()}, + receive protocol_continue -> ok after 10000 -> error(gate_timeout) end; + _ -> ok + end. +on_registry_process_updated(Scope, Name, Pid, Meta, Reason) -> + report({updated5, node(), Scope, Name, Pid, Meta, Reason}). +on_registry_process_updated(Scope, Name, Pid, OldMeta, Meta, Reason) -> + report({updated6, node(), Scope, Name, Pid, OldMeta, Meta, Reason}). +collect() -> receive M -> [M | collect()] after 0 -> [] end. + +%% an update that makes the owner lose its conflict: the caller's node gets the result before the withdrawal, for a +%% call from another node, for a call from a scope process, and for a call on the owner's node +ordered_update_withdraw(C) -> update_withdraw(C, applied). +scope_update_withdraw(C) -> update_withdraw(C, handled). +local_update_withdraw(C) -> update_withdraw(C, local). +update_withdraw(Config, Kind) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + [rpc:call(N, syn, set_event_handler, [?MODULE]) || N <- [node(), A, B]], + PA = pid(A), PB = pid(B), + ok = rpc:call(A, syn, register, [protocol, x, PA, 1]), + wait(fun() -> view(node()) =:= {PA, 1} end), + %% A has the registration of B and keeps its own, while B doesn't know about the one of A yet + ScopeB = scope(B, syn_registry), ScopeA = scope(A, syn_registry), + ScopeA ! {'3.5', sync_register, ScopeB, x, PB, 2, 0, normal}, + scope(node(), syn_registry) ! {'3.5', sync_register, ScopeB, x, PB, 2, 0, normal}, + _ = sys:get_state(scope(node(), syn_registry)), + _ = sys:get_state(ScopeA), + S = rpc:call(A, ?MODULE, install_stepped_sender, []), + rpc:call(A, application, set_env, [syn, protocol_observer, self()]), + Test = self(), + Call = fun() -> syn:update_registry(protocol, x, fun(_, _) -> 3 end) end, + case Kind of + applied -> spawn(fun() -> Test ! {update_result, Call()} end); + handled -> sys:replace_state(scope(node(), syn_registry), fun(St) -> + Test ! {update_result, Call()}, St end); + local -> Test ! {update_result, rpc:call(A, syn, update_registry, [protocol, x, fun(_, _) -> 3 end])} + end, + ExpectedCount = case Kind of local -> 2; _ -> 3 end, + wait(fun() -> length(messages(S)) =:= ExpectedCount end), + Instructions = messages(S), + Types = [case I of {broadcast, M, _, _, _} -> element(2, M); + {send_single, _, M, _} -> element(2, M) end || I <- Instructions], + ExpectedTypes = case Kind of local -> [sync_register, sync_unregister]; + _ -> [sync_register, sync_register_reply, sync_unregister] end, + ExpectedTypes = Types, + [step(S) || _ <- Instructions], + receive {update_result, Result} -> {ok, {PA, 3}} = Result after 5000 -> ct:fail(no_result) end, + wait(fun() -> view(node()) =:= {PB, 2} end), + Events = collect(), + Update5 = {updated5, A, protocol, x, PA, 3, normal}, + Update6 = {updated6, A, protocol, x, PA, 1, 3, normal}, + Unregister = {unregistered, A, protocol, x, PA, 3, syn_conflict_resolution}, + Relevant = [E || E <- Events, lists:member(element(1, E), [updated5, updated6, unregistered])], + [Update5, Update6, Unregister] = Relevant, + ct:pal("~p result queued before withdrawal; both update arities precede conflict unregister", [Kind]). + +%% a DOWN that arrives after the peer's snapshot: the node drops the peer, sees it's still connected, and asks again +late_down_after_resync(Config) -> + A = proplists:get_value(a, Config), P = pid(A), + ok = syn:register(protocol, x, P, meta), + Local = scope(node(), syn_registry), Remote = scope(A, syn_registry), + {Remote, Ref} = maps:get(A, (sys:get_state(Local))#state.nodes_map), + {ok, Data} = rpc:call(A, syn_registry, get_local_data, [sys:get_state(Remote)]), + Local ! {'3.5', ack_sync, Remote, Data}, + Local ! {'DOWN', Ref, process, Remote, noconnection}, + wait(fun() -> S = sys:get_state(Local), + case maps:get(A, S#state.nodes_map, undefined) of + {Remote, NewRef} when NewRef =/= Ref -> view(node()) =:= {P, meta}; + _ -> false + end end), + ct:pal("Late noconnection after snapshot purged and rediscovered current peer"). + +%% snapshots replace what a node had from a peer, registrations set aside come and go with them, and messages from +%% a scope process the node doesn't know are ignored, in the registry and in process groups +snapshot_aside_and_fences(Config) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + PA = pid(A), PA2 = pid(A), PB = pid(B), + SA = scope(A, syn_registry), SB = scope(B, syn_registry), Local = scope(node(), syn_registry), + Local ! {'3.5', ack_sync, SA, [{x, PA, old, 50}]}, + Local ! {'3.5', ack_sync, SB, [{x, PB, b, 100}]}, + Local ! {'3.5', ack_sync, SA, [{x, PA2, new, 10}]}, + _ = sys:get_state(Local), + [{x, A, PA2, new, 10}] = read_aside(Local), + {PB, b} = view(node()), + Local ! {'3.5', ack_sync, SB, []}, + _ = sys:get_state(Local), + {PA2, new} = view(node()), [] = read_aside(Local), + Fake = pid(A), + Local ! {'3.5', sync_register, Fake, x, PA, stale, 200, normal}, + Local ! {'3.5', sync_unregister, Fake, x, PA2, new, normal}, + Local ! {'3.5', sync_register_reply, Fake, x, PA, stale, 200, undefined}, + Local ! {'3.5', sync_unregister_reply, Fake, x, PA2, new, undefined}, + _ = sys:get_state(Local), {PA2, new} = view(node()), + Local ! {'3.5', ack_sync, SA, []}, + _ = sys:get_state(Local), undefined = view(node()), [] = read_aside(Local), + %% process groups ignore messages from a scope process they don't know in the same way + PG = scope(node(), syn_pg), PGA = scope(A, syn_pg), + PG ! {'3.5', ack_sync, PGA, [{g, PA, meta, 10}]}, + PG ! {'3.5', sync_join, Fake, g, PA, stale, 200, normal}, + PG ! {'3.5', sync_leave, Fake, g, PA, meta, normal}, + PG ! {'3.5', sync_join_reply, Fake, g, PA, stale, 200, undefined}, + PG ! {'3.5', sync_leave_reply, Fake, g, PA, meta, undefined}, + _ = sys:get_state(PG), [{PA, meta}] = syn:members(protocol, g). +read_aside(Local) -> + S = sys:replace_state(Local, fun(St) -> + put(protocol_aside, ets:tab2list(maps:get(aside, St#state.handler_state))), St end), + {dictionary, D} = rpc:call(node(Local), erlang, process_info, [Local, dictionary]), + true = is_map(S#state.handler_state), proplists:get_value(protocol_aside, D). + +%% a scope process restart forgets the registrations set aside, keeps the node's own ones, and watches their +%% processes once each +restart_claims(Config) -> + A = proplists:get_value(a, Config), Local = scope(node(), syn_registry), P = pid(node()), Q = pid(A), + syn:set_event_handler(?MODULE), + ok = syn:register(protocol, x, P, 1), ok = syn:register(protocol, y, P, 1), + Remote = scope(A, syn_registry), + Local ! {'3.5', sync_register, Remote, x, Q, 2, 0, normal}, + _ = sys:get_state(Local), [_] = read_aside(Local), + OldAside = maps:get(aside, (sys:get_state(Local))#state.handler_state), + exit(Local, kill), + wait(fun() -> case scope(node(), syn_registry) of New0 when is_pid(New0) -> New0 =/= Local; _ -> false end end), + New = scope(node(), syn_registry), + _ = sys:get_state(New), undefined = ets:info(OldAside), + [] = read_aside(New), {P, 1} = view(node()), {P, 1} = syn:lookup(protocol, y), + {monitors, Mons} = process_info(New, monitors), + 1 = length([ok || {process, MP} <- Mons, MP =:= P]), + exit(P, kill), + wait(fun() -> syn:lookup(protocol, x) =:= undefined andalso syn:lookup(protocol, y) =:= undefined + andalso syn:registry_count(protocol) =:= 0 end), + ct:pal("Restart discarded aside, retained own aliases, rebuilt one monitor, and cleaned both on death"). + +%% one snapshot brings newer registrations for both names of a process: it loses both conflicts, so both names go with +%% syn_conflict_resolution, even though the first one already killed it +killed_process_loses_both_names(Config) -> + A = proplists:get_value(a, Config), P = pid(node()), Q = pid(A), + syn:set_event_handler(syn_test_event_handler_protocol), + application:set_env(syn, protocol_observer, self()), + ok = syn:register(protocol, x, P, local), ok = syn:register(protocol, y, P, local), collect(), + Local = scope(node(), syn_registry), Remote = scope(A, syn_registry), + Local ! {'3.5', ack_sync, Remote, [{x, Q, remote, future()}, {y, Q, remote, future()}]}, + _ = sys:get_state(Local), + Events = collect(), + Reasons = [{Name, Reason} || {unregistered, _, protocol, Name, LP, _, Reason} <- Events, LP =:= P], + ct:pal("Local alias unregister reasons: ~p", [Reasons]), + [{x, syn_conflict_resolution}, {y, syn_conflict_resolution}] = Reasons. + +%% the resolver runs again only for a registration that changed +unchanged_pairs_not_resolved(Config) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + P = pid(node()), Q = pid(A), R = pid(B), + syn:set_event_handler(?MODULE), application:set_env(syn, protocol_observer, self()), + ok = syn:register(protocol, x, P, 0), collect(), + Local = scope(node(), syn_registry), SA = scope(A, syn_registry), SB = scope(B, syn_registry), + Local ! {'3.5', sync_register, SA, x, Q, 1, 10, normal}, + _ = sys:get_state(Local), [_] = [E || E = {resolved, _, _, _, _} <- collect()], + Local ! {'3.5', sync_register, SB, x, R, 2, 20, normal}, + _ = sys:get_state(Local), + NewPairCalls = [E || E = {resolved, _, _, _, _} <- collect()], + Local ! {'3.5', ack_sync, SA, [{x, Q, 1, 10}]}, + _ = sys:get_state(Local), + DuplicateCalls = [E || E = {resolved, _, _, _, _} <- collect()], + ct:pal("Adding B called resolver ~p times; identical A snapshot called it ~p times", [length(NewPairCalls), length(DuplicateCalls)]), + {1, 0} = {length(NewPairCalls), length(DuplicateCalls)}. + +%% the connection drops while a node is busy with a new peer scope process: the node doesn't connect them again +monitor_does_not_reconnect(Config) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + P = pid(A), Fake = pid(A), + ok = syn:register(protocol, x, P, meta), + syn:set_event_handler(syn_test_event_handler_protocol), + application:set_env(syn, protocol_purge_gate, self()), + Local = scope(node(), syn_registry), + Local ! {'3.5', discover, Fake}, + receive {purge_blocked, Local} -> ok after 5000 -> ct:fail(no_purge) end, + %% cut the connection through B, so that this node doesn't talk to A again after it + rpc:call(B, rpc, call, [A, erlang, disconnect_node, [node()]]), + erlang:disconnect_node(A), + false = lists:member(A, nodes()), + Local ! protocol_continue, + _ = sys:get_state(Local), + timer:sleep(200), + Connected = lists:member(A, nodes()), + ct:pal("Connection after adopting the new pid across a disconnect: ~p", [Connected]), + false = Connected. + +%% a node learns of a scope process through a call from it: it sends its data and asks for that node's +ordered_adoption_fetches_snapshot(Config) -> + A = proplists:get_value(a, Config), P = pid(node()), Q = pid(A), + ok = syn:register(protocol, owned_here, P, local), + wait(fun() -> rpc:call(A, syn, lookup, [protocol, owned_here]) =:= {P, local} end), + LocalNode = node(), Remote = scope(A, syn_registry), + sys:replace_state(Remote, fun(S) -> + {_, Ref} = maps:get(LocalNode, S#state.nodes_map), erlang:demonitor(Ref, [flush]), + syn_registry:purge_local_data_for_node(LocalNode, S), + S#state{nodes_map = maps:remove(LocalNode, S#state.nodes_map), + asked = lists:delete(LocalNode, S#state.asked)} + end), + undefined = rpc:call(A, syn, lookup, [protocol, owned_here]), + ok = syn:register(protocol, owned_there, Q, remote), + wait(fun() -> rpc:call(A, syn, lookup, [protocol, owned_here]) =:= {P, local} end), + {Q, remote} = syn:lookup(protocol, owned_there). + +%% a late handshake from a scope process that has restarted since: the node ends up with the live one +late_old_incarnation_recovers(Config) -> + A = proplists:get_value(a, Config), P = pid(A), + ok = syn:register(protocol, x, P, meta), + [begin + Local = scope(node(), Handler), Old = scope(A, Handler), + exit(Old, kill), + wait(fun() -> case scope(A, Handler) of N when is_pid(N), N =/= Old -> true; _ -> false end end), + New = scope(A, Handler), + wait(fun() -> case maps:get(A, (sys:get_state(Local))#state.nodes_map, undefined) of + {New, _} -> true; _ -> false end end), + Local ! {'3.5', ack_sync, Old, []}, + _ = sys:get_state(Local), + wait(fun() -> case maps:get(A, (sys:get_state(Local))#state.nodes_map, undefined) of + {New, _} -> true; _ -> false end end) + end || Handler <- [syn_registry, syn_pg]], + wait(fun() -> view(node()) =:= {P, meta} end). + +%% handshakes from a node that isn't connected are ignored, and the nodes sync once connected. A callback for the stale +%% registration would show it was applied, even if the DOWN of a monitor on the disconnected node then purged it +disconnected_handshakes_ignored(Config) -> + A = proplists:get_value(a, Config), B = proplists:get_value(b, Config), + SA = scope(A, syn_registry), SB = scope(B, syn_registry), Q = pid(B), + ok = rpc:call(A, syn, set_event_handler, [syn_test_event_handler_protocol]), + ok = rpc:call(A, application, set_env, [syn, protocol_observer, self()]), + split(A, B), + SA ! {'3.5', discover, SB}, + SA ! {'3.5', ack_sync, SB, [{x, Q, stale, 1}]}, + %% the callbacks run in SA and report before it replies + _ = sys:get_state(SA), + receive {registered, A, protocol, x, Q, stale, _} = Event -> ct:fail({applied, Event}) after 0 -> ok end, + false = maps:is_key(B, (sys:get_state(SA))#state.nodes_map), + undefined = view(A), + false = lists:member(B, rpc:call(A, erlang, nodes, [])), + true = rpc:call(A, net_kernel, connect_node, [B]), + wait(fun() -> maps:is_key(B, (sys:get_state(SA))#state.nodes_map) end). + +%% a node running Syn 3.4 sends its messages in the format of its own protocol, tagged '3.0': the scope processes ignore +%% them, so the two nodes don't sync, and a call gets undefined back +messages_from_another_version_ignored(Config) -> + A = proplists:get_value(a, Config), + Old = pid(A), P = pid(A), Time = erlang:system_time(), + Registry = scope(node(), syn_registry), Pg = scope(node(), syn_pg), + RegistryNodes = (sys:get_state(Registry))#state.nodes_map, + PgNodes = (sys:get_state(Pg))#state.nodes_map, + Registry ! {'3.0', discover, Old}, + Registry ! {'3.0', ack_sync, Old, [{y, P, meta, Time}]}, + Registry ! {'3.0', sync_register, x, P, meta, Time, normal}, + Pg ! {'3.0', discover, Old}, + Pg ! {'3.0', ack_sync, Old, [{g, P, meta, Time}]}, + Pg ! {'3.0', sync_join, g, P, meta, Time, normal}, + undefined = gen_server:call(Registry, {'3.0', register_or_update_on_node, A, z, pid(node()), meta}), + undefined = gen_server:call(Registry, {'3.0', subcluster_nodes}), + RegistryNodes = (sys:get_state(Registry))#state.nodes_map, + PgNodes = (sys:get_state(Pg))#state.nodes_map, + undefined = syn:lookup(protocol, x), undefined = syn:lookup(protocol, y), undefined = syn:lookup(protocol, z), + [] = syn:members(protocol, g). + +%% a node that connects without running the scope never answers the request for its data: once it goes, the scope +%% processes stop waiting for that answer, so short-lived nodes don't pile up +asked_forgets_nodes_that_go(_Config) -> + Asked = fun(Handler) -> (sys:get_state(scope(node(), Handler)))#state.asked end, + {ok, Peer, Node} = peer:start(#{name => syn_protocol_c, connection => 0, args => ["-connect_all", "false"]}), + true = net_kernel:connect_node(Node), + wait(fun() -> lists:member(Node, Asked(syn_registry)) andalso lists:member(Node, Asked(syn_pg)) end), + peer:stop(Peer), + wait(fun() -> not lists:member(Node, Asked(syn_registry)) andalso not lists:member(Node, Asked(syn_pg)) end). + +%% a node that stops reading from its connections, for example in a long pause, holds up only what is sent to it: the +%% results of calls for processes of the node that sends to it still come back right away. X gets a tiny distribution +%% buffer, so that what it sends to the stopped node N soon makes that connection busy +stalled_node_holds_up_only_itself(_Config) -> + Start = fun(Name, Extra) -> + {ok, Peer, Node} = peer:start(#{name => Name, connection => 0, + args => ["-connect_all", "false", "-kernel", "dist_auto_connect", "never"] ++ Extra}), + ok = rpc:call(Node, code, add_pathsa, [code:get_path()]), + true = net_kernel:connect_node(Node), + {Peer, Node} + end, + {PeerX, X} = Start(syn_protocol_x, ["+zdbbl", "1"]), + {PeerN, N} = Start(syn_protocol_n, []), + NOsPid = rpc:call(N, os, getpid, []), + %% resumes and stops the nodes even if the test process gets killed, for example by its timetrap + Test = self(), + Cleanup = fun() -> os:cmd("kill -CONT " ++ NOsPid), catch peer:stop(PeerN), catch peer:stop(PeerX), ok end, + Guard = spawn(fun() -> + MRef = monitor(process, Test), + receive {'DOWN', MRef, process, Test, _} -> Cleanup(); done -> ok end + end), + try + true = rpc:call(X, net_kernel, connect_node, [N]), + [begin ok = rpc:call(Node, syn, start, []), ok = rpc:call(Node, syn, add_node_to_scopes, [[stalled]]) end + || Node <- [node(), X, N]], + wait(fun() -> lists:sort(syn:subcluster_nodes(registry, stalled)) =:= lists:sort([X, N]) + andalso lists:sort(rpc:call(X, syn, subcluster_nodes, [registry, stalled])) =:= lists:sort([node(), N]) + andalso lists:sort(syn:subcluster_nodes(pg, stalled)) =:= lists:sort([X, N]) end), + "" = os:cmd("kill -STOP " ++ NOsPid), + %% everything X registers goes to N too, and piles up on that connection + ok = rpc:call(X, ?MODULE, fill, [stalled, 400, 64 * 1024], 20000), + P = pid(X), + {RegisterTime, ok} = timer:tc(fun() -> syn:register(stalled, during, P, meta) end), + {JoinTime, ok} = timer:tc(fun() -> syn:join(stalled, group, P, meta) end), + ct:pal("calls during the stall: register ~p ms, join ~p ms", [RegisterTime div 1000, JoinTime div 1000]), + {P, meta} = syn:lookup(stalled, during), + [{P, meta}] = syn:members(stalled, group) + after + Guard ! done, + Cleanup() + end. + +%% registers Count processes of this node, with metadata of Size bytes +fill(Scope, Count, Size) -> + Meta = binary:copy(<<"x">>, Size), + [ok = syn:register(Scope, {fill, I}, spawn(?MODULE, idle, []), Meta) || I <- lists:seq(1, Count)], + ok. diff --git a/test/syn_registry_SUITE.erl b/test/syn_registry_SUITE.erl index 576e6b03..6dab0604 100644 --- a/test/syn_registry_SUITE.erl +++ b/test/syn_registry_SUITE.erl @@ -60,12 +60,13 @@ three_nodes_remote_register_from_scope_process/1, three_nodes_remote_register_from_another_scope_process/1, three_nodes_remote_register_with_undiscovered_node/1, + three_nodes_remote_register_from_hidden_node/1, three_nodes_remote_register_loses_conflict_resolved_elsewhere/1, three_nodes_ignored_registration_shown_when_newer_one_goes/1, - three_nodes_replaced_registration_shown_when_newer_one_goes/1, three_nodes_kept_aside_registration_dropped_by_its_node/1, - three_nodes_kept_aside_registration_shown_when_other_node_goes/1, - three_nodes_legacy_node_gets_direct_calls/1, + three_nodes_registration_set_aside_takes_over_with_reason/1, + three_nodes_remote_register_while_other_registration_shown/1, + three_nodes_resolution_kills_process_with_two_names/1, three_nodes_clock_going_backwards/1, three_nodes_ack_sync_from_unknown_node_keeps_remote/1, three_nodes_ack_sync_from_unknown_node_resolution_reaches_peer/1, @@ -141,12 +142,13 @@ groups() -> three_nodes_remote_register_from_scope_process, three_nodes_remote_register_from_another_scope_process, three_nodes_remote_register_with_undiscovered_node, + three_nodes_remote_register_from_hidden_node, three_nodes_remote_register_loses_conflict_resolved_elsewhere, three_nodes_ignored_registration_shown_when_newer_one_goes, - three_nodes_replaced_registration_shown_when_newer_one_goes, three_nodes_kept_aside_registration_dropped_by_its_node, - three_nodes_kept_aside_registration_shown_when_other_node_goes, - three_nodes_legacy_node_gets_direct_calls, + three_nodes_registration_set_aside_takes_over_with_reason, + three_nodes_remote_register_while_other_registration_shown, + three_nodes_resolution_kills_process_with_two_names, three_nodes_clock_going_backwards, three_nodes_ack_sync_from_unknown_node_keeps_remote, three_nodes_ack_sync_from_unknown_node_resolution_reaches_peer, @@ -195,7 +197,7 @@ do_init_per_group(TestGroup, Count, Config) -> case syn_test_suite_helper:init_cluster(Count) of {error_initializing_cluster, Other} -> end_per_group(TestGroup, Config), - {skip, Other}; + ct:fail({could_not_init_cluster, Other}); NodesConfig -> NodesConfig ++ Config @@ -232,7 +234,8 @@ init_per_testcase(TestCase, Config) -> %% Config0 = Config1 = [tuple()] %% Reason = any() %% ------------------------------------------------------------------- -end_per_testcase(_, _Config) -> +end_per_testcase(_, Config) -> + syn_test_suite_helper:reconnect_cluster(Config), syn_test_suite_helper:clean_after_test(). %% =================================================================== @@ -692,7 +695,7 @@ three_nodes_register_filter_unknown_node(Config) -> %% send sync message from out of scope node InvalidPid = syn_test_suite_helper:start_process(), - {syn_registry_scope_bc, SlaveNode1} ! {'3.0', sync_register, <<"proc-name">>, InvalidPid, undefined, os:system_time(millisecond), normal}, + {syn_registry_scope_bc, SlaveNode1} ! {'3.5', sync_register, self(), <<"proc-name">>, InvalidPid, undefined, os:system_time(millisecond), normal}, %% check undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, <<"proc-name">>]). @@ -975,8 +978,10 @@ three_nodes_cluster_conflicts(Config) -> %% --> conflict by netsplit ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn1, "meta-1"]), + syn_test_suite_helper:wait_clock_after(SlaveNode2, SlaveNode1), ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn2, "meta-2"]), ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn1, "meta-1"]), + syn_test_suite_helper:wait_clock_after(SlaveNode2, SlaveNode1), ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn2, "meta-2"]), %% re-join @@ -1037,29 +1042,6 @@ three_nodes_cluster_conflicts(Config) -> fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2RemoteOn2]) end ), - %% --> conflict by race condition - PidOnMaster = syn_test_suite_helper:start_process(), - PidOn1 = syn_test_suite_helper:start_process(SlaveNode1), - rpc:call(SlaveNode1, syn_registry, add_to_local_table, - [default, <<"my proc">>, PidOn1, "meta-2", erlang:system_time(), undefined] - ), - ok = syn:register(scope_all, <<"my proc">>, PidOnMaster, "meta-1"), - - %% retrieve - syn_test_suite_helper:assert_wait( - {PidOnMaster, "meta-1"}, - fun() -> syn:lookup(scope_all, <<"my proc">>) end - ), - syn_test_suite_helper:assert_wait( - {PidOnMaster, "meta-1"}, - fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, <<"my proc">>]) end - ), - syn_test_suite_helper:assert_wait( - {PidOnMaster, "meta-1"}, - fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, <<"my proc">>]) end - ), - %% NB: we can't check for process alive here because we injected the conflicting process in the DB - %% -> it's not actually monitored ok. three_nodes_custom_event_handler_reg_unreg(Config) -> @@ -1223,6 +1205,7 @@ three_nodes_custom_event_handler_reg_unreg(Config) -> Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2), ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict", Pid2RemoteOn1, {recipient, self(), <<"meta-1">>}]), + syn_test_suite_helper:wait_clock_after(SlaveNode2, SlaveNode1), ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict", Pid2RemoteOn2, {recipient, self(), <<"meta-2">>}]), %% check callbacks called @@ -1328,8 +1311,10 @@ three_nodes_custom_event_handler_conflict_resolution(Config) -> %% --> conflict by netsplit ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn1, {recipient, TestPid, keepthis}]), + syn_test_suite_helper:wait_clock_after(SlaveNode2, SlaveNode1), ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn2, {recipient, TestPid, "meta-2"}]), ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, {recipient, TestPid, keepthis}]), + syn_test_suite_helper:wait_clock_after(SlaveNode2, SlaveNode1), ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, {recipient, TestPid, "meta-2"}]), %% check callbacks @@ -1456,6 +1441,7 @@ three_nodes_custom_event_handler_conflict_resolution(Config) -> %% register with meta with no 'keepthis' element ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, {recipient, TestPid, "meta-1"}]), + syn_test_suite_helper:wait_clock_after(SlaveNode2, SlaveNode1), ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, {recipient, TestPid, "meta-2"}]), %% check callbacks @@ -1501,11 +1487,9 @@ three_nodes_custom_event_handler_conflict_resolution(Config) -> 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]), %% check callbacks - syn_test_suite_helper:assert_received_messages([ - {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution}, - {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", syn_conflict_resolution}, - {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution} - ]), + assert_keep_neither_callbacks( + SlaveNode1, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", PidOn2, "meta-2" + ), %% process alive (discarded process does not get killed with a custom handler) syn_test_suite_helper:assert_wait( @@ -1527,6 +1511,7 @@ three_nodes_custom_event_handler_conflict_resolution(Config) -> %% register with meta with no 'crash' element ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, {recipient, TestPid, crash}]), + syn_test_suite_helper:wait_clock_after(SlaveNode2, SlaveNode1), ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, {recipient, TestPid, crash}]), %% check callbacks @@ -1572,11 +1557,9 @@ three_nodes_custom_event_handler_conflict_resolution(Config) -> 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]), %% check callbacks - syn_test_suite_helper:assert_received_messages([ - {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution}, - {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, syn_conflict_resolution}, - {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution} - ]), + assert_keep_neither_callbacks( + SlaveNode1, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, PidOn2, crash + ), %% process alive (discarded process does not get killed with a custom handler) syn_test_suite_helper:assert_wait( @@ -1739,20 +1722,10 @@ three_nodes_update(Config) -> ]). three_nodes_ack_sync_ordered_delivery(Config) -> - %% Issue #87: Race condition causing stale pids in syn lookup. - %% - %% Root cause: ack_sync was sent directly from gen_server while sync_register/ - %% sync_unregister was sent via sender_loop. Since these are different sender - %% processes, Erlang does not guarantee message ordering between them. This means - %% sync_register can arrive at a remote node BEFORE ack_sync, causing it to be - %% dropped (remote node not yet in nodes_map), leaving stale data. - %% - %% Fix: All messages to remote nodes are now routed through sender_loop, - %% guaranteeing FIFO delivery. - %% - %% This test verifies the fix by tracing the local sender_loop process during - %% node discovery. With the fix, ack_sync is routed through sender_loop and the - %% trace captures {send_single, ...} messages. + %% Issue #87. Everything a scope process sends to another node goes through its sender process, ack_sync included: + %% Erlang only keeps the order of messages from the same process, and a sync_register that arrived before the + %% ack_sync would be dropped, as its node isn't known yet, leaving stale data. The test traces the sender during + %% discovery and checks that the ack_sync goes through it. %% get slave SlaveNode1 = proplists:get_value(syn_slave_1, Config), @@ -1782,9 +1755,7 @@ three_nodes_ack_sync_ordered_delivery(Config) -> syn_test_suite_helper:connect_node(SlaveNode1), syn_test_suite_helper:assert_scope_subcluster(registry,node(), scope_order, [SlaveNode1]), - %% verify that sender_loop received a {send_single, ...} message containing ack_sync. - %% this proves ack_sync is routed through sender_loop (the fix), ensuring FIFO - %% ordering with sync_register/sync_unregister broadcasts. + %% the sender got the ack_sync to send, so it goes out in order with the sync_register and sync_unregister messages true = check_trace_for_ack_sync_via_sender(), %% stop tracing after the check: the node can show up in the subcluster @@ -1827,7 +1798,7 @@ three_nodes_ack_sync_reconciles_registry_snapshot(Config) -> RemoteScopePid = rpc:call(SlaveNode1, erlang, whereis, [syn_registry_scope_reconcile]), true = is_pid(RemoteScopePid), - syn_registry_scope_reconcile ! {'3.0', ack_sync, RemoteScopePid, RemoteSnapshot}, + syn_registry_scope_reconcile ! {'3.5', ack_sync, RemoteScopePid, RemoteSnapshot}, syn_test_suite_helper:assert_wait( undefined, @@ -1910,7 +1881,7 @@ three_nodes_ack_sync_reconciles_large_snapshot(Config) -> ScopePid = whereis(syn_registry_scope_reconcile_large), {reductions, ReductionsBefore} = erlang:process_info(ScopePid, reductions), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, syn_test_suite_helper:assert_wait( {LastPid, applied_meta}, @@ -1947,7 +1918,7 @@ three_nodes_stale_snapshot_does_not_undo_remote_register(Config) -> %% hold the slave's snapshot back in this node's scope mailbox, then register {RemoteScopePid, Snapshot} = get_remote_snapshot(SlaveNode1, syn_registry_scope_stale), ok = sys:suspend(ScopePid), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, CallRef = syn_test_suite_helper:run_async(fun() -> syn:register(scope_stale, "proc", Pid, meta) end), %% the slave has it, the call waits for our scope @@ -1977,7 +1948,7 @@ three_nodes_stale_snapshot_does_not_undo_remote_update(Config) -> {RemoteScopePid, Snapshot} = get_remote_snapshot(SlaveNode1, syn_registry_scope_stale), [{"proc", Pid, meta, _}] = Snapshot, ok = sys:suspend(ScopePid), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, CallRef = syn_test_suite_helper:run_async(fun() -> syn:update_registry(scope_stale, "proc", fun(_, meta) -> new_meta end) end), @@ -2004,7 +1975,7 @@ three_nodes_stale_snapshot_does_not_undo_remote_unregister(Config) -> {RemoteScopePid, Snapshot} = get_remote_snapshot(SlaveNode1, syn_registry_scope_stale), [{"proc", Pid, meta, _}] = Snapshot, ok = sys:suspend(ScopePid), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, CallRef = syn_test_suite_helper:run_async(fun() -> syn:unregister(scope_stale, "proc") end), wait_for_message_queue_len(ScopePid, 2), syn_test_suite_helper:assert_async_waiting(CallRef), @@ -2016,8 +1987,8 @@ three_nodes_stale_snapshot_does_not_undo_remote_unregister(Config) -> undefined = rpc:call(SlaveNode1, syn, lookup, [scope_stale, "proc"]). three_nodes_late_unregister_does_not_undo_remote_register(Config) -> - %% the slave unregisters its process and we register it again right after: the unregister the slave sent - %% before our call reaches our scope before the registration, so it can't remove it + %% the slave unregisters its process and we register it again right after: the slave sends the result of our + %% registration after the unregister it sent before, so the unregister reaches our scope first and can't remove it SlaveNode1 = proplists:get_value(syn_slave_1, Config), ScopePid = start_scope_stale(SlaveNode1), @@ -2025,14 +1996,16 @@ three_nodes_late_unregister_does_not_undo_remote_register(Config) -> Pid = syn_test_suite_helper:start_process(SlaveNode1), ok = syn:register(scope_stale, "proc", Pid, meta), - %% the slave's unregister waits in our scope's queue while we register again - ok = sys:suspend(ScopePid), + %% the slave's sender holds its unregister, then the registration's broadcast to the other nodes and its result + %% for us + SlaveScopePid = rpc:call(SlaveNode1, erlang, whereis, [syn_registry_scope_stale]), + Sender = (rpc:call(SlaveNode1, sys, get_state, [SlaveScopePid]))#state.sender_pid, + Resume = syn_test_suite_helper:pause_process(Sender), ok = rpc:call(SlaveNode1, syn, unregister, [scope_stale, "proc"]), - wait_for_message_queue_len(ScopePid, 1), CallRef = syn_test_suite_helper:run_async(fun() -> syn:register(scope_stale, "proc", Pid, new_meta) end), - wait_for_message_queue_len(ScopePid, 2), + wait_for_message_queue_len(Sender, 3), syn_test_suite_helper:assert_async_waiting(CallRef), - ok = sys:resume(ScopePid), + Resume(), ok = syn_test_suite_helper:wait_async(CallRef), _ = sys:get_state(ScopePid), @@ -2056,7 +2029,7 @@ three_nodes_scope_down_does_not_undo_remote_register(Config) -> wait_for_message_queue_len(ScopePid, 2), %% and sends us its snapshot, without the name RemoteScopePid = rpc:call(SlaveNode1, erlang, whereis, [syn_registry_scope_stale]), - RemoteScopePid ! {'3.0', discover, ScopePid}, + RemoteScopePid ! {'3.5', discover, ScopePid}, wait_for_message_queue_len(ScopePid, 3), CallRef = syn_test_suite_helper:run_async(fun() -> syn:register(scope_stale, "proc", Pid, meta) end), wait_for_message_queue_len(ScopePid, 4), @@ -2145,7 +2118,7 @@ three_nodes_remote_register_from_scope_process(Config) -> %% an older snapshot of the slave waits in the queue of our scope process, then it unregisters {RemoteScopePid, Snapshot} = get_remote_snapshot(SlaveNode1, syn_registry_scope_stale), ok = sys:suspend(ScopePid), - ScopePid ! {'3.0', ack_sync, RemoteScopePid, Snapshot}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, Snapshot}, _ = sys:replace_state(ScopePid, fun(State) -> TestPid ! {unregistered, syn:unregister(scope_stale, "proc")}, State @@ -2191,8 +2164,8 @@ three_nodes_remote_register_with_undiscovered_node(Config) -> Pid = syn_test_suite_helper:start_process(SlaveNode1), %% the scope processes forget each other - RemoteScopePid ! {'DOWN', make_ref(), process, ScopePid, test}, - ScopePid ! {'DOWN', make_ref(), process, RemoteScopePid, test}, + forget_scope(SlaveNode1, RemoteScopePid, node()), + forget_scope(node(), ScopePid, SlaveNode1), syn_test_suite_helper:assert_scope_subcluster(registry, node(), scope_stale, []), syn_test_suite_helper:assert_scope_subcluster(registry, SlaveNode1, scope_stale, []), @@ -2202,6 +2175,27 @@ three_nodes_remote_register_with_undiscovered_node(Config) -> syn_test_suite_helper:assert_scope_subcluster(registry, SlaveNode1, scope_stale, [node()]), {Pid, meta} = rpc:call(SlaveNode1, syn, lookup, [scope_stale, "proc"]). +three_nodes_remote_register_from_hidden_node(_Config) -> + %% a hidden node takes no part in syncing: its call for a process of ours works, but the two nodes don't take each + %% other in, so neither shows what the other registers + + ok = syn:start(), + ok = syn:add_node_to_scopes([scope_hidden]), + {PeerPid, HiddenNode} = start_hidden_node(syn_hidden_registry, scope_hidden), + try + Pid = syn_test_suite_helper:start_process(), + ok = rpc:call(HiddenNode, syn, register, [scope_hidden, "proc", Pid, meta]), + {Pid, meta} = syn:lookup(scope_hidden, "proc"), + HiddenPid = syn_test_suite_helper:start_process(HiddenNode), + ok = rpc:call(HiddenNode, syn, register, [scope_hidden, "hidden-proc", HiddenPid, meta]), + [] = syn:subcluster_nodes(registry, scope_hidden), + [] = rpc:call(HiddenNode, syn, subcluster_nodes, [registry, scope_hidden]), + undefined = rpc:call(HiddenNode, syn, lookup, [scope_hidden, "proc"]), + undefined = syn:lookup(scope_hidden, "hidden-proc") + after + peer:stop(PeerPid) + end. + three_nodes_remote_register_loses_conflict_resolved_elsewhere(Config) -> %% slave 1 registers the name for us on its process while slave 2 registers it on its own: slave 2 resolves %% the conflict and its result reaches us before the reply of slave 1, which then unregisters its process. @@ -2220,6 +2214,8 @@ three_nodes_remote_register_loses_conflict_resolved_elsewhere(Config) -> syn_test_suite_helper:assert_scope_subcluster(registry, SlaveNode1, scope_all, [node(), SlaveNode2]), syn_test_suite_helper:assert_scope_subcluster(registry, SlaveNode2, scope_all, [node(), SlaveNode1]), ScopePid = whereis(syn_registry_scope_all), + RemoteScopePid1 = rpc:call(SlaveNode1, erlang, whereis, [syn_registry_scope_all]), + RemoteScopePid2 = rpc:call(SlaveNode2, erlang, whereis, [syn_registry_scope_all]), PidOn1 = syn_test_suite_helper:start_process(SlaveNode1), PidOn2 = syn_test_suite_helper:start_process(SlaveNode2), @@ -2227,10 +2223,10 @@ three_nodes_remote_register_loses_conflict_resolved_elsewhere(Config) -> %% the resolution of slave 2, the reply of slave 1, and the unregister of its process, in this order ok = sys:suspend(ScopePid), Later = erlang:system_time() + erlang:convert_time_unit(60, second, native), - ScopePid ! {'3.0', sync_register, "proc", PidOn2, on_2, Later, syn_conflict_resolution}, + ScopePid ! {'3.5', sync_register, RemoteScopePid2, "proc", PidOn2, on_2, Later, syn_conflict_resolution}, CallRef = syn_test_suite_helper:run_async(fun() -> syn:register(scope_all, "proc", PidOn1, on_1) end), wait_for_message_queue_len(ScopePid, 2), - ScopePid ! {'3.0', sync_unregister, "proc", PidOn1, on_1, syn_conflict_resolution}, + ScopePid ! {'3.5', sync_unregister, RemoteScopePid1, "proc", PidOn1, on_1, syn_conflict_resolution}, ok = sys:resume(ScopePid), {error, race_condition} = syn_test_suite_helper:wait_async(CallRef), @@ -2244,52 +2240,31 @@ three_nodes_ignored_registration_shown_when_newer_one_goes(Config) -> SlaveNode1 = proplists:get_value(syn_slave_1, Config), SlaveNode2 = proplists:get_value(syn_slave_2, Config), ScopePid = start_scope_all(SlaveNode1, SlaveNode2), + RemoteScopePid1 = rpc:call(SlaveNode1, erlang, whereis, [syn_registry_scope_all]), + RemoteScopePid2 = rpc:call(SlaveNode2, erlang, whereis, [syn_registry_scope_all]), PidOn1 = syn_test_suite_helper:start_process(SlaveNode1), PidOn2 = syn_test_suite_helper:start_process(SlaveNode2), - ScopePid ! {'3.0', sync_register, "proc", PidOn2, on_2, later_time(), normal}, + ScopePid ! {'3.5', sync_register, RemoteScopePid2, "proc", PidOn2, on_2, later_time(), normal}, ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc", PidOn1, on_1]), wait_for_registry_entry(SlaveNode2, scope_all, "proc", PidOn1), {PidOn2, on_2} = syn:lookup(scope_all, "proc"), - ScopePid ! {'3.0', sync_unregister, "proc", PidOn2, on_2, normal}, + ScopePid ! {'3.5', sync_unregister, RemoteScopePid2, "proc", PidOn2, on_2, normal}, _ = sys:get_state(ScopePid), {PidOn1, on_1} = syn:lookup(scope_all, "proc"), %% same with registrations at the same time {_, _, _, Time, _, _} = get_registry_entry_with_mref(node(), scope_all, "proc"), OtherPidOn2 = syn_test_suite_helper:start_process(SlaveNode2), - ScopePid ! {'3.0', sync_register, "proc-same-time", OtherPidOn2, on_2, Time, normal}, - ScopePid ! {'3.0', sync_register, "proc-same-time", PidOn1, on_1, Time, normal}, + ScopePid ! {'3.5', sync_register, RemoteScopePid2, "proc-same-time", OtherPidOn2, on_2, Time, normal}, + ScopePid ! {'3.5', sync_register, RemoteScopePid1, "proc-same-time", PidOn1, on_1, Time, normal}, _ = sys:get_state(ScopePid), {OtherPidOn2, on_2} = syn:lookup(scope_all, "proc-same-time"), - ScopePid ! {'3.0', sync_unregister, "proc-same-time", OtherPidOn2, on_2, normal}, + ScopePid ! {'3.5', sync_unregister, RemoteScopePid2, "proc-same-time", OtherPidOn2, on_2, normal}, _ = sys:get_state(ScopePid), {PidOn1, on_1} = syn:lookup(scope_all, "proc-same-time"). -three_nodes_replaced_registration_shown_when_newer_one_goes(Config) -> - %% we have the name on a process of slave 1, and slave 2 registers it on its own process with a later time, - %% then drops it before the two resolve the conflict: we show the registration of slave 1 again - - SlaveNode1 = proplists:get_value(syn_slave_1, Config), - SlaveNode2 = proplists:get_value(syn_slave_2, Config), - ScopePid = start_scope_all(SlaveNode1, SlaveNode2), - - PidOn1 = syn_test_suite_helper:start_process(SlaveNode1), - PidOn2 = syn_test_suite_helper:start_process(SlaveNode2), - ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc", PidOn1, on_1]), - syn_test_suite_helper:assert_wait( - {PidOn1, on_1}, - fun() -> syn:lookup(scope_all, "proc") end - ), - ScopePid ! {'3.0', sync_register, "proc", PidOn2, on_2, later_time(), normal}, - _ = sys:get_state(ScopePid), - {PidOn2, on_2} = syn:lookup(scope_all, "proc"), - - ScopePid ! {'3.0', sync_unregister, "proc", PidOn2, on_2, normal}, - _ = sys:get_state(ScopePid), - {PidOn1, on_1} = syn:lookup(scope_all, "proc"). - three_nodes_kept_aside_registration_dropped_by_its_node(Config) -> %% a registration we keep aside goes when its node unregisters it, sends us a snapshot without it, %% or goes down: it doesn't come back when the name's other process goes @@ -2297,13 +2272,14 @@ three_nodes_kept_aside_registration_dropped_by_its_node(Config) -> SlaveNode1 = proplists:get_value(syn_slave_1, Config), SlaveNode2 = proplists:get_value(syn_slave_2, Config), ScopePid = start_scope_all(SlaveNode1, SlaveNode2), + RemoteScopePid2 = rpc:call(SlaveNode2, erlang, whereis, [syn_registry_scope_all]), RemoteScopePid1 = rpc:call(SlaveNode1, erlang, whereis, [syn_registry_scope_all]), PidsOn1 = [syn_test_suite_helper:start_process(SlaveNode1) || _ <- lists:seq(1, 3)], PidOn2 = syn_test_suite_helper:start_process(SlaveNode2), Names = ["proc-unregistered", "proc-not-in-snapshot", "proc-node-down"], lists:foreach(fun({Name, PidOn1}) -> - ScopePid ! {'3.0', sync_register, Name, PidOn2, on_2, later_time(), normal}, + ScopePid ! {'3.5', sync_register, RemoteScopePid2, Name, PidOn2, on_2, later_time(), normal}, ok = rpc:call(SlaveNode1, syn, register, [scope_all, Name, PidOn1, on_1]), wait_for_registry_entry(SlaveNode2, scope_all, Name, PidOn1) end, lists:zip(Names, PidsOn1)), @@ -2311,21 +2287,21 @@ three_nodes_kept_aside_registration_dropped_by_its_node(Config) -> %% unregistered by slave 1 ok = rpc:call(SlaveNode1, syn, unregister, [scope_all, "proc-unregistered"]), wait_for_registry_entry(SlaveNode2, scope_all, "proc-unregistered", undefined), - ScopePid ! {'3.0', sync_unregister, "proc-unregistered", PidOn2, on_2, normal}, + ScopePid ! {'3.5', sync_unregister, RemoteScopePid2, "proc-unregistered", PidOn2, on_2, normal}, _ = sys:get_state(ScopePid), undefined = syn:lookup(scope_all, "proc-unregistered"), %% not in a snapshot of slave 1 {_, Snapshot} = get_remote_snapshot(SlaveNode1, syn_registry_scope_all), - ScopePid ! {'3.0', ack_sync, RemoteScopePid1, lists:keydelete("proc-not-in-snapshot", 1, Snapshot)}, - ScopePid ! {'3.0', sync_unregister, "proc-not-in-snapshot", PidOn2, on_2, normal}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid1, lists:keydelete("proc-not-in-snapshot", 1, Snapshot)}, + ScopePid ! {'3.5', sync_unregister, RemoteScopePid2, "proc-not-in-snapshot", PidOn2, on_2, normal}, _ = sys:get_state(ScopePid), undefined = syn:lookup(scope_all, "proc-not-in-snapshot"), %% slave 1 goes down syn_test_suite_helper:disconnect_node(SlaveNode1), syn_test_suite_helper:assert_scope_subcluster(registry, node(), scope_all, [SlaveNode2]), - ScopePid ! {'3.0', sync_unregister, "proc-node-down", PidOn2, on_2, normal}, + ScopePid ! {'3.5', sync_unregister, RemoteScopePid2, "proc-node-down", PidOn2, on_2, normal}, _ = sys:get_state(ScopePid), undefined = syn:lookup(scope_all, "proc-node-down"), @@ -2333,54 +2309,117 @@ three_nodes_kept_aside_registration_dropped_by_its_node(Config) -> true = syn_test_suite_helper:connect_node(SlaveNode1), syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]). -three_nodes_kept_aside_registration_shown_when_other_node_goes(Config) -> - %% we keep aside the registration of slave 1, and the node of the process we show goes down: we show it +three_nodes_registration_set_aside_takes_over_with_reason(Config) -> + %% we show the name on a process of slave 2 and keep the one of slave 1 aside: when the one of slave 2 goes, the + %% callbacks of the one taking its place get the reason it went with SlaveNode1 = proplists:get_value(syn_slave_1, Config), SlaveNode2 = proplists:get_value(syn_slave_2, Config), + ok = application:set_env(syn, event_handler, syn_test_event_handler_callbacks), ScopePid = start_scope_all(SlaveNode1, SlaveNode2), + RemoteScopePid2 = rpc:call(SlaveNode2, erlang, whereis, [syn_registry_scope_all]), + TestPid = self(), + LocalNode = node(), + Meta1 = {recipient, TestPid, on_1}, + Meta2 = {recipient, TestPid, on_2}, PidOn1 = syn_test_suite_helper:start_process(SlaveNode1), PidOn2 = syn_test_suite_helper:start_process(SlaveNode2), - ScopePid ! {'3.0', sync_register, "proc", PidOn2, on_2, later_time(), normal}, - ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc", PidOn1, on_1]), - wait_for_registry_entry(SlaveNode2, scope_all, "proc", PidOn1), - {PidOn2, on_2} = syn:lookup(scope_all, "proc"), + ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc", PidOn1, Meta1]), + wait_for_registry_entry(node(), scope_all, "proc", PidOn1), + ScopePid ! {'3.5', sync_register, RemoteScopePid2, "proc", PidOn2, Meta2, later_time(), normal}, + _ = sys:get_state(ScopePid), + syn_test_suite_helper:assert_received_messages([ + {on_process_registered, LocalNode, scope_all, "proc", PidOn1, on_1, normal}, + {on_process_unregistered, LocalNode, scope_all, "proc", PidOn1, on_1, normal}, + {on_process_registered, LocalNode, scope_all, "proc", PidOn2, on_2, normal} + ]), + %% the process of slave 2 dies + ScopePid ! {'3.5', sync_unregister, RemoteScopePid2, "proc", PidOn2, Meta2, killed}, + syn_test_suite_helper:assert_received_messages([ + {on_process_unregistered, LocalNode, scope_all, "proc", PidOn2, on_2, killed}, + {on_process_registered, LocalNode, scope_all, "proc", PidOn1, on_1, killed} + ]), + + %% slave 2 goes down + ScopePid ! {'3.5', sync_register, RemoteScopePid2, "proc", PidOn2, Meta2, later_time(), normal}, + syn_test_suite_helper:assert_received_messages([ + {on_process_unregistered, LocalNode, scope_all, "proc", PidOn1, on_1, normal}, + {on_process_registered, LocalNode, scope_all, "proc", PidOn2, on_2, normal} + ]), syn_test_suite_helper:disconnect_node(SlaveNode2), - syn_test_suite_helper:assert_scope_subcluster(registry, node(), scope_all, [SlaveNode1]), - {PidOn1, on_1} = syn:lookup(scope_all, "proc"), + NodeDown = {syn_remote_scope_node_down, scope_all, SlaveNode2}, + syn_test_suite_helper:assert_received_messages([ + {on_process_unregistered, LocalNode, scope_all, "proc", PidOn2, on_2, NodeDown}, + {on_process_registered, LocalNode, scope_all, "proc", PidOn1, on_1, NodeDown} + ]), %% reconnect for the next tests true = syn_test_suite_helper:connect_node(SlaveNode2), syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]). -three_nodes_legacy_node_gets_direct_calls(Config) -> - %% a node running a version that doesn't know the ordered calls gets direct ones, until it disconnects +three_nodes_remote_register_while_other_registration_shown(Config) -> + %% we show the name on a process of slave 2, whose clock is ahead, and register it for a process of slave 1, + %% which has it free: slave 1 registers it, but we keep showing the one of slave 2, so the call returns + %% race_condition. Once slave 2 drops its own, we show the one of slave 1 SlaveNode1 = proplists:get_value(syn_slave_1, Config), - start_scope_stale(SlaveNode1), - TableByName = syn_backbone:get_table_name(syn_registry_by_name, scope_stale), + SlaveNode2 = proplists:get_value(syn_slave_2, Config), + ScopePid = start_scope_all(SlaveNode1, SlaveNode2), + RemoteScopePid2 = rpc:call(SlaveNode2, erlang, whereis, [syn_registry_scope_all]), - Pid = syn_test_suite_helper:start_process(SlaveNode1), + PidOn1 = syn_test_suite_helper:start_process(SlaveNode1), + PidOn2 = syn_test_suite_helper:start_process(SlaveNode2), + ScopePid ! {'3.5', sync_register, RemoteScopePid2, "proc", PidOn2, on_2, later_time(), normal}, + _ = sys:get_state(ScopePid), - %% as if the slave had answered an ordered call with undefined - true = syn_backbone:save_legacy_node(SlaveNode1), - ok = syn:register(scope_stale, "proc", Pid, meta), - [{"proc", Pid, meta, _, caller, SlaveNode1}] = ets:lookup(TableByName, "proc"), - ok = syn:unregister(scope_stale, "proc"), + {error, race_condition} = syn:register(scope_all, "proc", PidOn1, on_1), + {PidOn1, on_1} = rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc"]), + {PidOn2, on_2} = syn:lookup(scope_all, "proc"), - %% forgotten when it disconnects - syn_test_suite_helper:disconnect_node(SlaveNode1), - syn_test_suite_helper:assert_wait( - false, - fun() -> syn_backbone:is_legacy_node(SlaveNode1) end - ), - true = syn_test_suite_helper:connect_node(SlaveNode1), - syn_test_suite_helper:assert_scope_subcluster(registry, node(), scope_stale, [SlaveNode1]), - syn_test_suite_helper:assert_scope_subcluster(registry, SlaveNode1, scope_stale, [node()]), - ok = syn:register(scope_stale, "proc", Pid, meta), - [{"proc", Pid, meta, _, undefined, SlaveNode1}] = ets:lookup(TableByName, "proc"). + ScopePid ! {'3.5', sync_unregister, RemoteScopePid2, "proc", PidOn2, on_2, normal}, + _ = sys:get_state(ScopePid), + {PidOn1, on_1} = syn:lookup(scope_all, "proc"). + +three_nodes_resolution_kills_process_with_two_names(Config) -> + %% a process registered under two names loses a conflict on one of them: the default resolver kills it, and it + %% goes from the other name with its exit reason + + SlaveNode1 = proplists:get_value(syn_slave_1, Config), + SlaveNode2 = proplists:get_value(syn_slave_2, Config), + [ok = rpc:call(Node, application, set_env, [syn, event_handler, syn_test_event_handler_callbacks]) + || Node <- [node(), SlaveNode1, SlaveNode2]], + start_scope_all(SlaveNode1, SlaveNode2), + RemoteScopePid1 = rpc:call(SlaveNode1, erlang, whereis, [syn_registry_scope_all]), + RemoteScopePid2 = rpc:call(SlaveNode2, erlang, whereis, [syn_registry_scope_all]), + TestPid = self(), + LocalNode = node(), + + PidOn1 = syn_test_suite_helper:start_process(SlaveNode1), + PidOn2 = syn_test_suite_helper:start_process(SlaveNode2), + MetaX = {recipient, TestPid, x_1}, + ok = rpc:call(SlaveNode1, syn, register, [scope_all, "x", PidOn1, MetaX]), + ok = rpc:call(SlaveNode1, syn, register, [scope_all, "y", PidOn1, {recipient, TestPid, y_1}]), + syn_test_suite_helper:assert_received_messages([ + {on_process_registered, Node, scope_all, Name, PidOn1, AdditionalMeta, normal} + || Node <- [LocalNode, SlaveNode1, SlaveNode2], {Name, AdditionalMeta} <- [{"x", x_1}, {"y", y_1}] + ]), + + %% slave 1 hears of a newer registration of "x" by slave 2 + RemoteScopePid1 ! {'3.5', sync_register, RemoteScopePid2, "x", PidOn2, {recipient, TestPid, x_2}, later_time(), normal}, + + KillReason = {syn_resolve_kill, "x", MetaX}, + syn_test_suite_helper:assert_received_messages([ + {on_process_unregistered, SlaveNode1, scope_all, "x", PidOn1, x_1, syn_conflict_resolution}, + {on_process_registered, SlaveNode1, scope_all, "x", PidOn2, x_2, syn_conflict_resolution}, + {on_process_unregistered, SlaveNode1, scope_all, "y", PidOn1, y_1, KillReason}, + {on_process_unregistered, LocalNode, scope_all, "x", PidOn1, x_1, syn_conflict_resolution}, + {on_process_unregistered, LocalNode, scope_all, "y", PidOn1, y_1, KillReason}, + {on_process_unregistered, SlaveNode2, scope_all, "x", PidOn1, x_1, syn_conflict_resolution}, + {on_process_unregistered, SlaveNode2, scope_all, "y", PidOn1, y_1, KillReason} + ]), + false = rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]). three_nodes_clock_going_backwards(Config) -> %% times come from the clock of each node, which can step backwards: what a node sends about @@ -2394,27 +2433,26 @@ three_nodes_clock_going_backwards(Config) -> NewPid = syn_test_suite_helper:start_process(SlaveNode1), %% a snapshot with the name on another process, with an older time - ScopePid ! {'3.0', ack_sync, RemoteScopePid, [{"proc", OldPid, old, 200}]}, - ScopePid ! {'3.0', ack_sync, RemoteScopePid, [{"proc", NewPid, new, 100}]}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, [{"proc", OldPid, old, 200}]}, + ScopePid ! {'3.5', ack_sync, RemoteScopePid, [{"proc", NewPid, new, 100}]}, _ = sys:get_state(ScopePid), {NewPid, new} = syn:lookup(scope_stale, "proc"), %% an update of the same process, with an older time - ScopePid ! {'3.0', sync_register, "proc", NewPid, newer, 50, normal}, + ScopePid ! {'3.5', sync_register, RemoteScopePid, "proc", NewPid, newer, 50, normal}, _ = sys:get_state(ScopePid), {NewPid, newer} = syn:lookup(scope_stale, "proc"). three_nodes_ack_sync_from_unknown_node_keeps_remote(Config) -> %% The local node sees both slaves, which cannot see each other. The name is registered - %% on slave 1 first and on slave 2 later, so the local node holds the process on slave 2. + %% on slave 1 first and on slave 2 later, so the local node shows the process on slave 2. %% The resolver keeps the process on slave 1. %% %% When the slaves connect, slave 2 handles the ack_sync of slave 1 while slave 1 is not %% in its nodes_map yet, and slave 1 handles the snapshot of slave 2 as a known node, as %% in a join where slave 1 handles the discover of slave 2 before its own nodeup. - %% Slave 2 keeps the remote process and broadcasts the unregistration of its own process. - %% Slave 1 must get the conflicting entry in the snapshot of slave 2 and resolve the - %% conflict too: only slave 1 sends its process to the local node. + %% Slave 2 keeps the remote process and tells every node it dropped its own; slave 1, + %% which keeps its process, sends nothing. The local node shows the process on slave 1. %% get slaves SlaveNode1 = proplists:get_value(syn_slave_1, Config), @@ -2473,10 +2511,10 @@ three_nodes_ack_sync_from_unknown_node_keeps_remote(Config) -> three_nodes_ack_sync_from_unknown_node_resolution_reaches_peer(Config) -> %% Slave 1 adds a scope in which slave 2 already holds a name, and registers the same %% name before slave 2 answers its discover. Slave 1 then handles the ack_sync of slave 2 - %% while slave 2 is not in its nodes_map, keeps its local process, the later registration, - %% and broadcasts it with a new time. The slaves stay connected, so no nodeup follows and - %% slave 1 handles no other ack_sync: slave 2 gets the new time only through that - %% broadcast. Both slaves must end with the same process and the same time. + %% while slave 2 is not in its nodes_map, and sends slave 2 its own data before resolving. + %% The resolver keeps the later registration, slave 1's: slave 2 drops its process. The + %% slaves stay connected, so no nodeup follows. Both slaves must end with the same + %% process and the same time. %% get slaves SlaveNode1 = proplists:get_value(syn_slave_1, Config), @@ -2500,6 +2538,7 @@ three_nodes_ack_sync_from_unknown_node_resolution_reaches_peer(Config) -> %% register on slave 1 later, then let slave 2 answer the discover PidOn1 = syn_test_suite_helper:start_process(SlaveNode1), + syn_test_suite_helper:wait_clock_after(SlaveNode1, SlaveNode2), ok = rpc:call(SlaveNode1, syn, register, [scope_bc, Name, PidOn1, "meta-1"]), ok = sys:resume(ScopePid2), @@ -2576,11 +2615,14 @@ three_nodes_resolver_keeps_neither_while_process_dies(Config) -> wait_for_message_queue_len(ScopePid1, 3), ok = sys:resume(ScopePid1), + %% slave 2 shows the process of slave 1, which is still registered there, until slave 1 drops it syn_test_suite_helper:assert_received_messages([ {on_process_unregistered, SlaveNode1, scope_bc, "proc", PidOn1, crash, killed}, {on_process_registered, SlaveNode1, scope_bc, "proc", PidOn2, crash, {syn_remote_scope_node_up, scope_bc, SlaveNode2}}, {on_process_unregistered, SlaveNode1, scope_bc, "proc", PidOn2, crash, syn_conflict_resolution}, - {on_process_unregistered, SlaveNode2, scope_bc, "proc", PidOn2, crash, syn_conflict_resolution} + {on_process_unregistered, SlaveNode2, scope_bc, "proc", PidOn2, crash, syn_conflict_resolution}, + {on_process_registered, SlaveNode2, scope_bc, "proc", PidOn1, crash, syn_conflict_resolution}, + {on_process_unregistered, SlaveNode2, scope_bc, "proc", PidOn1, crash, killed} ]), undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc"]), undefined = rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc"]). @@ -2741,13 +2783,9 @@ four_nodes_concurrency(Config) -> spawn(SlaveNode2, WorkerFun), spawn(SlaveNode3, WorkerFun), - %% wait for workers done - syn_test_suite_helper:assert_received_messages([ - {done, LocalNode}, - {done, SlaveNode1}, - {done, SlaveNode2}, - {done, SlaveNode3} - ]), + %% wait for workers done: each runs for several seconds, longer than the usual wait for a message + [receive {done, N} -> ok after 60000 -> ct:fail({worker_not_done, N}) end + || N <- [LocalNode, SlaveNode1, SlaveNode2, SlaveNode3]], %% check results are same across network syn_test_suite_helper:assert_wait( @@ -2769,7 +2807,7 @@ four_nodes_concurrency(Config) -> %% =================================================================== check_trace_for_ack_sync_via_sender() -> receive - {trace, _, 'receive', {send_single, _, {'3.0', ack_sync, _, _}, _}} -> + {trace, _, 'receive', {send_single, _, {'3.5', ack_sync, _, _}, _}} -> flush_trace_messages(), true; {trace, _, 'receive', _} -> @@ -2812,7 +2850,7 @@ get_registry_entry(Node, Scope, Name) -> %% Add scope_all on the local node and both slaves, disconnect slave 1 from slave 2, %% then register Name on slave 1 and later on slave 2. The local node sees both -%% registrations and holds the later one. +%% registrations and shows the later one. register_conflict_in_partial_netsplit(SlaveNode1, SlaveNode2, Name, Meta1, Meta2) -> %% add scopes ok = syn:add_node_to_scopes([scope_all]), @@ -2835,6 +2873,7 @@ register_conflict_in_partial_netsplit(SlaveNode1, SlaveNode2, Name, Meta1, Meta2 {PidOn1, Meta1}, fun() -> syn:lookup(scope_all, Name) end ), + syn_test_suite_helper:wait_clock_after(SlaveNode2, SlaveNode1), ok = rpc:call(SlaveNode2, syn, register, [scope_all, Name, PidOn2, Meta2]), syn_test_suite_helper:assert_wait( {PidOn2, Meta2}, @@ -2852,14 +2891,59 @@ join_with_ack_sync_first(Node, PeerNode, Name) -> {PeerScopePid, PeerSnapshot} = get_remote_snapshot(PeerNode, syn_registry_scope_all), true = lists:keymember(Name, 1, PeerSnapshot), ok = sys:suspend(ScopePid), - ScopePid ! {'3.0', ack_sync, PeerScopePid, PeerSnapshot}, + ScopePid ! {'3.5', ack_sync, PeerScopePid, PeerSnapshot}, wait_for_message_queue_len(ScopePid, 1), true = rpc:call(PeerNode, syn_test_suite_helper, connect_node, [Node]), syn_test_suite_helper:assert_cluster(Node, [node(), PeerNode]), - PeerScopePid ! {'3.0', discover, ScopePid}, + PeerScopePid ! {'3.5', discover, ScopePid}, syn_test_suite_helper:assert_scope_subcluster(registry, PeerNode, scope_all, [node(), Node]), ok = sys:resume(ScopePid). +%% the resolver keeps neither process: each slave drops its own and shows the other one until that one goes too; +%% this node shows the process of slave 1 for a moment only if slave 2 drops its own first +assert_keep_neither_callbacks(SlaveNode1, SlaveNode2, Scope, Name, PidOn1, Meta1, PidOn2, Meta2) -> + LocalNode = node(), + Slaves = [ + {on_process_unregistered, SlaveNode1, Scope, Name, PidOn1, Meta1, syn_conflict_resolution}, + {on_process_registered, SlaveNode1, Scope, Name, PidOn2, Meta2, syn_conflict_resolution}, + {on_process_unregistered, SlaveNode1, Scope, Name, PidOn2, Meta2, syn_conflict_resolution}, + {on_process_unregistered, SlaveNode2, Scope, Name, PidOn2, Meta2, syn_conflict_resolution}, + {on_process_registered, SlaveNode2, Scope, Name, PidOn1, Meta1, syn_conflict_resolution}, + {on_process_unregistered, SlaveNode2, Scope, Name, PidOn1, Meta1, syn_conflict_resolution} + ], + syn_test_suite_helper:assert_received_messages_one_of([ + [{on_process_unregistered, LocalNode, Scope, Name, PidOn2, Meta2, syn_conflict_resolution} | Slaves], + [ + {on_process_unregistered, LocalNode, Scope, Name, PidOn2, Meta2, syn_conflict_resolution}, + {on_process_registered, LocalNode, Scope, Name, PidOn1, Meta1, syn_conflict_resolution}, + {on_process_unregistered, LocalNode, Scope, Name, PidOn1, Meta1, syn_conflict_resolution} + | Slaves + ] + ]). + +%% starts a hidden node with syn and Scope on it, connected to this node only +start_hidden_node(Name, Scope) -> + {ok, PeerPid, Node} = peer:start(#{ + name => Name, + args => ["-hidden", "-connect_all", "false", "-kernel", "dist_auto_connect", "never"], + connection => 0 + }), + ok = rpc:call(Node, code, add_pathsa, [code:get_path()]), + true = net_kernel:connect_node(Node), + false = lists:member(Node, nodes()), + ok = rpc:call(Node, syn, start, []), + ok = rpc:call(Node, syn, add_node_to_scopes, [[Scope]]), + {PeerPid, Node}. + +%% the scope process ScopePid of Node stops knowing the one of OtherNode, as if it had never discovered it +forget_scope(Node, ScopePid, OtherNode) -> + rpc:call(Node, sys, replace_state, [ScopePid, fun(#state{nodes_map = NodesMap} = State) -> + {_, MRef} = maps:get(OtherNode, NodesMap), + erlang:demonitor(MRef, [flush]), + State#state{nodes_map = maps:remove(OtherNode, NodesMap)} + end]), + ok. + %% start syn and add scope_all on this node and on Node1 and Node2, returns the scope process of this node start_scope_all(Node1, Node2) -> ok = syn:start(), diff --git a/test/syn_test_event_handler_protocol.erl b/test/syn_test_event_handler_protocol.erl new file mode 100644 index 00000000..40adaca6 --- /dev/null +++ b/test/syn_test_event_handler_protocol.erl @@ -0,0 +1,37 @@ +%% ========================================================================================================== +%% Syn - A global Process Registry and Process Group manager. +%% +%% The MIT License (MIT) +%% +%% Copyright (c) 2015-2026 Roberto Ostinelli and Neato Robotics, Inc. +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ========================================================================================================== +-module(syn_test_event_handler_protocol). +-behaviour(syn_event_handler). + +%% the callbacks of syn_protocol_SUITE, without its resolver, so that Syn's default resolution applies +-export([on_process_registered/5]). +-export([on_process_unregistered/5]). + +on_process_registered(Scope, Name, Pid, Meta, Reason) -> + syn_protocol_SUITE:on_process_registered(Scope, Name, Pid, Meta, Reason). + +on_process_unregistered(Scope, Name, Pid, Meta, Reason) -> + syn_protocol_SUITE:on_process_unregistered(Scope, Name, Pid, Meta, Reason). diff --git a/test/syn_test_suite_helper.erl b/test/syn_test_suite_helper.erl index 0bd76dcd..2e0ea647 100755 --- a/test/syn_test_suite_helper.erl +++ b/test/syn_test_suite_helper.erl @@ -27,7 +27,6 @@ %% API -export([init_cluster/1, end_cluster/2]). --export([start_slave/1, stop_slave/1]). -export([connect_node/1, disconnect_node/1, disconnect_nodes/2]). -export([clean_after_test/0]). -export([start_process/0, start_process/1, start_process/2]). @@ -35,12 +34,15 @@ -export([run_async/1, run_async/2, wait_async/1, assert_async_waiting/1]). -export([wait_cluster_mesh_connected/1]). -export([wait_process_name_ready/1, wait_process_name_ready/2]). --export([wait_message_queue_empty/0]). -export([assert_cluster/2]). -export([assert_scope_subcluster/4]). -export([assert_received_messages/1]). +-export([assert_received_messages_one_of/1]). +-export([reconnect_cluster/1]). -export([assert_empty_queue/0]). -export([assert_wait/2]). +-export([wait_clock_after/2]). +-export([pause_process/1]). %% internal -export([process_main/0]). @@ -76,7 +78,7 @@ init_cluster(NodesCount) -> NodesConfig; Other -> - ct:pal("*********** Could not get full cluster of ~p nodes, skipping", [NodesCount]), + ct:pal("*********** Could not get full cluster of ~p nodes", [NodesCount]), {error_initializing_cluster, Other} end. @@ -84,7 +86,7 @@ end_cluster(NodesCount, Config) -> SlavesCount = NodesCount - 1, %% clean clean_after_test(), - %% shutdown (reconnect first since peer:stop needs distribution with connection => 0) + %% shutdown lists:foreach(fun(I) -> IBin = integer_to_binary(I), NodeShortName = list_to_atom(binary_to_list(<<"syn_slave_", IBin/binary>>)), @@ -105,19 +107,15 @@ start_slave(NodeShortName) -> %% controller process that is independent of the calling process and %% survives CT process lifecycle changes. %% - %% connection => 0 starts a detached peer that uses distribution for the - %% control channel. This allows passing -kernel dist_auto_connect never - %% which would otherwise prevent the standard_io boot handshake. + %% connection => 0 controls the peer through a TCP connection of its own instead + %% of standard io, so that the peer can run with -kernel dist_auto_connect never. {ok, PeerPid, Node} = peer:start(#{ name => NodeShortName, args => ["-connect_all", "false", "-kernel", "dist_auto_connect", "never"], connection => 0 }), - %% add syn code path to slaves - CodePath = lists:filter(fun(Path) -> - nomatch =/= string:find(Path, "/syn/") - end, code:get_path()), - ok = rpc:call(Node, code, add_pathsa, [CodePath]), + %% the slaves run the same code as this node, test modules included + ok = rpc:call(Node, code, add_pathsa, [code:get_path()]), %% return {ok, Node, PeerPid}. @@ -135,6 +133,16 @@ disconnect_nodes(Node1, Node2) -> rpc:call(Node1, erlang, disconnect_node, [Node2]), rpc:call(Node2, erlang, disconnect_node, [Node1]). +%% connects the nodes of Config with each other again, in case a failed test left them apart +reconnect_cluster(Config) -> + SlaveNodes = [Node || {Key, Node} <- Config, lists:member(Key, [syn_slave_1, syn_slave_2, syn_slave_3])], + Nodes = [node() | SlaveNodes], + [rpc:call(Node1, net_kernel, connect_node, [Node2]) || Node1 <- Nodes, Node2 <- Nodes, Node1 < Node2], + case SlaveNodes of + [] -> ok; + _ -> ok = wait_cluster_mesh_connected(Nodes) + end. + clean_after_test() -> Nodes = [node() | nodes()], %% shutdown @@ -269,16 +277,6 @@ wait_process_name_ready(Name, StartAt) -> end end. -wait_message_queue_empty() -> - timer:sleep(500), - syn_test_suite_helper:assert_wait( - ok, - fun() -> - flush_inbox(), - syn_test_suite_helper:assert_empty_queue() - end - ). - assert_cluster(Node, ExpectedNodes) -> assert_cluster(Node, ExpectedNodes, os:system_time(millisecond)). assert_cluster(Node, ExpectedNodes, StartAt) -> @@ -325,6 +323,41 @@ assert_received_messages_evaluate(MissingMessages, UnexpectedMessages) -> [get_line_from_stacktrace(), lists:reverse(MissingMessages), lists:reverse(UnexpectedMessages)] ). +%% like assert_received_messages/1, when the messages can be any one of several lists, in any order: waits until the +%% messages received match one of them, then takes whatever else comes, since one list can be part of another +assert_received_messages_one_of(Alternatives) -> + Sorted = [lists:sort(Messages) || Messages <- Alternatives], + Received = receive_until_one_of(Sorted, [], os:system_time(millisecond)) ++ receive_messages_until_quiet([]), + case lists:member(lists:sort(Received), Sorted) of + true -> + ok; + + false -> + ct:fail("~n\tReceive messages error (line ~p)~n\tReceived: ~p~n\tExpected one of: ~p~n", + [get_line_from_stacktrace(), Received, Alternatives]) + end. + +receive_until_one_of(Sorted, Received, StartAt) -> + case lists:member(lists:sort(Received), Sorted) of + true -> + Received; + + false -> + Remaining = max(0, ?DEFAULT_WAIT_TIMEOUT - (os:system_time(millisecond) - StartAt)), + receive + Message -> receive_until_one_of(Sorted, Received ++ [Message], StartAt) + after Remaining -> + Received + end + end. + +receive_messages_until_quiet(Received) -> + receive + Message -> receive_messages_until_quiet([Message | Received]) + after ?UNEXPECTED_MESSAGES_WAIT_TIMEOUT -> + lists:reverse(Received) + end. + assert_empty_queue() -> assert_empty_queue([]). assert_empty_queue(UnexpectedMessages) -> @@ -338,6 +371,25 @@ assert_empty_queue(UnexpectedMessages) -> end end. +%% suspends Pid, which can run on another node, until the returned fun is called: a process on its node holds the +%% suspension, which would end with the process that asked for it +pause_process(Pid) -> + Test = self(), + Holder = spawn(node(Pid), fun() -> + true = erlang:suspend_process(Pid), + Test ! {paused, self()}, + receive resume -> erlang:resume_process(Pid) end + end), + receive {paused, Holder} -> ok after 5000 -> ct:fail(pause_timeout) end, + fun() -> Holder ! resume, ok end. + +%% waits until the clock of Node reads later than the clock of OtherNode does now. The clocks of two nodes on the +%% same machine can be apart by more than the time between two calls, so a test that needs a registration on Node to be +%% more recent than one just made on OtherNode waits for this first +wait_clock_after(Node, OtherNode) -> + Time = rpc:call(OtherNode, erlang, system_time, []), + assert_wait(true, fun() -> rpc:call(Node, erlang, system_time, []) > Time end). + assert_wait(ExpectedResult, Fun) -> assert_wait(ExpectedResult, Fun, os:system_time(millisecond)). assert_wait(ExpectedResult, Fun, StartAt) -> diff --git a/test/tla/README.md b/test/tla/README.md index 856d896d..f4728d9f 100644 --- a/test/tla/README.md +++ b/test/tla/README.md @@ -1,170 +1,108 @@ -# TLA+ Model for Syn Registry - -The TLA+ specifications in this directory were authored by [Graham Hay](https://github.com/grahamrhay/syn_tla) -as a formal model of Syn's distributed registry conflict resolution protocol. - -## The Bug - -Graham's model found an invariant violation (`AllRegistered`) demonstrating a race condition -that produces zombie registry entries — entries pointing at dead processes that persist -indefinitely until the next network partition cycle. - -The scenario involves two nodes (n1, n2) and a single name "a": - -1. Both nodes register "a" while disconnected (n1 at time T1, n2 at time T2, T2 > T1). -2. The nodes reconnect, exchanging `discover` and `ack_sync` messages. -3. n2 unregisters "a", but this is queued behind the `discover` message in n2's gen_server mailbox. -4. n2 processes `discover` from n1 — sends `ack_sync` containing `{a, Pid2, T2}` (still registered at this point). -5. n1 processes `discover` from n2 — sends `ack_sync` containing `{a, Pid1, T1}`. -6. n1 processes n2's `ack_sync` — conflict resolution: T2 > T1, remote wins. n1 kills Pid1 - and replaces its entry with Pid2. **No broadcast is sent for the losing Pid1.** -7. n2 processes `unregister_on_node` — removes "a", broadcasts `sync_unregister` for Pid2 to n1. -8. n1 processes `sync_unregister` — removes Pid2. Now n1 has no entry for "a". -9. n2 processes n1's stale `ack_sync` (containing `{a, Pid1, T1}`) — no existing entry for "a", - so `handle_registry_sync` adds it unconditionally. **Zombie entry created.** - -The zombie entry on n2 points at Pid1 (dead on n1). n1 has no record of it. -No cleanup message will ever be sent because: - -- The monitor for Pid1 on n1 was flushed during conflict resolution (`maybe_demonitor`). -- n1 already removed Pid1 from its tables; no `DOWN` will fire, no `sync_unregister` will be broadcast. - -Consequences: - -- `syn:lookup(Scope, a)` on n2 returns a dead pid. -- Registering "a" on n2 returns `{error, taken}`. -- The zombie persists until the next disconnect/reconnect cycle purges n1's entries on n2. - -## The Fix - -In `syn_registry.erl`, in the `resolve_conflict/5` function, the "keep remote" path -(where the remote process wins the conflict) updates the local table and kills the losing -process, but does not broadcast a `sync_unregister` for the loser. The "keep neither" path -already does this. The fix adds the missing broadcast: - -```erlang -Pid -> - %% -> we keep the remote pid - ... - %% callbacks - syn_event_handler:call_event_handler(on_process_unregistered, ...), - syn_event_handler:call_event_handler(on_process_registered, ...), - %% broadcast unregister for the losing local pid - %% so other nodes clean up stale ack_sync data - syn_gen_scope:broadcast( - {'3.0', sync_unregister, Name, TablePid, TableMeta, syn_conflict_resolution}, - State - ); -``` - -This works because of the FIFO guarantee established by PR #87 (routing `ack_sync` through -the sender process, `sender_loop`). Both the stale `ack_sync` and the corrective `sync_unregister` -flow through the same node's `sender_loop`, so the receiver always processes the stale data first -and the cleanup second. The zombie is created momentarily but immediately removed. - -Fix applied in [this commit](https://github.com/ostinelli/syn/commit/7716a9885924643ce84bb47cbc36b5efb891e577). - -## Calls Made From Another Node - -A node can register or unregister a process that lives on another node: the call goes to that node, -which handles it like one for its own processes and sends the usual `sync_register` / `sync_unregister` -to the other nodes, but not to the calling node. The calling node gets the result instead in a -`register_reply` / `unregister_reply`, through the same queue as everything else that node sends it. -Its scope process applies it like a sync message, conflict resolution included, and only then does -the call return: a lookup on the calling node right after the call sees it, and nothing the other -node sent before can undo it. If the calling node keeps the name on another process, because it -resolved a conflict against this one or has a more recent registration of it, the call returns -`{error, race_condition}`. A call made from a scope process (i.e. from a -callback) can't wait for one: it gets its reply right away, and the scope process of the calling node -applies the result in the same way when it gets to it. - -In the model, `Register` and `Unregister` can be called from the node that runs the process or from -any node that can see it. - -When the calling node writes the result into its own table itself, outside of its scope process, the -write isn't lined up with what the other node sends to that scope. That is what happens with nodes -running a version that doesn't support the reply above, until they are upgraded. In those cases an older -`ack_sync` snapshot still waiting in the calling node's scope queue can't remove a name registered -from another node: the calling node marks what it writes from a reply, and reconciliation keeps a -marked entry that is newer than everything in the snapshot, since the snapshot can't know about it yet. -But: - -- A name unregistered from another node can come back if an older snapshot still has it. -- An unregister the other node sent before the call can arrive after it, and remove the entry the call - just wrote for the same process. -- If the calling node processes a `DOWN` for the other node after writing, it drops the entry, and a - snapshot sent before the call won't bring it back. -- If two nodes register the same name on each other's processes at the same time, each one writes - the other's process over its own entry, and the conflict is never resolved. - -The model doesn't cover these cases. - -### DOWN Messages Ahead of Earlier Messages - -Erlang only guarantees the order of messages between a pair of processes. When a scope process -restarts, its `DOWN` and the messages of its sender process come from two different processes, so -the `DOWN` could reach another node ahead of messages that were sent before it. The model has no -scope restarts: with `DownOvertakes = TRUE` (see `syn_down_overtakes.cfg`), the `DOWN` of a -`Disconnect` can land anywhere in the inbox instead. - -All the invariants still hold, but `ReadYourWrites` doesn't: if the calling node handles the `DOWN` -before the reply, it no longer knows the other node and doesn't apply the reply, so the call returns -`ok` while the name isn't in its table yet. It gets there with the other node's `ack_sync`, once the -two nodes have found each other again. `syn_down_overtakes.cfg` leaves `ReadYourWrites` out for this -reason. - -## Running the Model Checker - -Requires Java 8+: - -``` -cd test/tla -java -XX:+UseParallelGC -cp tla2tools.jar tlc2.TLC \ - -metadir /tmp/syn-tla-states \ - -config syn.cfg -workers auto -cleanup syn.tla -``` - -Use `-config syn_down_overtakes.cfg` for DOWN messages that can arrive ahead of earlier ones. - -The model checks these properties: - -- **AllRegistered** (invariant): When all messages are processed and all nodes are connected, - every node's view of registered names matches the global truth. -- **ThereCanBeOnlyOne** (invariant): No name is registered on two different nodes simultaneously - on any single node's view. -- **ViewsMatchOwners** (invariant): When all messages are processed and all nodes are connected, - every node has each name on the node that actually runs it. -- **AllMessagesProcessed** (liveness): All message queues eventually drain. -- **ReadYourWrites** (action property): A register call made from another node that returns `ok` - has the name in the calling node's table. A reply that the calling node applies always does, so this - fails only when the reply comes from a node the calling node doesn't know (anymore). - -The model starts from an already discovered, fully connected subcluster. It then models -membership loss and rediscovery through disconnect/reconnect cycles. This keeps startup -discovery out of scope while covering the protocol races that happen after a known -remote scope process goes down and later rejoins. It only covers the registry, without -registering a process again with other meta. - -A node can only have one unregister per name waiting in its inbox. More would just fail with -`{error, undefined}`, and without that limit the inbox can grow forever, so the checker never finishes. - -## Unit Tests - -`syn_tests.tla` contains unit tests for the TLA+ helper functions (`AllRegisteredForNode`, -`AllRegisteredNames`, `Duplicates`, `MergeRegistries`, `MergeRegistrySnapshot`). Run with: +# TLA+ Model of Syn's Registry + +`syn.tla` describes how Syn keeps registrations in sync between nodes. TLC, the TLA+ model checker, runs it +through every order in which messages can arrive, with connections dropping, messages getting lost and scope +processes restarting along the way, and checks that the nodes always end up agreeing. + +The rules it follows are the ones described in `INTERNALS.md` at the root of the repository. + +## What the model contains + +- **Claims.** A claim is a registration as held by the node that runs the process. Only that node creates, changes or + drops its claims. Every node keeps a copy of the claims of the nodes it knows. +- **What a node shows.** A node that holds its own claim for a name shows it. Otherwise it shows the newest claim it + has a copy of, with node names breaking ties. It never hides a claim because it expects another node to drop it. +- **Conflicts.** When a node holds a claim for a name and gets a copy of another node's claim for it, it asks the + resolver which process keeps the name, passing it the two claims in node order, as the other node does. If its own + loses, it drops it and tells everyone. If its own wins, it does nothing: the other node will drop its own. +- **Messages.** Messages from one scope process to one node arrive in the order they were sent, unless some are lost + when the connection drops. Nothing else is ordered: messages from different nodes, "went away" notices (DOWN) and + "connected" notices (nodeup) can arrive in any order, late included. +- **Connections.** A connection can drop: what is already in a mailbox stays, what is on the wire is lost, and what a + sender hasn't sent yet waits in its queue and is lost if it gets sent before the connection is back. A connection can + come back at any moment, whatever reopens it. A scope process ignores a handshake from a node it isn't connected to. +- **Scope process restarts.** A scope process can crash, and a new one starts later. In between, messages sent to that + node are lost. The crash loses its mailbox and what its sender hadn't sent; the claims of the node's own processes + survive, as Syn's tables do. Nodes find each other again by asking for each other's data (`discover`) and sending it + (`ack_sync`). A node that loses track of another node while still connected to it asks it for its data again. +- **Calls from another node.** A registration or update called on one node for a process on another is handled by the + process' node, which sends the result to the caller's node like any other message. If the caller's scope process is + new to it, it sends it its data and asks for the caller node's. + +It doesn't model process groups (pg), callbacks, what a call returns, or a process registered under several names. +A call returns what the caller's node shows once it applies the result, so return values, callbacks and process +groups are covered by the Erlang test suites. Monitors are modeled by the scope process they watch: the Erlang code +also checks the monitor reference. + +## What it checks + +Each of these holds once nothing is left to deliver or handle: + +- **Membership**: every running scope process knows the current scope process of every node it's connected to that + runs one, and has an exact copy of that node's claims; it has nothing from the other nodes. +- **Agreement**: with every node connected and running, all nodes show the same claim for each name, the claim shown + is still held by its node, and a name that some node holds is shown everywhere. +- **OneOwner**: with every node connected and running, at most one node holds each name. +- **ResolutionBounded**: conflicts only ever make a node drop its claim, and each claim is dropped at most once, so + there is no endless back and forth. +- **Survivor**: with a resolver that compares things that don't change during the conflict (Syn's default with clocks + that don't go back, or a start time stored in the metadata), a name that got registered, and whose processes were + never unregistered and never exited, is still held by someone. Other resolvers can make every node drop it; all + nodes still agree on that. +- **DeliveredInChannel**: the model's own bookkeeping of what reached a mailbox stays consistent. +- **Termination**: message traffic always stops, and stays stopped, once outside actions stop. + +## Running it + +Requires Java 11 or later. From this directory: ``` -java -cp tla2tools.jar tlc2.TLC -metadir /tmp/syn-tla-unit-states \ - -config syn_tests.cfg syn_tests.tla +java -XX:+UseParallelGC -cp tla2tools.jar tlc2.TLC -workers auto -deadlock -cleanup \ + -metadir /tmp/syn-tla-states -config syn.cfg syn.tla ``` -The `-metadir` arguments keep TLC's disk state outside the repository. This avoids -interference with `make test`, which copies the `test/` directory during compilation. - -## Copyright - -The TLA+ specifications in this directory were authored by -[Graham Hay](https://github.com/grahamrhay) and are included here with minor modifications -(the bug fix described above). The original model is available at -https://github.com/grahamrhay/syn_tla. All rights to the original work remain with -the author. +`-deadlock` is needed because a run that has used up all its outside actions simply stops, which is expected. +`-metadir` keeps TLC's files outside the repository, so that they don't get copied into the test build. Give each +run its own directory when running several at once, and start them a few seconds apart: TLC writes its standard +modules to the system's temporary folder as it starts, and a run starting at the same moment as another can fail to +parse them. + +| Config | What it covers | +|--------------------------------|----------------------------------------------------------------------------------| +| `syn.cfg` | Two nodes register the same name, and any connection can drop once. | +| `syn_conflict.cfg` | Issue #96: one connection drops twice, the resolver keeps the older process. | +| `syn_three_owners.cfg` | Issue #96 with three nodes that register the name, the older process kept. | +| `syn_calls.cfg` | A call from another node, clocks that disagree, a conflict lost after an update. | +| `syn_calls_two.cfg` | Two calls from other nodes at the same time. | +| `syn_skew.cfg` | Clocks that disagree with Syn's default resolver, an update during the conflict. | +| `syn_backward.cfg` | The default resolver with a clock that moves backwards during the conflict. | +| `syn_two_names.cfg` | Two names, a dropped connection, a process exiting. | +| `syn_cyclic.cfg` | Three nodes register the same name, and no process beats both others. | +| `syn_second.cfg` | A resolver that keeps the claim it gets second when the two have the same time. | +| `syn_restart.cfg` | A scope process restarts and a connection drops. | +| `syn_restart_call.cfg` | Two nodes: a scope process restarts, a connection drops, a call from the other. | +| `syn_termination*.cfg` | Message traffic always stops, checked in four setups. | + +All of them must report `Model checking completed. No error has been found.` See the comment at the top of each file +for what it covers. + +### Checking the checks + +The files in `mutations/` each switch on a mistake the protocol has to avoid. Each one must make TLC report a failure: +that shows the checks above would catch it. + +| Config | The mistake | +|-----------------------------------------|----------------------------------------------------------------------------| +| `mutations/hide_after_conflict.cfg` | A node that loses a conflict makes the others hide every claim for the name. | +| `mutations/refresh_winner.cfg` | The node that keeps its process gives it a new time and tells everyone. | +| `mutations/withdraw_before_result.cfg` | The caller of an update hears of the withdrawal before the update. | +| `mutations/no_rediscover.cfg` | A node that loses track of a connected node doesn't ask it for its data again. | +| `mutations/no_ask_on_adopt.cfg` | A node that learns of a new scope process through a call doesn't ask for its data. | +| `mutations/resolve_by_role.cfg` | Each node passes the other node's claim to the resolver first and its own second. | + +Run them like the others, with `-config mutations/.cfg`. + +## Credits + +The first TLA+ model of Syn was written by [Graham Hay](https://github.com/grahamrhay/syn_tla): it found a race that left +registry entries pointing at dead processes. This model builds on that work. diff --git a/test/tla/mutations/hide_after_conflict.cfg b/test/tla/mutations/hide_after_conflict.cfg new file mode 100644 index 00000000..3bd602dd --- /dev/null +++ b/test/tla/mutations/hide_after_conflict.cfg @@ -0,0 +1,36 @@ +\* Must fail: a node that loses a conflict makes the others hide every claim for the name. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "oldest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = TRUE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/mutations/no_ask_on_adopt.cfg b/test/tla/mutations/no_ask_on_adopt.cfg new file mode 100644 index 00000000..b18eb86a --- /dev/null +++ b/test/tla/mutations/no_ask_on_adopt.cfg @@ -0,0 +1,36 @@ +\* Must fail: a node that learns of a new scope process through a call doesn't ask for its data. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {2} + MaxRegister = 1 + MaxRemoteCall = 1 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 1 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = TRUE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/mutations/no_rediscover.cfg b/test/tla/mutations/no_rediscover.cfg new file mode 100644 index 00000000..af2ad73a --- /dev/null +++ b/test/tla/mutations/no_rediscover.cfg @@ -0,0 +1,36 @@ +\* Must fail: a node that loses track of a connected node doesn't ask it for its data again. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {2} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 1 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = TRUE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/mutations/refresh_winner.cfg b/test/tla/mutations/refresh_winner.cfg new file mode 100644 index 00000000..7825e458 --- /dev/null +++ b/test/tla/mutations/refresh_winner.cfg @@ -0,0 +1,37 @@ +\* Must fail (ResolutionBounded): the node that keeps its process gives it a new time and tells +\* everyone. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}, {1, 3}, {2, 3}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = TRUE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/mutations/resolve_by_role.cfg b/test/tla/mutations/resolve_by_role.cfg new file mode 100644 index 00000000..30fb219a --- /dev/null +++ b/test/tla/mutations/resolve_by_role.cfg @@ -0,0 +1,37 @@ +\* Must fail (Agreement): each node passes the other node's claim first and its own second, so a resolver that +\* keeps the claim passed second on a tie makes both nodes keep their own process. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "second" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = TRUE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/mutations/withdraw_before_result.cfg b/test/tla/mutations/withdraw_before_result.cfg new file mode 100644 index 00000000..4fc1b858 --- /dev/null +++ b/test/tla/mutations/withdraw_before_result.cfg @@ -0,0 +1,36 @@ +\* Must fail: the caller of an update hears that the claim was withdrawn before hearing the update. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1, 2} + Resolver = "oldest_time" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 1 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = TRUE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn.cfg b/test/tla/syn.cfg index 95dbcb1a..2f84694f 100644 --- a/test/tla/syn.cfg +++ b/test/tla/syn.cfg @@ -1,20 +1,36 @@ +\* Two nodes register the same name, and any connection can drop once. + INIT Init NEXT Next CONSTANTS -Nodes = {n1, n2} -Names = {a} -MaxDisconnections = 1 -DownOvertakes = FALSE - -SYMMETRY -Symmetry + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}, {1, 3}, {2, 3}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE INVARIANTS -AllRegistered -ThereCanBeOnlyOne -ViewsMatchOwners - -PROPERTIES -AllMessagesProcessed -ReadYourWrites + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn.tla b/test/tla/syn.tla index 56ec1f5a..98ef7061 100644 --- a/test/tla/syn.tla +++ b/test/tla/syn.tla @@ -1,392 +1,622 @@ ----- MODULE syn ---- -EXTENDS FiniteSets, Integers, Sequences, TLC +-------------------------------- MODULE syn -------------------------------- +(***************************************************************************) +(* A model of how Syn 3.5 keeps registrations in sync between nodes, as *) +(* described in INTERNALS.md. TLC checks it against every order in which *) +(* messages can arrive, with connections dropping and scope processes *) +(* restarting along the way. *) +(* *) +(* A "claim" is a registration as held by the node that runs the process. *) +(* Every node keeps a copy of the claims of the nodes it knows, and shows *) +(* one claim per name. *) +(* *) +(* Messages from one scope process to one node arrive in the order they *) +(* were sent, unless some are lost when the connection drops. Nothing else *) +(* is ordered. Instead of using a mailbox, a scope process takes the next *) +(* message from any of its incoming channels, which covers every order a *) +(* mailbox could hold them in. *) +(***************************************************************************) +EXTENDS Naturals, Sequences, FiniteSets, TLC + +CONSTANTS + NONE, \* a model value meaning "nothing": no claim, no process + Nodes, \* the nodes running the scope: 1..N, in the order their names sort + Names, \* the names that can be registered + Times, \* the clock readings a claim can get; any node can get any of them + Resolver, \* how conflicts are decided: "newest", "oldest", "oldest_time", "second", "neither" or + \* "cyclic" + MonotonicClocks, \* TRUE: updating a claim never gives it an earlier time + RegisterNodes, \* the nodes whose processes get registered + CutPairs, \* the pairs of nodes whose connection can drop, as sets {n, m} + RestartNodes, \* the nodes whose scope process can restart + MaxRegister, \* how many registrations are made on the process' own node + MaxRemoteCall, \* how many registrations and updates are called from another node + MaxUpdate, \* how many updates are made on the process' own node + MaxUnregister, \* how many unregistrations + MaxExit, \* how many registered processes exit + MaxDisconnect, \* how many times a connection drops + MaxRestart, \* how many times a scope process restarts + \* switches that bring back known mistakes: each must make TLC report a failure, which shows the + \* checks below would catch it + HideAfterConflict, \* a node that loses a conflict makes the others hide every claim for the name + RefreshWinner, \* the node that keeps its process gives it a new time and tells everyone + WithdrawBeforeResult, \* the caller of an update hears that the claim was withdrawn before hearing the update + NoRediscover, \* a node that loses track of a connected node doesn't ask it for its data again + NoAskOnAdopt, \* a node that learns of a new scope process through a call doesn't ask for its data + ResolveByRole \* a node passes the other node's claim to the resolver first and its own second, instead + \* of passing the two in node order + +ASSUME Nodes = 1..Cardinality(Nodes) +ASSUME RegisterNodes \subseteq Nodes +ASSUME RestartNodes \subseteq Nodes +ASSUME Resolver \in {"newest", "oldest", "oldest_time", "second", "neither", "cyclic"} + +Incs == 1..(MaxRestart + 1) +NodeSeq == [i \in 1..Cardinality(Nodes) |-> i] +MaxTime == CHOOSE t \in Times: \A u \in Times: u <= t + +VARIABLES + alive, \* processes that are running, as <> + own, \* own[n][nm]: the claim node n holds for name nm, or NONE + remote, \* remote[n][m][nm]: n's copy of node m's claim for nm, or NONE + known, \* known[n][m]: the scope process n believes is m's, and watches with a monitor, or NONE + asked, \* asked[n]: nodes n asked for their data and hasn't heard back from + inc, \* inc[n]: how many times n's scope process started; that process is <> + conn, \* the pairs of nodes that are connected, as sets {n, m} + chan, \* chan[<>]: messages the i-th scope process of node n sent to node m, not handled yet + dlv, \* dlv[k]: how many of the first messages in chan[k] already reached the receiver's mailbox, + \* so that a dropped connection doesn't lose them + pdown, \* pdown[n]: scope processes whose "went away" notice (DOWN) is on its way to n + pexit, \* pexit[n]: n's registered processes that exited, with the notice on its way to n's scope process + pnodeup, \* pnodeup[n]: nodes that connected to n, with the notice on its way + pcall, \* pcall[n]: calls from other nodes on their way to n's scope process + up, \* up[n]: n's scope process is running + nextPid, \* the id the next new process gets + counts, \* how many times each outside action happened so far, and how many claims were created + resMsgs, \* how many messages conflict resolutions sent + held, \* the names that got registered at some point + removed \* the names whose process got unregistered or exited at some point + +vars == <> + +ChanKeys == Nodes \X Incs \X Nodes + +Me(n) == <> +ScopeAlive(s) == up[s[1]] /\ inc[s[1]] = s[2] +Connected(n, m) == {n, m} \in conn +NoClaims == [nm \in Names |-> NONE] +\* the length of the longest of the channels in S, 0 if none +Longest(S) == IF S = {} THEN 0 ELSE CHOOSE l \in {Len(chan[k]): k \in S}: \A k \in S: Len(chan[k]) <= l + +----------------------------------------------------------------------------- +(* Which claim a node shows for a name *) + +\* the newer of two claims from different nodes: the later time, or with equal times the node that sorts last +Newer(c, d) == \/ c.time > d.time + \/ c.time = d.time /\ c.pid[1] > d.pid[1] + +\* a node holding its own claim shows it; otherwise it shows the newest claim it has a copy of +ShownIn(ownN, remoteN, n, nm) == + IF ownN[nm] # NONE THEN ownN[nm] + ELSE LET cs == {remoteN[m][nm]: m \in Nodes \ {n}} \ {NONE} + IN IF cs = {} THEN NONE ELSE CHOOSE c \in cs: \A d \in cs: c = d \/ Newer(c, d) + +Shown(n, nm) == ShownIn(own[n], remote[n], n, nm) + +----------------------------------------------------------------------------- +(* Conflicts: which of two processes keeps the name, or NONE for neither. Both nodes involved *) +(* decide it on their own and must get the same answer: they pass the two claims to the *) +(* resolver in node order, so that a resolver that depends on the order of its arguments *) +(* still decides the same on both. *) + +PidLess(p, q) == p[2] < q[2] +Beats(a, b) == (a = 2 /\ b = 1) \/ (a = 3 /\ b = 2) \/ (a = 1 /\ b = 3) + +Keep(c, d) == + CASE Resolver = "newest" -> \* Syn's default: the newer registration wins + IF c.time # d.time THEN (IF c.time > d.time THEN c.pid ELSE d.pid) + ELSE IF PidLess(c.pid, d.pid) THEN d.pid ELSE c.pid + [] Resolver = "oldest" -> \* the one started first wins, judged by something that never changes + IF PidLess(c.pid, d.pid) THEN c.pid ELSE d.pid + [] Resolver = "oldest_time" -> \* the older time wins, but an update gives the claim a new time + IF c.time # d.time THEN (IF c.time < d.time THEN c.pid ELSE d.pid) + ELSE IF PidLess(c.pid, d.pid) THEN c.pid ELSE d.pid + [] Resolver = "second" -> \* the older time wins, and a tie keeps the claim passed second + IF c.time < d.time THEN c.pid ELSE d.pid + [] Resolver = "neither" -> NONE + [] Resolver = "cyclic" -> \* like rock-paper-scissors: node 2 beats 1, 3 beats 2, 1 beats 3 + IF Beats(c.pid[1], d.pid[1]) THEN c.pid ELSE d.pid + +\* the decision between the other node's claim c and the deciding node's own claim d +Decide(c, d) == IF ResolveByRole \/ c.pid[1] < d.pid[1] THEN Keep(c, d) ELSE Keep(d, c) + +----------------------------------------------------------------------------- +(* Each step of a scope process works on a copy of its node's data (St), collects the messages *) +(* it sends in st.out, and then saves everything at once (Commit). *) + +St(n) == [own |-> own[n], remote |-> remote[n], known |-> known[n], asked |-> asked[n], pdown |-> pdown[n], + out |-> <<>>, res |-> 0, created |-> 0, names |-> {}] + +Send(st, m, msg) == [st EXCEPT !.out = Append(@, <>)] + +Broadcast(st, msg, excluded) == + LET ts == SelectSeq(NodeSeq, LAMBDA m: st.known[m] # NONE /\ m \notin excluded) + IN [st EXCEPT !.out = @ \o [i \in 1..Len(ts) |-> <>]] + +Snapshot(n, st) == [type |-> "ack_sync", from |-> Me(n), data |-> st.own] +Discover(n) == [type |-> "discover", from |-> Me(n)] +ClaimMsg(n, nm, c) == [type |-> "claim", from |-> Me(n), name |-> nm, claim |-> c] +WithdrawMsg(n, nm, p, why) == [type |-> "withdraw", from |-> Me(n), name |-> nm, pid |-> p, why |-> why] + +\* n starts watching scope process s of node m, instead of whatever it watched for m before; a "went away" +\* notice still pending from the old one is thrown away +Monitor(st, n, m, s) == + [st EXCEPT !.known[m] = s, + !.pdown = {d \in @: d[1] # m} \cup (IF ScopeAlive(s) THEN {} ELSE {s})] + +Forget(st, m) == [st EXCEPT !.remote[m] = NoClaims] + +\* n starts treating s as node m's scope process, which it only does while connected to m; if it knew another one +\* for m, that one restarted, so the claims n had from it go +Adopt(st, n, s) == + LET m == s[1] + IN Monitor(IF st.known[m] # NONE /\ st.known[m] # s THEN Forget(st, m) ELSE st, n, m, s) + +\* n asks node m for its data, unless it already asked and is waiting for the answer +Ask(st, n, m) == + IF m \in st.asked THEN st + ELSE [Send(st, m, Discover(n)) EXCEPT !.asked = @ \cup {m}] + +\* the same, when n learns of m's scope process through a call +AskForCall(st, n, m) == IF NoAskOnAdopt THEN st ELSE Ask(st, n, m) + +\* n drops its own claim for nm and tells everyone +Withdraw(st, n, nm, why) == + LET p == st.own[nm].pid + IN Broadcast([st EXCEPT !.own[nm] = NONE], WithdrawMsg(n, nm, p, why), {}) + +\* n holds a claim for nm and has copies of other nodes' claims for it: n keeps its own only if it wins against +\* every one of them, and otherwise drops it +Resolve(st, n, nm) == + LET others == {m \in Nodes \ {n}: st.remote[m][nm] # NONE} + IN IF st.own[nm] = NONE \/ others = {} THEN st + ELSE IF \A m \in others: Decide(st.remote[m][nm], st.own[nm]) = st.own[nm].pid + THEN IF RefreshWinner + THEN LET c == [st.own[nm] EXCEPT !.time = MaxTime] + IN Broadcast([st EXCEPT !.own[nm] = c, !.res = @ + 1], ClaimMsg(n, nm, c), {}) + ELSE st + ELSE [Withdraw(st, n, nm, "conflict") EXCEPT !.res = @ + 1] + +RECURSIVE ResolveSet(_, _, _) +ResolveSet(st, n, S) == + IF S = {} THEN st + ELSE LET nm == CHOOSE x \in S: TRUE IN ResolveSet(Resolve(st, n, nm), n, S \ {nm}) + +RECURSIVE WithdrawSet(_, _, _) +WithdrawSet(st, n, S) == + IF S = {} THEN st + ELSE LET nm == CHOOSE x \in S: TRUE IN WithdrawSet(Withdraw(st, n, nm, "normal"), n, S \ {nm}) + +\* saves st as node n's data and queues its messages; ch is the channels to add them to, base the counts to +\* add its new claims to +Commit(n, st, ch, base) == + /\ own' = [own EXCEPT ![n] = st.own] + /\ remote' = [remote EXCEPT ![n] = st.remote] + /\ known' = [known EXCEPT ![n] = st.known] + /\ asked' = [asked EXCEPT ![n] = st.asked] + /\ pdown' = [pdown EXCEPT ![n] = st.pdown] + /\ chan' = [k \in ChanKeys |-> + IF k[1] = n /\ k[2] = inc[n] + THEN LET sel == SelectSeq(st.out, LAMBDA e: e[1] = k[3]) + IN ch[k] \o [i \in 1..Len(sel) |-> sel[i][2]] + ELSE ch[k]] + /\ resMsgs' = resMsgs + st.res + /\ counts' = [base EXCEPT !.created = @ + st.created] + /\ held' = held \cup st.names + +----------------------------------------------------------------------------- +(* What a scope process does with each kind of message *) + +\* a handshake from a node n isn't connected to was sent before the connection dropped: n ignores it, as the other +\* node asks again once the connection is back +OnDiscover(n, msg, st0) == + LET s == msg.from + m == s[1] + IN IF ~Connected(n, m) THEN st0 + ELSE IF st0.known[m] = s THEN Send(st0, m, Snapshot(n, st0)) + ELSE Ask(Send(Adopt(st0, n, s), m, Snapshot(n, st0)), n, m) + +OnAckSync(n, msg, st0) == + LET s == msg.from + m == s[1] + \* a node n didn't know gets n's data first, before a conflict decided below can change it + st1 == IF st0.known[m] = s THEN st0 ELSE Send(Adopt(st0, n, s), m, Snapshot(n, st0)) + st2 == [st1 EXCEPT !.remote[m] = msg.data, !.asked = @ \ {m}] + IN IF ~Connected(n, m) THEN st0 ELSE ResolveSet(st2, n, Names) + +OnClaim(n, msg, st0) == + LET m == msg.from[1] + IN IF st0.known[m] = msg.from + THEN Resolve([st0 EXCEPT !.remote[m][msg.name] = msg.claim], n, msg.name) + ELSE st0 + +OnWithdraw(n, msg, st0) == + LET m == msg.from[1] + nm == msg.name + cur == st0.remote[m][nm] + IN IF st0.known[m] = msg.from /\ cur # NONE /\ cur.pid = msg.pid + THEN IF HideAfterConflict /\ msg.why = "conflict" + THEN [st0 EXCEPT !.remote = [k \in Nodes |-> [@[k] EXCEPT ![nm] = NONE]]] + ELSE [st0 EXCEPT !.remote[m][nm] = NONE] + ELSE st0 + +Handle(n, msg, st0) == + CASE msg.type = "discover" -> OnDiscover(n, msg, st0) + [] msg.type = "ack_sync" -> OnAckSync(n, msg, st0) + [] msg.type = "claim" -> OnClaim(n, msg, st0) + [] msg.type = "withdraw" -> OnWithdraw(n, msg, st0) + +\* the scope process of node k[3] handles the next message in channel k +Receive(k) == + /\ chan[k] # <<>> + /\ up[k[3]] + /\ dlv[k] > 0 \/ Connected(k[1], k[3]) + /\ Commit(k[3], Handle(k[3], Head(chan[k]), St(k[3])), [chan EXCEPT ![k] = Tail(@)], counts) + /\ dlv' = [dlv EXCEPT ![k] = IF @ > 0 THEN @ - 1 ELSE 0] + /\ UNCHANGED <> + +\* the next message in channel k that hasn't reached a mailbox is sent while the two nodes are apart, or while the +\* receiving scope process isn't running: it is lost +Lose(k) == + /\ Len(chan[k]) > dlv[k] + /\ ~Connected(k[1], k[3]) \/ ~up[k[3]] + /\ chan' = [chan EXCEPT ![k] = SubSeq(@, 1, dlv[k]) \o SubSeq(@, dlv[k] + 2, Len(@))] + /\ UNCHANGED <> + +\* if n is connected to the node when the notice arrives, the node didn't go: its scope process restarted, or the +\* notice is from a connection that dropped and came back. n asks whatever scope process runs there now for its data +ReceiveDown(n, s) == + /\ s \in pdown[n] + /\ up[n] + /\ LET m == s[1] + st0 == [St(n) EXCEPT !.pdown = @ \ {s}] + st1 == [Forget(st0, m) EXCEPT !.known[m] = NONE] + st2 == IF st0.known[m] # s THEN st0 + ELSE IF Connected(n, m) /\ ~NoRediscover + THEN [Send(st1, m, Discover(n)) EXCEPT !.asked = @ \cup {m}] + ELSE st1 + IN Commit(n, st2, chan, counts) + /\ UNCHANGED <> + +ReceiveExit(n, p) == + /\ p \in pexit[n] + /\ up[n] + /\ LET st0 == St(n) + IN Commit(n, WithdrawSet(st0, n, {nm \in Names: st0.own[nm] # NONE /\ st0.own[nm].pid = p}), chan, counts) + /\ pexit' = [pexit EXCEPT ![n] = @ \ {p}] + /\ UNCHANGED <> + +ReceiveNodeup(n, m) == + /\ m \in pnodeup[n] + /\ up[n] + /\ Commit(n, [Send(St(n), m, Discover(n)) EXCEPT !.asked = @ \cup {m}], chan, counts) + /\ pnodeup' = [pnodeup EXCEPT ![n] = @ \ {m}] + /\ UNCHANGED <> + +----------------------------------------------------------------------------- +(* Calls made on one node for a process running on another, handled by the scope process of the *) +(* process' node. The result goes to the caller's node like the other messages; what the caller *) +(* gets back is decided from what its node shows then, and is checked by the Erlang tests. *) + +\* the times a node can give a new version of claim c +NextTimes(c) == IF MonotonicClocks /\ c # NONE THEN {t \in Times: t >= c.time} ELSE Times + +ReceiveCall(y, call) == + /\ call \in pcall[y] + /\ up[y] + /\ LET x == call.caller + nm == call.name + st0 == St(y) + \* if the caller's scope process is new to y, y sends it its data and asks for the caller node's + st1 == IF call.requester = NONE \/ st0.known[x] = call.requester \/ ~Connected(y, x) THEN st0 + ELSE AskForCall(Send(Adopt(st0, y, call.requester), x, Snapshot(y, st0)), y, x) + IN \E t \in NextTimes(st1.own[nm]): + LET st2 == + CASE call.pid \notin alive -> st1 + [] call.kind = "register" -> + IF ShownIn(st1.own, st1.remote, y, nm) # NONE THEN st1 + ELSE LET c == [pid |-> call.pid, time |-> t, ver |-> 1] + st3 == [st1 EXCEPT !.own[nm] = c, !.created = @ + 1, !.names = @ \cup {nm}] + IN Send(Broadcast(st3, ClaimMsg(y, nm, c), {x}), x, ClaimMsg(y, nm, c)) + [] call.kind = "update" -> + IF st1.own[nm] = NONE \/ st1.own[nm].pid # call.pid THEN st1 + ELSE LET c == [st1.own[nm] EXCEPT !.time = t, !.ver = @ + 1] + st3 == Broadcast([st1 EXCEPT !.own[nm] = c], ClaimMsg(y, nm, c), {x}) + IN IF WithdrawBeforeResult + THEN Send(Resolve(st3, y, nm), x, ClaimMsg(y, nm, c)) + ELSE Resolve(Send(st3, x, ClaimMsg(y, nm, c)), y, nm) + IN Commit(y, st2, chan, counts) + /\ pcall' = [pcall EXCEPT ![y] = @ \ {call}] + /\ UNCHANGED <> + +----------------------------------------------------------------------------- +(* Things that happen from outside: calls made by applications, processes exiting, connections *) +(* dropping and coming back, scope processes restarting *) -CONSTANTS Nodes, Names, MaxDisconnections, DownOvertakes - -Symmetry == Permutations(Nodes) - -VARIABLES inbox, registered, locally_registered, names, visible_nodes, known_nodes, disconnections, time - -vars == <> - -AllOtherNodes(n) == - Nodes \ {n} - -RECURSIVE ReduceStruct(_, _, _) - -ReduceStruct(keys, struct, acc) == - IF keys = {} THEN acc - ELSE - LET k == CHOOSE k \in keys: TRUE - IN ReduceStruct(keys \ {k}, struct, acc \union DOMAIN struct[k]) - -AllRegisteredForNode(locals) == - ReduceStruct(DOMAIN locals, locals, {}) - -RECURSIVE AllRegisteredNames(_, _, _) - -AllRegisteredNames(nodes, locals, registrations) == - IF nodes = {} THEN registrations - ELSE - LET n == CHOOSE n \in nodes: TRUE - IN AllRegisteredNames(nodes \ {n}, locals, registrations \union DOMAIN locals[n][n]) - -RegisteredElsewhere(node) == - AllRegisteredNames(AllOtherNodes(node), locally_registered, {}) - -RegisteredOnThisNode(node) == - AllRegisteredNames({node}, locally_registered, {}) - -ConnectedKnownNodes(n) == - visible_nodes[n] \intersect known_nodes[n] - -\* what node o # n gets when n handles a register or unregister call: a node that made the call -\* from elsewhere gets the reply instead of the sync message, through the same queue as everything -\* else n sends it, as long as it can still be reached -Outgoing(o, n, requester, reply, sync) == - CASE (o = requester) -> (IF o \in visible_nodes[n] THEN Append(inbox[o], reply) ELSE inbox[o]) - [] (o \in ConnectedKnownNodes(n)) -> Append(inbox[o], sync) - [] OTHER -> inbox[o] - -InsertAt(s, i, e) == - SubSeq(s, 1, i - 1) \o <> \o SubSeq(s, i, Len(s)) +Register(n) == + /\ counts.reg < MaxRegister + /\ n \in RegisterNodes + /\ up[n] + /\ \E nm \in Names, t \in Times: + /\ Shown(n, nm) = NONE + /\ LET p == <> + c == [pid |-> p, time |-> t, ver |-> 1] + st == [St(n) EXCEPT !.own[nm] = c, !.created = 1, !.names = {nm}] + IN /\ Commit(n, Broadcast(st, ClaimMsg(n, nm, c), {}), chan, [counts EXCEPT !.reg = @ + 1]) + /\ alive' = alive \cup {p} + /\ nextPid' = nextPid + 1 + /\ UNCHANGED <> + +\* the caller's node gives its scope process with the call, if it is running +Requester(x) == IF up[x] THEN Me(x) ELSE NONE + +RegisterCall(x, y) == + /\ counts.call < MaxRemoteCall + /\ y \in RegisterNodes + /\ x # y + /\ Connected(x, y) + /\ up[y] + /\ \E nm \in Names: + pcall' = [pcall EXCEPT ![y] = @ \cup {[id |-> counts.call + 1, kind |-> "register", name |-> nm, + pid |-> <>, caller |-> x, requester |-> Requester(x)]}] + /\ alive' = alive \cup {<>} + /\ nextPid' = nextPid + 1 + /\ counts' = [counts EXCEPT !.call = @ + 1] + /\ UNCHANGED <> + +\* an update called on node x for the process x shows under nm, which runs on node y +UpdateCall(x, y) == + /\ counts.call < MaxRemoteCall + /\ x # y + /\ Connected(x, y) + /\ up[y] + /\ \E nm \in Names: + /\ Shown(x, nm) # NONE + /\ Shown(x, nm).pid[1] = y + /\ pcall' = [pcall EXCEPT ![y] = @ \cup {[id |-> counts.call + 1, kind |-> "update", name |-> nm, + pid |-> Shown(x, nm).pid, caller |-> x, requester |-> Requester(x)]}] + /\ counts' = [counts EXCEPT !.call = @ + 1] + /\ UNCHANGED <> + +Update(n) == + /\ counts.upd < MaxUpdate + /\ up[n] + /\ \E nm \in Names: + /\ own[n][nm] # NONE + /\ own[n][nm].pid \in alive + /\ \E t \in NextTimes(own[n][nm]): + LET c == [own[n][nm] EXCEPT !.time = t, !.ver = @ + 1] + IN Commit(n, Resolve(Broadcast([St(n) EXCEPT !.own[nm] = c], ClaimMsg(n, nm, c), {}), n, nm), + chan, [counts EXCEPT !.upd = @ + 1]) + /\ UNCHANGED <> -\* where a DOWN can land in the inbox of o: at the end, or with DownOvertakes anywhere, -\* ahead of messages the other node sent before it -DownPositions(o) == - IF DownOvertakes THEN 1..(Len(inbox[o]) + 1) ELSE {Len(inbox[o]) + 1} +Unregister(n) == + /\ counts.unreg < MaxUnregister + /\ up[n] + /\ \E nm \in Names: + /\ own[n][nm] # NONE + /\ Commit(n, Withdraw(St(n), n, nm, "normal"), chan, [counts EXCEPT !.unreg = @ + 1]) + /\ removed' = removed \cup {nm} + /\ UNCHANGED <> + +\* a process exits; if it is registered, its node's scope process hears of it, if it is running +Exit(p) == + /\ counts.exit < MaxExit + /\ p \in alive + /\ LET names == {nm \in Names: own[p[1]][nm] # NONE /\ own[p[1]][nm].pid = p} + IN /\ pexit' = IF names # {} /\ up[p[1]] THEN [pexit EXCEPT ![p[1]] = @ \cup {p}] ELSE pexit + /\ removed' = removed \cup names + /\ alive' = alive \ {p} + /\ counts' = [counts EXCEPT !.exit = @ + 1] + /\ UNCHANGED <> + +\* the connection between n and m drops: messages already in a mailbox stay, messages on the wire are lost, and +\* messages a sender hasn't sent yet stay in its queue +Disconnect(n, m) == + /\ counts.disc < MaxDisconnect + /\ {n, m} \in CutPairs + /\ Connected(n, m) + /\ LET k1 == <> + k2 == <> + \* the sender of a scope process that restarted is gone: what it left is in a mailbox or on the wire + olds == {k \in ChanKeys: ((k[1] = n /\ k[3] = m) \/ (k[1] = m /\ k[3] = n)) /\ k # k1 /\ k # k2 /\ Len(chan[k]) > dlv[k]} + IN \E a1, b1 \in dlv[k1]..Len(chan[k1]), a2, b2 \in dlv[k2]..Len(chan[k2]), keep \in [olds -> 0..Longest(olds)]: + /\ a1 <= b1 /\ a2 <= b2 + /\ \A k \in olds: dlv[k] <= keep[k] /\ keep[k] <= Len(chan[k]) + \* nothing reaches the mailbox of a scope process that isn't running + /\ ~up[m] => a1 = 0 + /\ ~up[n] => a2 = 0 + /\ \A k \in olds: ~up[k[3]] => keep[k] = 0 + /\ chan' = [k \in ChanKeys |-> + IF k = k1 THEN SubSeq(chan[k], 1, a1) \o SubSeq(chan[k], b1 + 1, Len(chan[k])) + ELSE IF k = k2 THEN SubSeq(chan[k], 1, a2) \o SubSeq(chan[k], b2 + 1, Len(chan[k])) + ELSE IF k \in olds THEN SubSeq(chan[k], 1, keep[k]) + ELSE chan[k]] + /\ dlv' = [k \in ChanKeys |-> + IF k = k1 THEN a1 + ELSE IF k = k2 THEN a2 + ELSE IF k \in olds THEN keep[k] + ELSE dlv[k]] + /\ conn' = conn \ {{n, m}} + /\ pdown' = [o \in Nodes |-> + IF o \in {n, m} /\ up[o] /\ known[o][IF o = n THEN m ELSE n] # NONE + THEN pdown[o] \cup {known[o][IF o = n THEN m ELSE n]} + ELSE pdown[o]] + /\ \E keptN \in SUBSET {c \in pcall[n]: c.caller = m}, keptM \in SUBSET {c \in pcall[m]: c.caller = n}: + pcall' = [o \in Nodes |-> + IF o = n THEN {c \in pcall[n]: c.caller # m} \cup keptN + ELSE IF o = m THEN {c \in pcall[m]: c.caller # n} \cup keptM + ELSE pcall[o]] + \* an answer to a request can't come over the dropped connection: both stop waiting for one + /\ asked' = [asked EXCEPT ![n] = @ \ {m}, ![m] = @ \ {n}] + /\ counts' = [counts EXCEPT !.disc = @ + 1] + /\ UNCHANGED <> + +Reconnect(n, m) == + /\ n # m + /\ ~Connected(n, m) + /\ conn' = conn \cup {{n, m}} + /\ pnodeup' = [pnodeup EXCEPT ![n] = IF up[n] THEN @ \cup {m} ELSE @, ![m] = IF up[m] THEN @ \cup {n} ELSE @] + /\ UNCHANGED <> + +\* the scope process of n is killed: its mailbox is lost, and so is what its sender hadn't sent yet; the nodes +\* watching it get a DOWN. Syn's tables, and so the claims n holds, survive +Crash(n) == + /\ counts.restart < MaxRestart + /\ n \in RestartNodes + /\ up[n] + /\ LET old == Me(n) + outgoing == {k \in ChanKeys: k[1] = n /\ k[2] = inc[n] /\ k[3] # n /\ Len(chan[k]) > dlv[k]} + incoming == {k \in ChanKeys: k[3] = n /\ k[1] # n} + \* while connected, any number of the messages sent to n can be in its mailbox already + open == {k \in incoming: Connected(k[1], n) /\ Len(chan[k]) > dlv[k]} + IN /\ \E keep \in [outgoing -> 0..Longest(outgoing)], lost \in [open -> 0..Longest(open)]: + /\ \A k \in outgoing: dlv[k] <= keep[k] /\ keep[k] <= Len(chan[k]) + /\ \A k \in open: dlv[k] <= lost[k] /\ lost[k] <= Len(chan[k]) + /\ chan' = [k \in ChanKeys |-> + IF k \in outgoing THEN SubSeq(chan[k], 1, keep[k]) + ELSE IF k \in open THEN SubSeq(chan[k], lost[k] + 1, Len(chan[k])) + ELSE IF k \in incoming THEN SubSeq(chan[k], dlv[k] + 1, Len(chan[k])) + ELSE chan[k]] + /\ dlv' = [k \in ChanKeys |-> IF k \in incoming THEN 0 ELSE dlv[k]] + /\ pdown' = [o \in Nodes |-> + IF o = n THEN {} + ELSE IF up[o] /\ known[o][n] = old THEN pdown[o] \cup {old} + ELSE pdown[o]] + /\ up' = [up EXCEPT ![n] = FALSE] + /\ pexit' = [pexit EXCEPT ![n] = {}] + /\ pnodeup' = [pnodeup EXCEPT ![n] = {}] + /\ pcall' = [pcall EXCEPT ![n] = {}] + /\ counts' = [counts EXCEPT !.restart = @ + 1] + /\ UNCHANGED <> + +\* a new scope process starts on n: it keeps the claims of n's processes that are still alive, forgets the other +\* nodes, and asks the connected ones for their data +Start(n) == + /\ ~up[n] + /\ LET new == inc[n] + 1 + peers == {m \in Nodes \ {n}: Connected(n, m)} + IN /\ inc' = [inc EXCEPT ![n] = new] + /\ chan' = [k \in ChanKeys |-> + IF k[1] = n /\ k[2] = new /\ k[3] \in peers THEN <<[type |-> "discover", from |-> <>]>> + ELSE chan[k]] + /\ asked' = [asked EXCEPT ![n] = peers] + /\ up' = [up EXCEPT ![n] = TRUE] + /\ own' = [own EXCEPT ![n] = [nm \in Names |-> IF @[nm] # NONE /\ @[nm].pid \in alive THEN @[nm] ELSE NONE]] + /\ remote' = [remote EXCEPT ![n] = [m \in Nodes |-> NoClaims]] + /\ known' = [known EXCEPT ![n] = [m \in Nodes |-> NONE]] + /\ UNCHANGED <> + +----------------------------------------------------------------------------- Init == - /\ inbox = [n \in Nodes |-> <<>>] - /\ registered = [n \in Names |-> 0] - /\ locally_registered = [n1 \in Nodes |-> [n2 \in Nodes |-> <<>>]] - /\ names = [n \in Nodes |-> Names] - /\ disconnections = 0 - /\ visible_nodes = [n \in Nodes |-> AllOtherNodes(n)] - /\ known_nodes = [n \in Nodes |-> AllOtherNodes(n)] - /\ time = 0 - -\* register a process that lives on n, called from n itself or from a node that can see n -Register(n) == - /\ LET available_names == names[n] \ AllRegisteredForNode(locally_registered[n]) - IN available_names # {} - /\ \E requester \in {n} \union visible_nodes[n]: - LET next_val == CHOOSE o \in available_names: TRUE - IN inbox' = [inbox EXCEPT![n] = Append(inbox[n], [action |-> "register_or_update_on_node", name |-> next_val, requester |-> requester])] - /\ names' = [names EXCEPT![n] = names[n] \ {next_val}] - /\ time' = time - /\ UNCHANGED <> - -RegisterOrUpdateOnNode(n) == - /\ Len(inbox[n]) > 0 - /\ Head(inbox[n]).action = "register_or_update_on_node" - /\ LET message == Head(inbox[n]) - l == [locally_registered[n] EXCEPT![n] = locally_registered[n][n] @@ [r \in {message.name} |-> time]] - already_registered == message.name \in AllRegisteredForNode(locally_registered[n]) - IN - (IF already_registered THEN - \* {error, taken} - registered' = registered - /\ locally_registered' = locally_registered - /\ inbox' = [inbox EXCEPT![n] = Tail(inbox[n])] - ELSE - registered' = [registered EXCEPT![message.name] = @ + 1] - /\ locally_registered' = [locally_registered EXCEPT![n] = l] - /\ inbox' = [o \in Nodes |-> IF o = n THEN Tail(inbox[n]) ELSE Outgoing(o, n, message.requester, - [action |-> "register_reply", name |-> message.name, from |-> n, time |-> time], - [action |-> "sync_register", name |-> message.name, from |-> n, time |-> time] - )] - ) - /\ time' = time + 1 - /\ UNCHANGED <> - -Merge(left, right) == - LET to_keep == {name \in DOMAIN left : (name \notin DOMAIN right \/ left[name] > right[name])} - IN [name \in to_keep |-> left[name]] - -RECURSIVE Flatten(_, _, _) - -Flatten(keys, struct, acc) == - IF keys = {} THEN acc - ELSE - LET k == CHOOSE k \in keys: TRUE - IN Flatten(keys \ {k}, struct, acc @@ struct[k]) - -\* Helper: append sync_unregister messages for each name in the set -RECURSIVE AppendSyncUnregisters(_, _, _) - -AppendSyncUnregisters(names_to_unregister, queue, from_node) == - IF names_to_unregister = {} THEN queue - ELSE - LET name == CHOOSE name \in names_to_unregister: TRUE - IN AppendSyncUnregisters( - names_to_unregister \ {name}, - Append(queue, [action |-> "sync_unregister", name |-> name, from |-> from_node]), - from_node - ) - -MergeRegistries(local, remote, remote_node) == - LET all_registered == Flatten(DOMAIN local, local, << >>) - IN [r \in DOMAIN local |-> CASE - (r = remote_node) -> local[r] @@ Merge(remote, all_registered) - [] OTHER -> Merge(local[r], remote) - ] - -MergeRegistrySnapshot(local, remote, remote_node) == - LET all_registered == Flatten(DOMAIN local \ {remote_node}, local, << >>) - IN [r \in DOMAIN local |-> CASE - (r = remote_node) -> Merge(remote, all_registered) - [] OTHER -> Merge(local[r], remote) - ] - -\* the reply to a register call this node made for a process on another node is applied like a -\* sync_register, conflict resolution included, and then the call returns -SyncRegister(n) == - /\ Len(inbox[n]) > 0 - /\ Head(inbox[n]).action \in {"sync_register", "register_reply"} - /\ LET message == Head(inbox[n]) - known_sender == message.from \in known_nodes[n] - conflict == message.name \in DOMAIN locally_registered[n][n] - remote_wins == conflict /\ message.time > locally_registered[n][n][message.name] - losers == IF remote_wins THEN {message.name} ELSE {} - l == MergeRegistries(locally_registered[n], [r \in {message.name} |-> message.time], message.from) - IN - (IF known_sender THEN - locally_registered' = [locally_registered EXCEPT![n] = l] - /\ registered' = (IF remote_wins THEN [registered EXCEPT![message.name] = @ - 1] ELSE registered) - /\ inbox' = [o \in Nodes |-> CASE - (o = n) -> Tail(inbox[n]) - [] (o \in ConnectedKnownNodes(n)) -> AppendSyncUnregisters(losers, inbox[o], n) - [] OTHER -> inbox[o] - ] - ELSE - \* sync_register is ignored until the remote scope node is discovered - locally_registered' = locally_registered - /\ registered' = registered - /\ inbox' = [inbox EXCEPT![n] = Tail(inbox[n])] - ) - /\ time' = time - /\ UNCHANGED <> - -ItemToRemove(n) == - CHOOSE r \in DOMAIN locally_registered[n][n]: TRUE - -\* unregister a name held by a process on n, called from n itself or from a node that can see n -\* and has the name in its own table -\* only one unregister per name can be waiting: more would just fail with {error, undefined}, and -\* without this limit the inbox can grow forever and the model checker never finishes -Unregister(n) == - /\ \E requester \in {n} \union visible_nodes[n]: - /\ Cardinality(DOMAIN locally_registered[requester][n]) > 0 - /\ LET item_to_remove == CHOOSE r \in DOMAIN locally_registered[requester][n]: TRUE - IN (~\E i \in 1..Len(inbox[n]): inbox[n][i].action = "unregister_on_node" /\ inbox[n][i].name = item_to_remove) - /\ inbox' = [inbox EXCEPT![n] = Append(inbox[n], [action |-> "unregister_on_node", name |-> item_to_remove, requester |-> requester])] - /\ time' = time - /\ UNCHANGED <> - -UnregisterOnNode(n) == - /\ Len(inbox[n]) > 0 - /\ Head(inbox[n]).action = "unregister_on_node" - /\ LET message == Head(inbox[n]) - l == [r \in (DOMAIN locally_registered[n][n] \ {message.name}) |-> locally_registered[n][n][r]] - already_removed == message.name \notin RegisteredOnThisNode(n) - IN - (IF already_removed THEN - \* {error, undefined} - registered' = registered - /\ locally_registered' = locally_registered - /\ inbox' = [inbox EXCEPT![n] = Tail(inbox[n])] - /\ names' = names - ELSE - registered' = [registered EXCEPT![message.name] = @ - 1] - /\ locally_registered' = [locally_registered EXCEPT![n] = ([locally_registered[n] EXCEPT![n] = l])] - /\ inbox' = [o \in Nodes |-> IF o = n THEN Tail(inbox[n]) ELSE Outgoing(o, n, message.requester, - [action |-> "unregister_reply", name |-> message.name, from |-> n], - [action |-> "sync_unregister", name |-> message.name, from |-> n] - )] - ) - /\ time' = time - /\ UNCHANGED <> - -\* the reply to an unregister call this node made for a process on another node is applied like a -\* sync_unregister, and then the call returns -SyncUnregister(n) == - /\ Len(inbox[n]) > 0 - /\ Head(inbox[n]).action \in {"sync_unregister", "unregister_reply"} - /\ LET message == Head(inbox[n]) - l == [r \in (DOMAIN locally_registered[n][message.from] \ {message.name}) |-> locally_registered[n][message.from][r]] - IN locally_registered' = [locally_registered EXCEPT![n] = [locally_registered[n] EXCEPT![message.from] = l]] - /\ inbox' = [inbox EXCEPT![n] = Tail(inbox[n])] - /\ time' = time - /\ UNCHANGED <> - -ProcessDown(n) == - /\ Cardinality(DOMAIN locally_registered[n][n]) > 0 - /\ LET item_to_remove == ItemToRemove(n) - l == [r \in (DOMAIN locally_registered[n][n] \ {item_to_remove}) |-> locally_registered[n][n][r]] - IN registered' = [registered EXCEPT![item_to_remove] = @ - 1] - /\ locally_registered' = [locally_registered EXCEPT![n] = ([locally_registered[n] EXCEPT![n] = l])] - /\ inbox' = [o \in Nodes |-> CASE - (o \in ConnectedKnownNodes(n)) -> Append(inbox[o], [action |-> "sync_unregister", name |-> item_to_remove, from |-> n]) - [] OTHER -> inbox[o] - ] - /\ time' = time - /\ UNCHANGED <> - -Disconnect(n) == - /\ disconnections < MaxDisconnections - /\ Cardinality(visible_nodes[n]) > 0 - /\ LET other_node == CHOOSE o \in visible_nodes[n]: TRUE - IN visible_nodes' = [o \in Nodes |-> CASE - (o = other_node) -> visible_nodes[o] \ {n} - [] (o = n) -> visible_nodes[o] \ {other_node} - [] OTHER -> visible_nodes[o] - ] - /\ \E i \in DownPositions(n), j \in DownPositions(other_node): - inbox' = [o \in Nodes |-> CASE - (o = n) -> InsertAt(inbox[o], i, [action |-> "DOWN", from |-> other_node]) - [] (o = other_node) -> InsertAt(inbox[o], j, [action |-> "DOWN", from |-> n]) - [] OTHER -> inbox[o] - ] - /\ disconnections' = disconnections + 1 - /\ time' = time - /\ UNCHANGED <> - -Reconnect(n) == - /\ Cardinality(AllOtherNodes(n) \ visible_nodes[n]) > 0 - /\ LET other_node == CHOOSE o \in (AllOtherNodes(n) \ visible_nodes[n]): TRUE - IN visible_nodes' = [o \in Nodes |-> CASE - (o = other_node) -> visible_nodes[o] \union {n} - [] (o = n) -> visible_nodes[o] \union {other_node} - [] OTHER -> visible_nodes[o] - ] - /\ inbox' = [o \in Nodes |-> CASE - (o = n) -> Append(inbox[o], [action |-> "discover", from |-> other_node]) - [] (o = other_node) -> Append(inbox[o], [action |-> "discover", from |-> n]) - [] OTHER -> inbox[o] - ] - /\ time' = time - /\ UNCHANGED <> - -Discover(n) == - /\ Len(inbox[n]) > 0 - /\ LET message == Head(inbox[n]) - can_discover == message.from \in visible_nodes[n] - IN message.action = "discover" - /\ inbox' = [o \in Nodes |-> CASE - (o = n) -> Tail(inbox[o]) - [] (can_discover /\ o = message.from) -> Append(inbox[o], [action |-> "ack_sync", local_data |-> locally_registered[n][n], from |-> n]) - [] OTHER -> inbox[o] - ] - /\ known_nodes' = (IF can_discover THEN [known_nodes EXCEPT![n] = @ \union {message.from}] ELSE known_nodes) - /\ time' = time - /\ UNCHANGED <> - -AckSync(n) == - /\ Len(inbox[n]) > 0 - /\ Head(inbox[n]).action = "ack_sync" - /\ LET message == Head(inbox[n]) - can_sync == message.from \in visible_nodes[n] - l == MergeRegistrySnapshot(locally_registered[n], message.local_data, message.from) - conflicts == DOMAIN locally_registered[n][n] \intersect DOMAIN message.local_data - losers == { r \in conflicts : message.local_data[r] > locally_registered[n][n][r] } - c1 == [c \in losers |-> registered[c] - 1] - c2 == [c \in { r \in conflicts : locally_registered[n][n][r] > message.local_data[r] } |-> registered[c]] - IN - (IF can_sync THEN - locally_registered' = [locally_registered EXCEPT![n] = l] - /\ registered' = c1 @@ c2 @@ [r \in (DOMAIN registered \ conflicts) |-> registered[r]] - /\ known_nodes' = [known_nodes EXCEPT![n] = @ \union {message.from}] - ELSE - locally_registered' = locally_registered - /\ registered' = registered - /\ known_nodes' = known_nodes - ) - /\ inbox' = [o \in Nodes |-> CASE - (o = n) -> Tail(inbox[n]) - [] (can_sync /\ o \in ConnectedKnownNodes(n)) -> AppendSyncUnregisters(losers, inbox[o], n) - [] OTHER -> inbox[o] - ] - /\ time' = time - /\ UNCHANGED <> - -Down(n) == - /\ Len(inbox[n]) > 0 - /\ Head(inbox[n]).action = "DOWN" - /\ inbox' = [inbox EXCEPT![n] = Tail(inbox[n])] - /\ LET message == Head(inbox[n]) - l == [locally_registered[n] EXCEPT![message.from] = <<>>] - IN locally_registered' = [locally_registered EXCEPT![n] = l] - /\ known_nodes' = [known_nodes EXCEPT![n] = known_nodes[n] \ {message.from}] - /\ time' = time - /\ UNCHANGED <> - -Complete(n) == - /\ LET available_names == names[n] \ AllRegisteredForNode(locally_registered[n]) - IN available_names = {} - /\ UNCHANGED <> - -Next == - /\ \E n \in Nodes: - \/ Register(n) - \/ RegisterOrUpdateOnNode(n) - \/ SyncRegister(n) - \/ Unregister(n) - \/ UnregisterOnNode(n) - \/ SyncUnregister(n) - \/ ProcessDown(n) - \/ Disconnect(n) - \/ Reconnect(n) - \/ Discover(n) - \/ AckSync(n) - \/ Down(n) - \/ Complete(n) - -Spec == Init /\ [][Next]_vars - -AllRegistered == - \A n \in Nodes: - LET reg == {r \in DOMAIN registered: registered[r] > 0} - IN (\A o \in Nodes: Len(inbox[o]) = 0) /\ visible_nodes[n] = AllOtherNodes(n) => AllRegisteredForNode(locally_registered[n]) = reg - -RECURSIVE Duplicates(_, _, _) - -Duplicates(keys, struct, acc) == - IF Cardinality(keys) < 2 THEN acc - ELSE - LET k1 == CHOOSE k \in keys: TRUE - k2 == CHOOSE k \in (keys \ {k1}): TRUE - duplicates == DOMAIN struct[k1] \intersect DOMAIN struct[k2] - IN Duplicates(keys \ {k1}, struct, duplicates \union acc) - -\* when all messages are processed and all nodes are connected, every node has each name on the -\* node that actually runs it -ViewsMatchOwners == - (\A o \in Nodes: Len(inbox[o]) = 0) /\ (\A n \in Nodes: visible_nodes[n] = AllOtherNodes(n)) => - \A n, o \in Nodes: DOMAIN locally_registered[n][o] = DOMAIN locally_registered[o][o] - -\* n's scope handles the reply to a register call n made for a process on another node -RegisterReplyHandled(n) == - /\ Len(inbox[n]) > 0 - /\ Head(inbox[n]).action = "register_reply" - /\ inbox'[n] = Tail(inbox[n]) - -\* the call returns ok if n now has the name on that node, or if n doesn't know that node yet and so -\* doesn't apply the reply; otherwise n resolved a clash against it and the call returns race_condition -RemoteRegisterOk(n) == - LET message == Head(inbox[n]) - IN message.from \notin known_nodes[n] \/ message.name \in DOMAIN locally_registered'[n][message.from] - -\* a register call made for a process on another node that returns ok has the name in the caller's table -ReadYourWrites == - [][\A n \in Nodes: RegisterReplyHandled(n) /\ RemoteRegisterOk(n) => - Head(inbox[n]).name \in DOMAIN locally_registered'[n][Head(inbox[n]).from]]_vars - -ThereCanBeOnlyOne == - \A n \in Nodes: - Duplicates(DOMAIN locally_registered[n], locally_registered[n], {}) = {} - -AllMessagesProcessed == - \A n \in Nodes: - <>(Len(inbox[n]) = 0) -==== + /\ alive = {} + /\ own = [n \in Nodes |-> NoClaims] + /\ remote = [n \in Nodes |-> [m \in Nodes |-> NoClaims]] + /\ known = [n \in Nodes |-> [m \in Nodes |-> IF m = n THEN NONE ELSE <>]] + /\ asked = [n \in Nodes |-> {}] + /\ inc = [n \in Nodes |-> 1] + /\ conn = {{n, m}: n \in Nodes, m \in Nodes} \ {{n}: n \in Nodes} + /\ chan = [k \in ChanKeys |-> <<>>] + /\ dlv = [k \in ChanKeys |-> 0] + /\ pdown = [n \in Nodes |-> {}] + /\ pexit = [n \in Nodes |-> {}] + /\ pnodeup = [n \in Nodes |-> {}] + /\ pcall = [n \in Nodes |-> {}] + /\ up = [n \in Nodes |-> TRUE] + /\ nextPid = 1 + /\ counts = [reg |-> 0, call |-> 0, upd |-> 0, unreg |-> 0, exit |-> 0, disc |-> 0, restart |-> 0, created |-> 0] + /\ resMsgs = 0 + /\ held = {} + /\ removed = {} + +Internal == + \/ \E k \in ChanKeys: Receive(k) \/ Lose(k) + \/ \E n \in Nodes: + \/ \E s \in pdown[n]: ReceiveDown(n, s) + \/ \E p \in pexit[n]: ReceiveExit(n, p) + \/ \E m \in pnodeup[n]: ReceiveNodeup(n, m) + \/ \E c \in pcall[n]: ReceiveCall(n, c) + +External == + \/ \E n \in Nodes: Register(n) \/ Update(n) \/ Unregister(n) \/ Crash(n) \/ Start(n) + \/ \E x, y \in Nodes: RegisterCall(x, y) \/ UpdateCall(x, y) \/ Disconnect(x, y) \/ Reconnect(x, y) + \/ \E p \in alive: Exit(p) + +Next == Internal \/ External + +Fairness == + /\ \A k \in ChanKeys: WF_vars(Receive(k)) /\ WF_vars(Lose(k)) + /\ \A n \in Nodes: + /\ WF_vars(\E s \in pdown[n]: ReceiveDown(n, s)) + /\ WF_vars(\E p \in pexit[n]: ReceiveExit(n, p)) + /\ WF_vars(\E m \in pnodeup[n]: ReceiveNodeup(n, m)) + /\ WF_vars(\E c \in pcall[n]: ReceiveCall(n, c)) + +Spec == Init /\ [][Next]_vars /\ Fairness + +----------------------------------------------------------------------------- +(* Properties *) + +\* nothing left to deliver or handle +Quiet == + /\ \A k \in ChanKeys: chan[k] = <<>> + /\ \A n \in Nodes: pdown[n] = {} /\ pexit[n] = {} /\ pnodeup[n] = {} /\ pcall[n] = {} + +AllConnected == \A n, m \in Nodes: n # m => Connected(n, m) +AllUp == \A n \in Nodes: up[n] + +\* the bookkeeping of what reached a mailbox stays within the channel +DeliveredInChannel == \A k \in ChanKeys: dlv[k] <= Len(chan[k]) + +\* once things settle, each running scope process knows the current scope process of every node it's connected to +\* that runs one, and has an exact copy of that node's claims; it has nothing from the other nodes +Membership == + Quiet => \A n, m \in Nodes: + n # m /\ up[n] => + IF Connected(n, m) /\ up[m] + THEN known[n][m] = Me(m) /\ remote[n][m] = own[m] + ELSE known[n][m] = NONE /\ remote[n][m] = NoClaims + +\* once things settle with every node connected and running: all nodes show the same claim for each name, the claim +\* shown is still held by its node, and a name that some node holds is shown everywhere +Agreement == + Quiet /\ AllConnected /\ AllUp => + \A nm \in Names: + /\ \A n, m \in Nodes: Shown(n, nm) = Shown(m, nm) + /\ \A n \in Nodes: Shown(n, nm) # NONE => own[Shown(n, nm).pid[1]][nm] = Shown(n, nm) + /\ (\E m \in Nodes: own[m][nm] # NONE) => \A n \in Nodes: Shown(n, nm) # NONE + +\* once things settle with every node connected and running, at most one node holds each name +OneOwner == + Quiet /\ AllConnected /\ AllUp => \A nm \in Names: Cardinality({m \in Nodes: own[m][nm] # NONE}) <= 1 + +\* conflicts only ever make a node drop its claim, and each claim is dropped at most once: no endless back and forth +ResolutionBounded == resMsgs <= counts.created + +\* with a resolver that compares things that don't change during the conflict, a name that got registered, and +\* whose processes were never unregistered and never exited, is still held by someone once things settle +SurvivorResolver == + \/ Resolver = "oldest" + \/ Resolver = "newest" /\ (MonotonicClocks \/ Cardinality(Times) = 1) + \/ Resolver = "second" /\ Cardinality(Times) = 1 +Survivor == + SurvivorResolver /\ Quiet => \A nm \in held \ removed: \E m \in Nodes: own[m][nm] # NONE + +\* message traffic always stops, and stays stopped, once outside actions stop +Termination == <>[]Quiet + +============================================================================= diff --git a/test/tla/syn_backward.cfg b/test/tla/syn_backward.cfg new file mode 100644 index 00000000..7abc5a32 --- /dev/null +++ b/test/tla/syn_backward.cfg @@ -0,0 +1,37 @@ +\* Syn's default resolver with a clock that can move backwards during the conflict: the nodes must +\* still agree, even if no winner is left. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1, 2} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 1 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_calls.cfg b/test/tla/syn_calls.cfg new file mode 100644 index 00000000..b1475dc2 --- /dev/null +++ b/test/tla/syn_calls.cfg @@ -0,0 +1,37 @@ +\* A registration or update called from another node, with clocks that disagree. The resolver keeps +\* the older time, and an update changes the time, so a claim can lose its conflict right after an update. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1, 2} + Resolver = "oldest_time" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 1 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_calls_two.cfg b/test/tla/syn_calls_two.cfg new file mode 100644 index 00000000..de31c46c --- /dev/null +++ b/test/tla/syn_calls_two.cfg @@ -0,0 +1,36 @@ +\* Two registrations or updates called from other nodes at the same time. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 2 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_conflict.cfg b/test/tla/syn_conflict.cfg new file mode 100644 index 00000000..8958f2b7 --- /dev/null +++ b/test/tla/syn_conflict.cfg @@ -0,0 +1,37 @@ +\* Issue #96: the connection between nodes 1 and 2 drops twice while both stay connected to node 3, +\* and the resolver keeps the process started first. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "oldest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 2 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_cyclic.cfg b/test/tla/syn_cyclic.cfg new file mode 100644 index 00000000..bbbff2ab --- /dev/null +++ b/test/tla/syn_cyclic.cfg @@ -0,0 +1,37 @@ +\* Three nodes register the same name, with a resolver where no process beats both others: every +\* node must still agree, even if all three drop it. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "cyclic" + MonotonicClocks = FALSE + RegisterNodes = {1, 2, 3} + CutPairs = {{1, 2}, {1, 3}, {2, 3}} + RestartNodes = {} + MaxRegister = 3 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_down_overtakes.cfg b/test/tla/syn_down_overtakes.cfg deleted file mode 100644 index 7305a7cd..00000000 --- a/test/tla/syn_down_overtakes.cfg +++ /dev/null @@ -1,21 +0,0 @@ -\* a DOWN can arrive ahead of messages the other node sent before it -\* ReadYourWrites is left out: here it doesn't hold, see README.md -INIT Init -NEXT Next - -CONSTANTS -Nodes = {n1, n2} -Names = {a} -MaxDisconnections = 1 -DownOvertakes = TRUE - -SYMMETRY -Symmetry - -INVARIANTS -AllRegistered -ThereCanBeOnlyOne -ViewsMatchOwners - -PROPERTIES -AllMessagesProcessed diff --git a/test/tla/syn_restart.cfg b/test/tla/syn_restart.cfg new file mode 100644 index 00000000..c95e9550 --- /dev/null +++ b/test/tla/syn_restart.cfg @@ -0,0 +1,36 @@ +\* The scope process of node 2 restarts and the connection between nodes 1 and 2 drops once. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {2} + MaxRegister = 1 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 1 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_restart_call.cfg b/test/tla/syn_restart_call.cfg new file mode 100644 index 00000000..c19d00cb --- /dev/null +++ b/test/tla/syn_restart_call.cfg @@ -0,0 +1,37 @@ +\* Two nodes: the scope process of node 2 restarts, the connection drops once, and a registration is +\* called from the other node. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {2} + MaxRegister = 1 + MaxRemoteCall = 1 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 1 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_second.cfg b/test/tla/syn_second.cfg new file mode 100644 index 00000000..e46c31ae --- /dev/null +++ b/test/tla/syn_second.cfg @@ -0,0 +1,37 @@ +\* Two nodes register the same name with the same time while their connection is down, and the resolver keeps +\* the claim passed second on a tie: both nodes pass the claims in node order, so they keep the same process. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "second" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_skew.cfg b/test/tla/syn_skew.cfg new file mode 100644 index 00000000..a22febc7 --- /dev/null +++ b/test/tla/syn_skew.cfg @@ -0,0 +1,37 @@ +\* Clocks that disagree with Syn's default resolver, and an update during the conflict: the name must +\* still end up with a winner. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1, 2} + Resolver = "newest" + MonotonicClocks = TRUE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 1 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_termination.cfg b/test/tla/syn_termination.cfg new file mode 100644 index 00000000..63f74e64 --- /dev/null +++ b/test/tla/syn_termination.cfg @@ -0,0 +1,34 @@ +\* Message traffic always stops once outside actions stop: two nodes register the same name, and any +\* connection can drop once. + +SPECIFICATION Spec + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}, {1, 3}, {2, 3}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + Membership + +PROPERTIES + Termination diff --git a/test/tla/syn_termination_exits.cfg b/test/tla/syn_termination_exits.cfg new file mode 100644 index 00000000..6a7e344f --- /dev/null +++ b/test/tla/syn_termination_exits.cfg @@ -0,0 +1,37 @@ +\* Message traffic always stops: two nodes, an unregistration and a process exiting. + +SPECIFICATION Spec + +CONSTANTS + NONE = NONE + Nodes = {1, 2} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 1 + MaxExit = 1 + MaxDisconnect = 0 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + Membership + Agreement + OneOwner + ResolutionBounded + Survivor + +PROPERTIES + Termination diff --git a/test/tla/syn_termination_neither.cfg b/test/tla/syn_termination_neither.cfg new file mode 100644 index 00000000..15e63002 --- /dev/null +++ b/test/tla/syn_termination_neither.cfg @@ -0,0 +1,37 @@ +\* Message traffic always stops: three nodes register the same name and the resolver keeps +\* none of them. + +SPECIFICATION Spec + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "neither" + MonotonicClocks = FALSE + RegisterNodes = {1, 2, 3} + CutPairs = {{1, 2}, {1, 3}, {2, 3}} + RestartNodes = {} + MaxRegister = 3 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 0 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + Membership + Agreement + OneOwner + ResolutionBounded + +PROPERTIES + Termination diff --git a/test/tla/syn_termination_restart.cfg b/test/tla/syn_termination_restart.cfg new file mode 100644 index 00000000..8eea5dbe --- /dev/null +++ b/test/tla/syn_termination_restart.cfg @@ -0,0 +1,37 @@ +\* Message traffic always stops: two nodes, a scope process restart and a dropped connection. + +SPECIFICATION Spec + +CONSTANTS + NONE = NONE + Nodes = {1, 2} + Names = {a} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {2} + MaxRegister = 1 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 1 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + Membership + Agreement + OneOwner + ResolutionBounded + Survivor + +PROPERTIES + Termination diff --git a/test/tla/syn_tests.cfg b/test/tla/syn_tests.cfg deleted file mode 100644 index 3195db98..00000000 --- a/test/tla/syn_tests.cfg +++ /dev/null @@ -1,8 +0,0 @@ -INIT I -NEXT N - -CONSTANTS -Nodes = {} -Names = {} -MaxDisconnections = 0 -DownOvertakes = FALSE diff --git a/test/tla/syn_tests.tla b/test/tla/syn_tests.tla deleted file mode 100644 index ccec77c6..00000000 --- a/test/tla/syn_tests.tla +++ /dev/null @@ -1,76 +0,0 @@ ----- MODULE syn_tests ---- -EXTENDS syn - -VARIABLE x - -I == FALSE /\ x = 0 -N == FALSE /\ x' = x - -\* AllRegisteredForNode - -ASSUME LET l == ("n1" :> <<>> @@ "n2" :> [a |-> 0, b |-> 2] @@ "n3" :> [c |-> 1]) - res == AllRegisteredForNode(l) - IN res = {"a", "b", "c"} - -\* AllRegisteredNames - -ASSUME LET l == ("n1" :> ("n1" :> [a |-> 1] @@ "n2" :> <<>> @@ "n3" :> <<>>) @@ "n2" :> ("n1" :> [a |-> 1] @@ "n2" :> [b |-> 2] @@ "n3" :> <<>>) @@ "n3" :> ("n1" :> <<>> @@ "n2" :> <<>> @@ "n3" :> [c |-> 3])) - res == AllRegisteredNames({"n2","n3"}, l, {}) - IN res = {"b", "c"} - -\* Duplicates - -ASSUME LET l == ("n1" :> <<>> @@ "n2" :> [a |-> 0] @@ "n3" :> [a |-> 1]) - res == Duplicates(DOMAIN l, l, {}) - IN res = {"a"} - -\* MergeRegistries - -ASSUME LET local == ("n1" :> [a |-> 1] @@ "n2" :> <<>> @@ "n3" :> <<>>) - remote == [a |-> 0] - local_node == "n1" - remote_node == "n2" - res == MergeRegistries(local, remote, remote_node) - IN res = [n1 |-> [a |-> 1], n2 |-> << >>, n3 |-> <<>>] - -ASSUME LET local == ("n1" :> [a |-> 1] @@ "n2" :> <<>> @@ "n3" :> <<>>) - remote == [a |-> 2] - local_node == "n1" - remote_node == "n2" - res == MergeRegistries(local, remote, remote_node) - IN res = [n1 |-> << >>, n2 |-> [a |-> 2], n3 |-> <<>>] - -ASSUME LET local == ("n1" :> <<>> @@ "n2" :> <<>> @@ "n3" :> [a |-> 1]) - remote == [a |-> 2] - local_node == "n1" - remote_node == "n2" - res == MergeRegistries(local, remote, remote_node) - IN res = [n1 |-> << >>, n2 |-> [a |-> 2], n3 |-> <<>>] - -ASSUME LET local == ("n1" :> << >> @@ "n2" :> <<>> @@ "n3" :> ("a" :> 1)) - remote == ("a" :> 0) - local_node == "n1" - remote_node == "n2" - res == MergeRegistries(local, remote, remote_node) - IN res = [n1 |-> << >>, n2 |-> << >>, n3 |-> [a |-> 1]] - -ASSUME LET local == ("n1" :> << >> @@ "n2" :> << >> @@ "n3" :> ("a" :> 0)) - remote == [b |-> 1] - remote_node == "n3" - res == MergeRegistries(local, remote, remote_node) - IN res = [n1 |-> << >>, n2 |-> << >>, n3 |-> [a |-> 0, b |-> 1]] - -\* MergeRegistrySnapshot - -ASSUME LET local == ("n1" :> << >> @@ "n2" :> << >> @@ "n3" :> ("a" :> 0)) - remote == [b |-> 1] - remote_node == "n3" - res == MergeRegistrySnapshot(local, remote, remote_node) - IN res = [n1 |-> << >>, n2 |-> << >>, n3 |-> [b |-> 1]] - -ASSUME LET local == ("n1" :> << >> @@ "n2" :> << >> @@ "n3" :> ("a" :> 0)) - remote == << >> - remote_node == "n3" - res == MergeRegistrySnapshot(local, remote, remote_node) - IN res = [n1 |-> << >>, n2 |-> << >>, n3 |-> << >>] -==== diff --git a/test/tla/syn_three_owners.cfg b/test/tla/syn_three_owners.cfg new file mode 100644 index 00000000..e411e688 --- /dev/null +++ b/test/tla/syn_three_owners.cfg @@ -0,0 +1,38 @@ +\* Issue #96 on a larger cluster: three nodes register the same name and the resolver keeps the older process. +\* Node 3 can lose its own conflict to node 1, then show node 2's registration, which loses on node 2 to node 3's +\* process; the connection between nodes 2 and 3 can drop before or after node 2 drops its process. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a} + Times = {1} + Resolver = "oldest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2, 3} + CutPairs = {{2, 3}} + RestartNodes = {} + MaxRegister = 3 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 0 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor diff --git a/test/tla/syn_two_names.cfg b/test/tla/syn_two_names.cfg new file mode 100644 index 00000000..ee2e91ce --- /dev/null +++ b/test/tla/syn_two_names.cfg @@ -0,0 +1,36 @@ +\* Two names, a dropped connection and a process exiting. + +INIT Init +NEXT Next + +CONSTANTS + NONE = NONE + Nodes = {1, 2, 3} + Names = {a, b} + Times = {1} + Resolver = "newest" + MonotonicClocks = FALSE + RegisterNodes = {1, 2} + CutPairs = {{1, 2}} + RestartNodes = {} + MaxRegister = 2 + MaxRemoteCall = 0 + MaxUpdate = 0 + MaxUnregister = 0 + MaxExit = 1 + MaxDisconnect = 1 + MaxRestart = 0 + HideAfterConflict = FALSE + RefreshWinner = FALSE + WithdrawBeforeResult = FALSE + NoRediscover = FALSE + NoAskOnAdopt = FALSE + ResolveByRole = FALSE + +INVARIANTS + DeliveredInChannel + Membership + Agreement + OneOwner + ResolutionBounded + Survivor