diff --git a/api/api-archive-search/archive-search/src/main/resources/application-dev.yml b/api/api-archive-search/archive-search/src/main/resources/application-dev.yml index 87f50e20dcf..1353be2a69e 100644 --- a/api/api-archive-search/archive-search/src/main/resources/application-dev.yml +++ b/api/api-archive-search/archive-search/src/main/resources/application-dev.yml @@ -12,7 +12,7 @@ spring: mongodb: - uri: mongodb://mongod_dbuser_archivesearch:mongod_dbpwd_archivesearch@localhost:27018/archivesearch?connectTimeoutMS=2000 + uri: mongodb://mongod_dbuser_archivesearch:mongod_dbpwd_archivesearch@localhost:27018/archivesearch?connectTimeoutMS=2000&appName=vitamui-archive-search multipart: enabled: true diff --git a/api/api-collect/collect/src/main/resources/application-dev.yml b/api/api-collect/collect/src/main/resources/application-dev.yml index 0b8aa9e14ed..4f825c3b3f4 100644 --- a/api/api-collect/collect/src/main/resources/application-dev.yml +++ b/api/api-collect/collect/src/main/resources/application-dev.yml @@ -12,7 +12,7 @@ spring: register: false mongodb: - uri: mongodb://mongod_dbuser_archivesearch:mongod_dbpwd_archivesearch@localhost:27018/archivesearch?connectTimeoutMS=2000 + uri: mongodb://mongod_dbuser_archivesearch:mongod_dbpwd_archivesearch@localhost:27018/archivesearch?connectTimeoutMS=2000&appName=vitamui-collect multipart: enabled: true diff --git a/api/api-iam/iam/src/main/resources/application-dev.yml b/api/api-iam/iam/src/main/resources/application-dev.yml index 1b591fa14cb..1f74d58c749 100644 --- a/api/api-iam/iam/src/main/resources/application-dev.yml +++ b/api/api-iam/iam/src/main/resources/application-dev.yml @@ -19,11 +19,11 @@ spring: enabled: false register: false mongodb: - uri: mongodb://mongod_dbuser_iam:mongod_dbpwd_iam@localhost:27018/iam?connectTimeoutMS=2000 + uri: mongodb://mongod_dbuser_iam:mongod_dbpwd_iam@localhost:27018/iam?connectTimeoutMS=2000&appName=vitamui-iam discussion: mongodb: - uri: mongodb://mongod_dbuser_discussions:mongod_dbpwd_discussions@localhost:27018/discussions?connectTimeoutMS=2000 + uri: mongodb://mongod_dbuser_discussions:mongod_dbpwd_discussions@localhost:27018/discussions?connectTimeoutMS=2000&appName=vitamui-discussions scheduling: purgeTransactionDiscussions: enabled: true diff --git a/api/api-security/security/src/main/resources/application-dev.yml b/api/api-security/security/src/main/resources/application-dev.yml index 4ee6c05d02b..ca3502520d3 100644 --- a/api/api-security/security/src/main/resources/application-dev.yml +++ b/api/api-security/security/src/main/resources/application-dev.yml @@ -8,7 +8,7 @@ spring: enabled: false register: false mongodb: - uri: mongodb://mongod_dbuser_security:mongod_dbpwd_security@localhost:27018/security?connectTimeoutMS=2000 + uri: mongodb://mongod_dbuser_security:mongod_dbpwd_security@localhost:27018/security?connectTimeoutMS=2000&appName=vitamui-security web: error: diff --git a/cas/cas-server/src/main/config/application-dev.yml b/cas/cas-server/src/main/config/application-dev.yml index 885a81815b2..8f41dd81f69 100644 --- a/cas/cas-server/src/main/config/application-dev.yml +++ b/cas/cas-server/src/main/config/application-dev.yml @@ -65,7 +65,7 @@ cas: signing.key: =oebPIRe18A0cAeBdZCHkVlLPa_Kbthxo70iRpAhbk84dQGQj_8AOEMvEg3y7GAKYxtpF5nn6nx7vj5iU-eHStg encryption.key: 3FzNquczUjhmeJyqu251Ow service-registry.mongo: - client-uri: mongodb://mongod_dbuser_cas:mongod_dbpwd_cas@localhost:27018/cas + client-uri: mongodb://mongod_dbuser_cas:mongod_dbpwd_cas@localhost:27018/cas?appName=vitamui-cas authn: # Authent with x509 certificate : # x509: diff --git a/cas/cas-server/src/main/config/application-recette.yml b/cas/cas-server/src/main/config/application-recette.yml index f05567b478e..8ffdfbd17cd 100644 --- a/cas/cas-server/src/main/config/application-recette.yml +++ b/cas/cas-server/src/main/config/application-recette.yml @@ -46,7 +46,7 @@ management: cas: server.prefix: https://dev.vitamui.com:8080/cas service-registry.mongo: - client-uri: mongodb://mongod_dbuser_cas:mongod_dbpwd_cas@localhost:27018/cas + client-uri: mongodb://mongod_dbuser_cas:mongod_dbpwd_cas@localhost:27018/cas?appName=vitamui-cas # Root: iam-client - Internal VITAM-UI IAM service client iam-client: diff --git a/deployment/README.rst b/deployment/README.rst index 5cc2c5fd56d..14c15527bd3 100644 --- a/deployment/README.rst +++ b/deployment/README.rst @@ -205,6 +205,28 @@ ATTENTION: il faut avoir déployer aussi les extras Vitam, sinon le déploiement en l'absence ce cette extra vars, le comportement par defaut est extra=no +Exploitation +============= + +Diagnostic des sessions logiques MongoDB +---------------------------------------- + +Le script ``scripts/diagnose_mongo_logical_sessions.js`` répond à la question « le *reaper* de sessions logiques tourne-t-il sur ce nœud ? », posée par le message ``Unable to add session ID ... into the cache because the number of active sessions is too high`` (bug #15294). Il est en lecture seule et peut être lancé sur un primaire de production. + +*Script* :: + + mongosh "mongodb://:/admin?replicaSet=" \ + --username --password \ + --quiet --file scripts/diagnose_mongo_logical_sessions.js + +Il sort en 1 dès qu'un ``PROBLEM`` est relevé et en 0 sinon, ce qui permet de l'appeler depuis une supervision. Un ``WARNING`` demande une seconde passe : relancer le script un cycle de rafraîchissement plus tard et comparer ``activeSessionsCount`` distingue un *reaper* bloqué d'un nœud simplement peu sollicité. + +Le *reaper* ne tournant que sur le primaire, c'est là que le paramètre ``replicaSet`` amène la connexion, quel que soit le membre indiqué en amorce. C'est aussi ce qui rend cette URI inutilisable pour comparer les membres entre eux : elle lirait le primaire à chaque fois. Pour interroger un membre précis, il faut une connexion directe :: + + mongosh "mongodb://:/admin?directConnection=true" \ + --username --password \ + --quiet --file scripts/diagnose_mongo_logical_sessions.js + Désinstallation ================= diff --git a/deployment/roles/mongo/handlers/main.yml b/deployment/roles/mongo/handlers/main.yml index 36b841ca966..5f612b3dc0f 100644 --- a/deployment/roles/mongo/handlers/main.yml +++ b/deployment/roles/mongo/handlers/main.yml @@ -1,11 +1,5 @@ --- -- name: "mongo - restart service" - systemd: - name: vitamui-mongod - state: restarted - daemon_reload: yes - - name: "mongo - reload consul configuration" systemd: name: "{{ consul.service_name | default('vitam-consul') }}" diff --git a/deployment/roles/mongo/tasks/main.yml b/deployment/roles/mongo/tasks/main.yml index f261f1d0ba5..05a67d3b880 100644 --- a/deployment/roles/mongo/tasks/main.yml +++ b/deployment/roles/mongo/tasks/main.yml @@ -5,11 +5,10 @@ name: - mongodb-org-server state: present - register: result + register: mongod_package retries: "{{ packages_install_retries_number }}" - until: result is succeeded + until: mongod_package is succeeded delay: "{{ packages_install_retries_delay }}" - notify: "mongo - restart service" when: install_mode != "container" - name: Disable mongod default service @@ -31,7 +30,7 @@ owner: root group: root mode: "0644" - notify: "mongo - restart service" + register: mongod_service_file #### Configuration #### @@ -51,7 +50,7 @@ - log - script - tmp - notify: "mongo - restart service" + register: mongod_directories tags: update_mongodb_configuration - name: Create db directory @@ -61,7 +60,72 @@ owner: "{{ vitamui_defaults.users.vitamuidb | default('vitamuidb') }}" group: "{{ vitamui_defaults.users.group | default('vitamui') }}" mode: "{{ '0777' if install_mode == 'container' else vitam_defaults.folder.folder_permission }}" - notify: mongo - restart service + register: mongod_db_directory + tags: update_mongodb_configuration + +# The configuration written below deliberately declares no sharding.clusterRole +# (bug #15294). mongod refuses to start when it finds a shardIdentity document +# in admin.system.version without that role, and it only finds out at restart +# time, once the new configuration has already replaced the old one. VitamUI +# never registers its replica set into a sharded cluster, so the document should +# not be there, but a node carrying one has to be caught before that point. +- name: Check whether this node already holds MongoDB data + stat: + path: "{{ mongod_effective_db_path }}/WiredTiger" + register: mongod_data_directory + tags: update_mongodb_configuration + +# An arbiter is left out of the whole check: it replicates no admin.system.users, +# so the authenticated lookup below can only be refused on one, and it holds no +# collection to shard, so it carries no shardIdentity document to begin with. +# Without this the guard further down would abort every deployment of an +# inventory declaring an arbiter. +- name: Look for a leftover shardIdentity document + command: >- + mongosh {{ ip_service }}:{{ mongodb.mongod_port }}/admin + -u {{ mongodb.admin.user }} -p {{ mongodb.admin.password }} --quiet + --eval 'print("VITAMUI_SHARD_IDENTITY=" + (db.system.version.findOne({_id: "shardIdentity"}) ? 1 : 0))' + register: mongod_shard_identity + no_log: "{{ hide_passwords_during_deploy }}" + failed_when: false + changed_when: false + when: not (mongo_arbiter | default(false) | bool) + tags: update_mongodb_configuration + +- name: Fail when a shardIdentity document would keep mongod from restarting + fail: + msg: >- + admin.system.version holds a shardIdentity document on this node, and the + mongod configuration deployed by VitamUI declares no sharding.clusterRole: + mongod would refuse to start on the next restart. Check first that this + replica set really is not a shard of a sharded cluster, then drop the + document with: mongosh {{ ip_service }}:{{ mongodb.mongod_port }}/admin + -u --eval 'db.system.version.deleteOne({_id: "shardIdentity"})' + when: mongod_shard_identity.stdout | default('') is search('VITAMUI_SHARD_IDENTITY=[1-9]') + tags: update_mongodb_configuration + +# Reading admin.system.version needs a running mongod and the admin credentials. +# A node being installed has neither, and no sharding history to carry either, +# so it goes through. A node that already holds data and still cannot be read is +# the dangerous case: it may well carry the document, and letting it through +# would only find that out when it refuses to restart. +- name: Fail when the shardIdentity check could not run on a node holding data + fail: + msg: >- + This node already holds MongoDB data but admin.system.version could not be + read, so the shardIdentity check did not run: mongod is down, or the admin + credentials were refused. The configuration deployed by VitamUI declares no + sharding.clusterRole, and a node carrying a shardIdentity document would + only refuse to start once that configuration is in place. Start mongod and + run the deployment again, or check the node by hand with: mongosh + {{ ip_service }}:{{ mongodb.mongod_port }}/admin -u --eval + 'db.system.version.findOne({_id: "shardIdentity"})' then re-run with + -e mongod_skip_shard_identity_check=true once it answers null. + when: + - not (mongo_arbiter | default(false) | bool) + - mongod_data_directory.stat.exists + - mongod_shard_identity.stdout | default('') is not search('VITAMUI_SHARD_IDENTITY=') + - not (mongod_skip_shard_identity_check | default(false) | bool) tags: update_mongodb_configuration - name: Create the mongod configuration server file @@ -71,7 +135,7 @@ owner: "{{ vitamui_defaults.users.vitamuidb | default('vitamuidb') }}" group: "{{ vitamui_defaults.users.group | default('vitamui') }}" mode: "{{ '0444' if install_mode == 'container' else vitam_defaults.folder.conf_permission }}" - notify: mongo - restart service + register: mongod_configuration tags: update_mongodb_configuration - name: Enable logrotate for vitamui-mongod @@ -120,6 +184,46 @@ - consul_conf - update_mongodb_configuration +#### Rolling restart #### +# mongod must never be restarted on every member at once: with the default +# parallelism (forks = 20 in ansible.cfg) the whole replica set goes down within +# the same second and the quorum is lost, where a member by member restart is +# just a failover. The loop below is what serialises it, driven from a single +# host so that each member is taken down, waited for, and only then followed by +# the next one. + +# The package installation and the systemd unit carry no tag, so a run limited +# to --tags update_mongodb_configuration never registers their variables. The +# "changed" test raises on an undefined value rather than answering false, so +# both are guarded: in such a run neither can have changed anyway, since neither +# ran. +- name: Check whether vitamui-mongod must be restarted + set_fact: + mongod_restart_required: >- + {{ (mongod_package is defined and mongod_package is changed) + or (mongod_service_file is defined and mongod_service_file is changed) + or mongod_directories is changed + or mongod_db_directory is changed + or mongod_configuration is changed }} + tags: update_mongodb_configuration + +- name: Restart vitamui-mongod, one replica set member at a time + include_tasks: + file: rolling_restart.yml + # Without apply, the tag below only gates the include itself: the tasks it + # brings in carry no tag of their own and a run limited to + # --tags update_mongodb_configuration would skip every restart and every + # wait, leaving the configuration it just wrote inactive. + apply: + tags: update_mongodb_configuration + vars: + mongod_member_restart_required: "{{ hostvars[mongod_member]['mongod_restart_required'] | default(false) | bool }}" + loop: "{{ groups['hosts_vitamui_mongod'] }}" + loop_control: + loop_var: mongod_member + run_once: true + tags: update_mongodb_configuration + - name: Ensure vitamui-mongod is started service: name: vitamui-mongod diff --git a/deployment/roles/mongo/tasks/rolling_restart.yml b/deployment/roles/mongo/tasks/rolling_restart.yml new file mode 100644 index 00000000000..91df7548fbb --- /dev/null +++ b/deployment/roles/mongo/tasks/rolling_restart.yml @@ -0,0 +1,68 @@ +--- + +# One member of the replica set, included once per member from main.yml. The +# loop is what makes the sequence sequential, and it has to be: "throttle: 1" +# on a block only serialises each task across the hosts, so every member would +# be restarted before the first wait had even started. Measured on Ansible +# 2.9.27, three hosts, a block of restart then sleep 3: with throttle the three +# sequences all ran between t and t+9.8s, with this loop they ran back to back. + +# The include loops over every member of hosts_vitamui_mongod, including the +# ones the current play does not cover: a run under --limit, or a member that +# errored out earlier, leaves no mongod_restart_required fact behind. Defaulting +# that to "nothing to do" is the only safe choice here, but it must not pass +# unnoticed: such a member keeps running the configuration it had, which is the +# very one this fix exists to replace. +- name: "Report that the restart state of {{ mongod_member }} is unknown" + debug: + msg: >- + WARNING: {{ mongod_member }} is not part of this run, so whether it needs a + restart could not be determined and it is left untouched. It may still be + running with the previous mongod configuration, sharding.clusterRole + included. Run the mongo role against it before considering the replica set + up to date. + when: hostvars[mongod_member]['mongod_restart_required'] is not defined + +- name: "Restart vitamui-mongod on {{ mongod_member }}" + # Restarting the primary needs no explicit rs.stepDown(): a clean SIGTERM + # makes mongod hand the primary role over before exiting. + systemd: + name: vitamui-mongod + state: restarted + daemon_reload: yes + delegate_to: "{{ mongod_member }}" + when: mongod_member_restart_required | bool + +- name: "Wait for the vitamui-mongod port to be open on {{ mongod_member }}" + wait_for: + host: "{{ hostvars[mongod_member]['ip_service'] }}" + port: "{{ mongodb.mongod_port }}" + timeout: "{{ vitamui_defaults.services.start_timeout }}" + delegate_to: "{{ mongod_member }}" + when: mongod_member_restart_required | bool + +- name: "Wait for {{ mongod_member }} to be back in the replica set" + # db.hello() is answered before authentication, so this holds whether or not + # the security block has already been added to mongod.conf. A member with no + # setName belongs to no replica set yet and has nothing to rejoin, which is + # the case on a first installation since the set is only initiated further + # down the role: it answers at once instead of waiting. + # + # The state is read here, after the restart, and never before it: a probe run + # beforehand cannot tell "belongs to no replica set" from "could not be + # reached", and taking the second for the first would move on to the next + # member while this one is still recovering - two members down at once on a + # three member set, which is exactly the quorum loss the rolling restart is + # there to avoid. Read afterwards, an unreachable member simply keeps the + # retries going and eventually stops the play on itself. + command: >- + mongosh --host {{ hostvars[mongod_member]['ip_service'] }} --port {{ mongodb.mongod_port }} + --quiet --eval 'const member = db.hello(); + quit(!member.setName || member.isWritablePrimary || member.secondary || member.arbiterOnly ? 0 : 1)' + delegate_to: "{{ mongod_member }}" + register: mongod_member_state + until: mongod_member_state.rc == 0 + retries: "{{ vitamui_defaults.services.status_retries_number | default(60) }}" + delay: "{{ vitamui_defaults.services.status_retries_delay | default(5) }}" + changed_when: false + when: mongod_member_restart_required | bool diff --git a/deployment/roles/mongo/templates/mongod.conf.j2 b/deployment/roles/mongo/templates/mongod.conf.j2 index dac928cf8e3..07d6ffe0962 100644 --- a/deployment/roles/mongo/templates/mongod.conf.j2 +++ b/deployment/roles/mongo/templates/mongod.conf.j2 @@ -43,8 +43,18 @@ replication: replSetName: shard{{ mongo_shard_id | default(0) }} # name of the replica set enableMajorityReadConcern: true -sharding: - clusterRole: shardsvr # role du shard +# NOTE: do NOT declare "sharding.clusterRole: shardsvr" here. +# VitamUI deploys a single replica set and no mongos / config server (the +# inventory only defines hosts_vitamui_mongod). A mongod declaring itself a +# shard server delegates the creation of config.system.sessions to the cluster, +# which therefore never happens: the logical session cache is never reaped and +# grows until it hits maxSessions, at which point MongoDB rejects every new +# session until the process is restarted (bug #15294). +# The "shard0" replica set name above comes from the same VITAM heritage, but +# unlike the cluster role it is load bearing: it is the replicaSet= of every +# application URI, the _id init-replica.js.j2 initiates the set with, and what +# mongo_init connects through. Renaming it would mean rewriting all of them for +# a purely cosmetic gain. # During (re)initialization, authentication is temporarily disabled to ensure replica-set & user creation/update. # Authentication will be (re)enabled during mongo_configure role execution diff --git a/deployment/roles/mongo/templates/restore-mongod.js.j2 b/deployment/roles/mongo/templates/restore-mongod.js.j2 deleted file mode 100644 index c6af478fdba..00000000000 --- a/deployment/roles/mongo/templates/restore-mongod.js.j2 +++ /dev/null @@ -1,52 +0,0 @@ -// PreRestore -//=========== -// Start all mongod as a simple mongod without --shardsvr nor --replSet as explained in official documentation -// For more information @see https://docs.mongodb.com/manual/tutorial/restore-sharded-cluster/ -// In Mongo Shard instance Only - -use admin - -// Authenticate as root user -db.auth("{{ mongodb.admin.user }}", "{{ mongodb.admin.password }}") - -// Create system user -db.createUser({user: "{{ mongodb.system.user }}", pwd: "{{ mongodb.system.password }}", roles: [ "__system" ]}) - -// Authenticate as system user -db.auth("{{ mongodb.system.user }}", "{{ mongodb.system.password }}") - -// Update system.version collections -db.system.version.deleteOne( { "_id": "minOpTimeRecovery" } ) - -var mongocList = []; - -// Loop over all mongoc and get corresponding ip and port number -{% for host in groups[mongoc] %} - {% if mongo_cluster_name == hostvars[host]['mongo_cluster_name'] %} -mongocList.push("{{ hostvars[host]['ip_service'] }}:{{ mongodb.mongoc_port }}") - {% endif %} -{% endfor %} - - -db.system.version.updateOne( - { "_id" : "shardIdentity" }, - { $set : { "configsvrConnectionString" : "configserver/" + mongocList.join(",") }} - ) - -// Drop local data base -use local -db.dropDatabase() - -// Remove system user -use admin -// Authenticate as root user -db.auth("{{ mongodb.admin.user }}","{{ mongodb.admin.password }}") -db.removeUser("{{ mongodb.system.user }}") - - -// PostRestore -//=========== -// Start all mongod with --shardsvr and --replSet corresponding the configuration -// Init replicatSet rs.initiate() on a given mongod instance for each shard -// Should waite until election of a primary mongod instance for each shard replicatSet - diff --git a/deployment/roles/mongo/vars/main.yml b/deployment/roles/mongo/vars/main.yml index 2ac56870a53..9b587be9d40 100644 --- a/deployment/roles/mongo/vars/main.yml +++ b/deployment/roles/mongo/vars/main.yml @@ -3,4 +3,8 @@ mongo_tmp_path: "{{ vitamui_defaults.folder.root_path }}/tmp/mongod" mongo_config_path: "{{ vitamui_defaults.folder.root_path }}/conf/mongod" mongo_db_path: "{{ vitamui_defaults.folder.root_path }}/data/mongod/db" + +# Where the data actually sits, which is not always storage.dbPath: the container +# unit passes --dbpath on its command line, and that overrides mongod.conf. +mongod_effective_db_path: "{{ '/vitamui/data/mongod' if install_mode == 'container' else mongo_db_path }}" mongo_folder_log: '{{ vitamui_defaults.folder.root_path }}/log/mongod' diff --git a/deployment/roles/mongo_configure/tasks/main.yml b/deployment/roles/mongo_configure/tasks/main.yml index bd449397e9a..8c8a8f8b898 100644 --- a/deployment/roles/mongo_configure/tasks/main.yml +++ b/deployment/roles/mongo_configure/tasks/main.yml @@ -34,19 +34,21 @@ tags: update_mongodb_configuration # Restart the mongo instances (to enable auth) -- name: Restart vitamui-mongod service - service: - name: vitamui-mongod - enabled: "{{ mongodb.at_boot | default(service_at_boot) }}" - state: restarted - delegate_to: "{{ item }}" - with_items: "{{ groups['hosts_vitamui_mongod'] }}" - tags: update_mongodb_configuration - -# Make sure the service is open (mongod listening on 27017) -- name: "Wait for the service port {{ mongodb.mongod_port }} to be open" - wait_for: - host: "{{ ip_service }}" - port: "{{ mongodb.mongod_port }}" - timeout: "{{ vitamui_defaults.services.start_timeout }}" +# Driven from a single host, one member at a time: the previous delegate_to loop +# had every host restart every member, so each member was restarted as many +# times as there are members, all of them concurrently, and the only wait came +# after the whole loop. +- name: Restart vitamui-mongod service, one replica set member at a time + include_tasks: + file: restart_member.yml + # Without apply, the tag below only gates the include itself: the tasks it + # brings in carry no tag of their own and a run limited to + # --tags update_mongodb_configuration would skip every restart and every + # wait, leaving the configuration it just wrote inactive. + apply: + tags: update_mongodb_configuration + loop: "{{ groups['hosts_vitamui_mongod'] }}" + loop_control: + loop_var: mongod_member + run_once: true tags: update_mongodb_configuration diff --git a/deployment/roles/mongo_configure/tasks/restart_member.yml b/deployment/roles/mongo_configure/tasks/restart_member.yml new file mode 100644 index 00000000000..55cc6ddaced --- /dev/null +++ b/deployment/roles/mongo_configure/tasks/restart_member.yml @@ -0,0 +1,34 @@ +--- + +# One member of the replica set, included once per member from main.yml, for the +# same reason as roles/mongo/tasks/rolling_restart.yml: only a loop sequences a +# whole sequence of tasks per host, "throttle: 1" on a block does not. + +- name: "Restart vitamui-mongod on {{ mongod_member }}" + service: + name: vitamui-mongod + enabled: "{{ mongodb.at_boot | default(service_at_boot) }}" + state: restarted + delegate_to: "{{ mongod_member }}" + +# Make sure the service is open (mongod listening on 27017) +- name: "Wait for the service port {{ mongodb.mongod_port }} to be open on {{ mongod_member }}" + wait_for: + host: "{{ hostvars[mongod_member]['ip_service'] }}" + port: "{{ mongodb.mongod_port }}" + timeout: "{{ vitamui_defaults.services.start_timeout }}" + delegate_to: "{{ mongod_member }}" + +# db.hello() is answered before authentication, which matters here since the +# restart is precisely what enables it. +- name: "Wait for {{ mongod_member }} to be back in the replica set" + command: >- + mongosh --host {{ hostvars[mongod_member]['ip_service'] }} --port {{ mongodb.mongod_port }} + --quiet --eval 'const member = db.hello(); + quit(member.isWritablePrimary || member.secondary || member.arbiterOnly ? 0 : 1)' + delegate_to: "{{ mongod_member }}" + register: mongod_member_state + until: mongod_member_state.rc == 0 + retries: "{{ vitamui_defaults.services.status_retries_number | default(60) }}" + delay: "{{ vitamui_defaults.services.status_retries_delay | default(5) }}" + changed_when: false diff --git a/deployment/roles/vitamui/templates/archive-search/application.yml.j2 b/deployment/roles/vitamui/templates/archive-search/application.yml.j2 index 1f7a2cffcb7..6bca60f3ee1 100644 --- a/deployment/roles/vitamui/templates/archive-search/application.yml.j2 +++ b/deployment/roles/vitamui/templates/archive-search/application.yml.j2 @@ -18,7 +18,7 @@ spring: instanceId: ${spring.application.name}-${spring.cloud.client.hostname}-${server.port} mongodb: - uri: mongodb://{{ mongodb.archivesearch.user }}:{{ mongodb.archivesearch.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.archivesearch.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }} + uri: mongodb://{{ mongodb.archivesearch.user }}:{{ mongodb.archivesearch.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.archivesearch.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}&appName=vitamui-archive-search cas.tenant.identifier: {{ vitamui_platform_informations.cas_tenant }} diff --git a/deployment/roles/vitamui/templates/cas-server/application.yml.j2 b/deployment/roles/vitamui/templates/cas-server/application.yml.j2 index 75fd9dd600b..dce0c0afa7a 100644 --- a/deployment/roles/vitamui/templates/cas-server/application.yml.j2 +++ b/deployment/roles/vitamui/templates/cas-server/application.yml.j2 @@ -105,7 +105,7 @@ cas.server.prefix: {{ url_prefix }}/cas {% endif %} login.url: ${cas.server.prefix}/login -cas.service-registry.mongo.client-uri: "mongodb://{{ mongodb.cas.user }}:{{ mongodb.cas.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port | default(27017) }}/{{ mongodb.cas.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}" +cas.service-registry.mongo.client-uri: "mongodb://{{ mongodb.cas.user }}:{{ mongodb.cas.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port | default(27017) }}/{{ mongodb.cas.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}&appName=vitamui-cas" cas.service-registry.mongo.collection: services cas.service-registry.mongo.user-id: {{ mongodb.cas.user }} cas.service-registry.mongo.password: {{ mongodb.cas.password }} diff --git a/deployment/roles/vitamui/templates/collect/application.yml.j2 b/deployment/roles/vitamui/templates/collect/application.yml.j2 index 9ac3b9d7a23..9470d8db19e 100644 --- a/deployment/roles/vitamui/templates/collect/application.yml.j2 +++ b/deployment/roles/vitamui/templates/collect/application.yml.j2 @@ -17,7 +17,7 @@ spring: tags: vitamui, {{ vitamui_struct.vitamui_component }} instanceId: ${spring.application.name}-${spring.cloud.client.hostname}-${server.port} mongodb: - uri: mongodb://{{ mongodb.archivesearch.user }}:{{ mongodb.archivesearch.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.archivesearch.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }} + uri: mongodb://{{ mongodb.archivesearch.user }}:{{ mongodb.archivesearch.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.archivesearch.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}&appName=vitamui-collect servlet: multipart: max-file-size: -1 diff --git a/deployment/roles/vitamui/templates/iam/application.yml.j2 b/deployment/roles/vitamui/templates/iam/application.yml.j2 index fa9381b4bcb..ef62a583d21 100644 --- a/deployment/roles/vitamui/templates/iam/application.yml.j2 +++ b/deployment/roles/vitamui/templates/iam/application.yml.j2 @@ -17,11 +17,11 @@ spring: tags: vitamui, {{ vitamui_struct.vitamui_component }} instanceId: ${spring.application.name}-${spring.cloud.client.hostname}-${server.port} mongodb: - uri: "mongodb://{{ mongodb.iam.user }}:{{ mongodb.iam.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.iam.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}" + uri: "mongodb://{{ mongodb.iam.user }}:{{ mongodb.iam.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.iam.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}&appName=vitamui-iam" discussion: mongodb: - uri: "mongodb://{{ mongodb.discussions.user }}:{{ mongodb.discussions.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.discussions.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}" + uri: "mongodb://{{ mongodb.discussions.user }}:{{ mongodb.discussions.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.discussions.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}&appName=vitamui-discussions" scheduling: purgeTransactionDiscussions: enabled: {{ vitamui_struct.discussion.enabled | default('true') | lower }} diff --git a/deployment/roles/vitamui/templates/security/application.yml.j2 b/deployment/roles/vitamui/templates/security/application.yml.j2 index 28f5968419b..2e2030e5841 100644 --- a/deployment/roles/vitamui/templates/security/application.yml.j2 +++ b/deployment/roles/vitamui/templates/security/application.yml.j2 @@ -14,7 +14,7 @@ spring: tags: vitamui, {{ vitamui_struct.vitamui_component }} instanceId: ${spring.application.name}-${spring.cloud.client.hostname}-${server.port} mongodb: - uri: "mongodb://{{ mongodb.security.user }}:{{ mongodb.security.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.security.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}" + uri: "mongodb://{{ mongodb.security.user }}:{{ mongodb.security.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.security.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}&appName=vitamui-security" logging: config: {{ vitamui_folder_conf }}/logback.xml diff --git a/deployment/scripts/diagnose_mongo_logical_sessions.js b/deployment/scripts/diagnose_mongo_logical_sessions.js new file mode 100644 index 00000000000..2c9b1019c13 --- /dev/null +++ b/deployment/scripts/diagnose_mongo_logical_sessions.js @@ -0,0 +1,458 @@ +// Diagnose MongoDB logical session cache growth (bug #15294). +// +// Symptom this script investigates: +// "Unable to add session ID ... into the cache because the number of active +// sessions is too high" +// appearing after days or weeks of uptime and only cleared by a mongod restart. +// +// The cache of active logical sessions is held in memory by mongod. It is +// trimmed by a background job that persists sessions into config.system.sessions +// and lets a TTL index on "lastUse" expire them. If that job cannot run, nothing +// is ever reaped, the cache grows monotonically until it reaches maxSessions, +// and restarting the process is the only way to empty it. +// +// The job picks its implementation from the node role: +// - plain replica set -> the primary creates config.system.sessions itself +// - clusterRole shardsvr -> the collection is expected to come from a mongos +// and the config servers, which VitamUI does not deploy +// +// This script is read-only and safe to run on a production primary. +// +// Usage. The reaper only runs on the primary, and replicaSet= is what takes the +// connection there whichever member is used to seed it: +// mongosh "mongodb://:/admin?replicaSet=" \ +// --username --password \ +// --quiet --file deployment/scripts/diagnose_mongo_logical_sessions.js +// +// That same routing makes this URI useless to compare members against one +// another: it would read the primary every time, whatever host is named. Each +// member has to be reached by a direct connection instead: +// mongosh "mongodb://:/admin?directConnection=true" \ +// --username --password \ +// --quiet --file deployment/scripts/diagnose_mongo_logical_sessions.js +// +// --password with no value makes mongosh prompt for it, which keeps the +// password out of the shell history and of the process arguments every other +// user of the machine can read. +// +// Run it twice a few minutes apart: activeSessionsCount rising while +// sessionsCollectionJobCount stays flat is the signature of a stalled reaper. + +const SESSIONS_NAMESPACE = "config.system.sessions"; +const TTL_INDEX_KEY_FIELD = "lastUse"; + +const findings = []; + +function heading(title) { + print(""); + print("=== " + title + " ==="); +} + +function report(label, value) { + print(" " + label + ": " + value); +} + +// Every probe is optional: a restricted account or an older server must degrade +// into a partial report rather than abort the whole diagnosis. +// +// Getting there takes both branches below. mongosh rewrites the shell API calls +// made inside fn into awaited ones and awaits the call to probe() itself, but it +// does not await fn() here: a denied command therefore came back as a rejected +// promise, long after the try block had been left, and killed the whole script +// instead of printing one unavailable line. Attaching the handler to the promise +// is what actually catches it. The try/catch is kept for whatever throws +// synchronously, and an explicit await cannot be used instead: mongosh parses +// the file as a plain script, where top level await is a syntax error. +// +// A refused command is not an error either: db.runCommand() hands back the raw +// response document, {ok: 0, errmsg: "not authorized on admin..."}, without +// throwing and without rejecting. Left alone that reads as a successful probe +// whose every field happens to be missing, which is how an unreadable node ends +// up reported as a healthy plain replica set. Anything carrying an "ok" other +// than 1 is therefore turned into an unavailable probe. Collection helpers and +// the wrappers below return arrays, numbers or plain objects with no "ok" field, +// so they go through untouched. +function probe(label, fn) { + const unavailable = (error) => { + print(" ! " + label + " unavailable: " + error.message); + return null; + }; + + const checkCommandOk = (result) => { + if (result && typeof result === "object" && result.ok !== undefined && result.ok !== 1) { + return unavailable( + new Error(result.errmsg || result.codeName || "command failed (ok=" + result.ok + ")") + ); + } + return result; + }; + + try { + const result = fn(); + return result && typeof result.then === "function" + ? result.then(checkCommandOk, unavailable) + : checkCommandOk(result); + } catch (error) { + return unavailable(error); + } +} + +// getParameter answers {parameterName: value, ok: 1}, so the value can only be +// read once the probe has confirmed the command itself succeeded. Reading it +// inside fn instead would hand back undefined on a refusal, which passes every +// "!== null" guard downstream and turns into NaN a few lines later. +function probeField(label, command, field) { + const response = probe(label, () => db.getSiblingDB("admin").runCommand(command)); + return response ? response[field] : null; +} + +function fail(message) { + findings.push({ level: "PROBLEM", message: message }); +} + +function warn(message) { + findings.push({ level: "WARNING", message: message }); +} + +print("MongoDB logical session diagnosis - " + new Date().toISOString()); + +// --------------------------------------------------------------------------- +// 1. Topology: which SessionsCollection implementation does this node use? +// --------------------------------------------------------------------------- + +heading("Topology"); + +const adminDb = db.getSiblingDB("admin"); + +const hello = probe("hello", () => adminDb.runCommand({ hello: 1 })); +if (hello) { + report("version", probeField("buildInfo", { buildInfo: 1 }, "version") || "(unknown)"); + report("replica set", hello.setName || "(none)"); + report("is primary", hello.isWritablePrimary === true); + if (hello.isWritablePrimary !== true) { + warn( + "Not connected to the primary. The session reaper only runs there, " + + "so re-run this script against the primary before drawing conclusions." + ); + } +} else { + // Silence here would leave the whole topology section blank, which reads far + // too much like a node with nothing to report. + warn( + "hello could not be read, so neither the replica set this node belongs to " + + "nor its primary state is known. Every conclusion below is drawn without " + + "knowing whether the reaper is even expected to run here." + ); +} + +// probe() returns null on a command error, and an absent sharding section reads +// as "no cluster role" too. Printing the same thing for both would let a node +// nobody could read pass for a healthy plain replica set, which is the one +// conclusion this script must never reach by accident. +const commandLine = probe("getCmdLineOpts", () => adminDb.runCommand({ getCmdLineOpts: 1 })); +const clusterRoleKnown = commandLine !== null; +const clusterRole = + clusterRoleKnown && commandLine.parsed && commandLine.parsed.sharding + ? commandLine.parsed.sharding.clusterRole + : null; +report("clusterRole", clusterRoleKnown ? clusterRole || "(none - plain replica set)" : "(unknown)"); +if (!clusterRoleKnown) { + warn( + "getCmdLineOpts could not be read, so the cluster role is unknown and the " + + "configuration behind this bug cannot be ruled out from here. Check " + + "sharding.clusterRole in mongod.conf by hand." + ); +} + +// findOne() legitimately answers null when the document is absent, so the lookup +// is wrapped: a null wrapper means the probe itself failed, a null document +// means the node is clean. +const shardIdentityLookup = probe("shardIdentity lookup", () => { + return { document: adminDb.system.version.findOne({ _id: "shardIdentity" }) }; +}); +const shardIdentity = shardIdentityLookup ? shardIdentityLookup.document : null; +report( + "shardIdentity document", + shardIdentityLookup ? (shardIdentity ? "present" : "absent") : "(unknown)" +); + +if (clusterRole === "shardsvr") { + fail( + "clusterRole is 'shardsvr'. On a shard server mongod does NOT create " + + SESSIONS_NAMESPACE + + " itself; it expects a mongos and config servers to have done it. " + + "VitamUI deploys neither (the inventory only has hosts_vitamui_mongod), " + + "so the session reaper cannot run and the cache never shrinks. " + + "Fix: remove the 'sharding: clusterRole: shardsvr' block from " + + "deployment/roles/mongo/templates/mongod.conf.j2 and restart the nodes " + + "(secondaries first, then step down the primary)." + ); + if (shardIdentity) { + warn( + "A shardIdentity document exists in admin.system.version. It must be " + + "removed before restarting without clusterRole, otherwise mongod " + + "will refuse to start." + ); + } +} + +// --------------------------------------------------------------------------- +// 2. Is the reaper actually running? +// --------------------------------------------------------------------------- + +heading("Logical session record cache"); + +const serverStatus = probe("serverStatus", () => adminDb.runCommand({ serverStatus: 1 })); +const recordCache = serverStatus ? serverStatus.logicalSessionRecordCache : null; + +// The refresh job only fires every logicalSessionRefreshMillis, and its first +// useful cycle happens after the node has been elected. A node that just +// restarted legitimately shows no sessions collection and no refreshed entry, +// so hold back the corresponding conclusions until a couple of cycles elapsed. +const refreshMillisProbe = probeField( + "logicalSessionRefreshMillis", + { getParameter: 1, logicalSessionRefreshMillis: 1 }, + "logicalSessionRefreshMillis" +); +// Every timing conclusion below is measured against this interval. A server +// started with a longer one than the 5 minute default would make them all wrong, +// so an assumed value is reported as such and never concludes on its own. +const refreshMillisKnown = typeof refreshMillisProbe === "number"; +const refreshMillis = refreshMillisKnown ? refreshMillisProbe : 300000; +const uptimeSeconds = serverStatus ? serverStatus.uptime : 0; +const elapsedCycles = uptimeSeconds / (refreshMillis / 1000); +const settled = elapsedCycles >= 2; + +report( + "refresh interval (minutes)", + (refreshMillis / 60000).toFixed(1) + (refreshMillisKnown ? "" : " (assumed, not read)") +); +report("uptime (minutes)", (uptimeSeconds / 60).toFixed(1)); +report("refresh cycles elapsed", elapsedCycles.toFixed(1)); +if (!settled) { + print( + " ! This node restarted less than two refresh cycles ago; findings below " + + "are inconclusive. Re-run in " + + Math.ceil((2 - elapsedCycles) * (refreshMillis / 60000)) + + " minutes." + ); +} + +// Below the settling threshold a missing collection or an idle job proves +// nothing, so the same evidence is reported as a warning instead of a problem. +const conclude = settled ? fail : warn; + +if (recordCache) { + report("activeSessionsCount", recordCache.activeSessionsCount); + report("sessionsCollectionJobCount", recordCache.sessionsCollectionJobCount); + report("lastSessionsCollectionJobTimestamp", recordCache.lastSessionsCollectionJobTimestamp); + report("lastSessionsCollectionJobDurationMillis", recordCache.lastSessionsCollectionJobDurationMillis); + report("lastSessionsCollectionJobEntriesRefreshed", recordCache.lastSessionsCollectionJobEntriesRefreshed); + report("lastSessionsCollectionJobEntriesEnded", recordCache.lastSessionsCollectionJobEntriesEnded); + report("lastTransactionReaperJobTimestamp", recordCache.lastTransactionReaperJobTimestamp); + + const lastJob = recordCache.lastSessionsCollectionJobTimestamp; + if (lastJob) { + // The refresh job runs every logicalSessionRefreshMillis, read above + // rather than assumed: missing three cycles in a row means it is + // erroring out every time, not merely running late. + const refreshMinutes = refreshMillis / 60000; + const staleMinutes = (Date.now() - new Date(lastJob).getTime()) / 60000; + report("last job age (minutes)", staleMinutes.toFixed(1)); + if (staleMinutes > 3 * refreshMinutes) { + const staleConclusion = refreshMillisKnown ? fail : warn; + staleConclusion( + "The sessions collection job last completed " + + staleMinutes.toFixed(0) + + " minutes ago; it should run every " + + refreshMinutes.toFixed(0) + + (refreshMillisKnown + ? " minutes. " + : " minutes, assumed since logicalSessionRefreshMillis could not be read. ") + + "Nothing is being reaped." + ); + } + } + + if (recordCache.sessionsCollectionJobCount === 0) { + conclude("sessionsCollectionJobCount is 0: the reaper has never completed a cycle since startup."); + } + + // The most discriminating signal, and the one a single sample cannot carry. + // On a misconfigured node the job still ticks, so sessionsCollectionJobCount + // and the timestamp both look healthy, but every cycle bails out before + // touching a single record. A healthy but quiet node reads exactly the same + // when its sessions were all opened since the last cycle, so this stays a + // warning whatever the settling state: only the same reading with a higher + // activeSessionsCount, one refresh cycle later, tells the two apart. + if ( + recordCache.sessionsCollectionJobCount > 0 && + recordCache.activeSessionsCount > 0 && + recordCache.lastSessionsCollectionJobEntriesRefreshed === 0 + ) { + warn( + "The last sessions collection job refreshed no record " + + "(lastSessionsCollectionJobEntriesRefreshed = 0) while " + + recordCache.activeSessionsCount + + " sessions are active. Re-run this script after a refresh cycle: " + + "the same reading with a higher activeSessionsCount means the job " + + "is running but doing nothing, so the cache can only grow." + ); + } +} else if (serverStatus) { + warn("logicalSessionRecordCache is not exposed by serverStatus on this node."); +} else { + // Distinct from the case above on purpose: a server that does not expose the + // section is a known shape, a server nobody could read is an open question. + warn( + "serverStatus could not be read, so the session cache counters, the uptime " + + "and the open cursor metrics are all missing. Nothing below rules the " + + "reaper out; re-run with an account allowed to run serverStatus on admin." + ); +} + +const maxSessions = probeField("maxSessions", { getParameter: 1, maxSessions: 1 }, "maxSessions"); +if (typeof maxSessions === "number") { + report("maxSessions", maxSessions); + if (recordCache && recordCache.activeSessionsCount) { + const usage = (recordCache.activeSessionsCount / maxSessions) * 100; + report("cache usage", usage.toFixed(2) + "%"); + if (usage > 50) { + fail("The logical session cache is " + usage.toFixed(1) + "% full."); + } + } +} + +// --------------------------------------------------------------------------- +// 3. The decisive check: does the TTL index exist? +// --------------------------------------------------------------------------- + +heading("Sessions collection"); + +const configDb = db.getSiblingDB("config"); + +const collections = probe("listCollections", () => + configDb.getCollectionNames().filter((name) => name === "system.sessions") +); + +if (collections !== null && collections.length === 0) { + report("collection", "ABSENT"); + conclude( + SESSIONS_NAMESPACE + + " does not exist. Without it no session is ever persisted or expired, " + + "so the in-memory cache can only grow." + ); +} else if (collections !== null) { + report("collection", "present"); + + const indexes = probe("getIndexes", () => configDb.system.sessions.getIndexes()); + if (indexes) { + const ttlIndex = indexes.find( + (index) => index.key && index.key[TTL_INDEX_KEY_FIELD] !== undefined && index.expireAfterSeconds !== undefined + ); + if (ttlIndex) { + report("TTL index", ttlIndex.name + " (expireAfterSeconds=" + ttlIndex.expireAfterSeconds + ")"); + } else { + fail( + "No TTL index on " + + SESSIONS_NAMESPACE + + "." + + TTL_INDEX_KEY_FIELD + + ". Persisted sessions are never expired." + ); + } + } + + // Reported last because it is the one line here that is merely nice to + // have, and the one most likely to be refused: counting documents in + // config.system.sessions needs privileges that the root role does not carry + // (observed on 8.0.23). estimatedDocumentCount() reads collection metadata + // instead of scanning, which matters once the collection holds millions of + // documents. + const documentCount = probe("estimatedDocumentCount", () => + configDb.system.sessions.estimatedDocumentCount() + ); + if (documentCount !== null) { + report("approximate document count", documentCount); + } +} + +const ttlMonitor = probeField("ttlMonitorEnabled", { getParameter: 1, ttlMonitorEnabled: 1 }, "ttlMonitorEnabled"); +if (typeof ttlMonitor === "boolean") { + report("ttlMonitorEnabled", ttlMonitor); + if (ttlMonitor === false) { + fail("The TTL monitor is disabled, so no TTL index expires anything."); + } +} + +// --------------------------------------------------------------------------- +// 4. Client-side counter-hypothesis: cursors pinning their session +// --------------------------------------------------------------------------- +// +// A cursor keeps its logical session alive for as long as it lives, and a change +// stream never ends on its own. If the reaper turns out to be healthy, look here +// instead. Only counts are printed, never the operations themselves. + +heading("Open cursors"); + +if (serverStatus && serverStatus.metrics && serverStatus.metrics.cursor) { + const cursor = serverStatus.metrics.cursor; + report("open.total", cursor.open.total); + report("open.pinned", cursor.open.pinned); + report("open.noTimeout", cursor.open.noTimeout); + report("timedOut (since startup)", cursor.timedOut); +} + +const changeStreams = probe("currentOp", () => { + const result = adminDb.aggregate([ + { $currentOp: { allUsers: true, idleCursors: true } }, + { $match: { "cursor.originatingCommand.pipeline.0.$changeStream": { $exists: true } } }, + { $group: { _id: "$ns", count: { $sum: 1 }, oldest: { $min: "$cursor.createdDate" } } } + ]); + return result.toArray(); +}); + +if (changeStreams === null) { + // Without this the section would print nothing at all, which reads exactly + // like an absence of change streams. + print(" Open change streams could not be listed: this hypothesis is left open."); +} else if (changeStreams.length === 0) { + report("open change streams", 0); +} else { + changeStreams.forEach((entry) => { + report("change streams on " + entry._id, entry.count + " (oldest opened " + entry.oldest + ")"); + }); + warn( + "Change streams are open. Each one pins a logical session for its whole " + + "lifetime. Compare the count against the number of running application " + + "instances: significantly more means they are being leaked." + ); +} + +// --------------------------------------------------------------------------- +// 5. Verdict +// --------------------------------------------------------------------------- + +heading("Verdict"); + +const problems = findings.filter((finding) => finding.level === "PROBLEM").length; + +if (findings.length === 0) { + print(" No anomaly detected. The session reaper looks healthy on this node."); + print(" If sessions still accumulate, run this script against each member in"); + print(" turn with directConnection=true and compare activeSessionsCount: the"); + print(" replicaSet= form would report the primary every time."); +} else { + report("problems", problems); + report("warnings", findings.length - problems); + print(""); + findings.forEach((finding, index) => { + print(" [" + finding.level + " " + (index + 1) + "] " + finding.message); + print(""); + }); +} + +// Exit status, so that a supervision job can consume the verdict without +// parsing the report. Warnings alone call for a second run of this script, not +// for an alert, hence only problems are counted. +quit(problems > 0 ? 1 : 0);