Bug #15294: stop declaring mongod as a shard server

api/api-archive-search/archive-search/src/main/resources/application-dev.yml

@@ -12,7 +12,7 @@ spring:
 
 
   mongodb:
-    uri: mongodb://mongod_dbuser_archivesearch:mongod_dbpwd_archivesearch@localhost:27018/archivesearch?connectTimeoutMS=2000
+    uri: mongodb://mongod_dbuser_archivesearch:mongod_dbpwd_archivesearch@localhost:27018/archivesearch?connectTimeoutMS=2000&appName=vitamui-archive-search
 
 multipart:
   enabled: true

api/api-collect/collect/src/main/resources/application-dev.yml

@@ -12,7 +12,7 @@ spring:
         register: false
 
   mongodb:
-    uri: mongodb://mongod_dbuser_archivesearch:mongod_dbpwd_archivesearch@localhost:27018/archivesearch?connectTimeoutMS=2000
+    uri: mongodb://mongod_dbuser_archivesearch:mongod_dbpwd_archivesearch@localhost:27018/archivesearch?connectTimeoutMS=2000&appName=vitamui-collect
 
 multipart:
   enabled: true

api/api-iam/iam/src/main/resources/application-dev.yml

@@ -19,11 +19,11 @@ spring:
         enabled: false
         register: false
   mongodb:
-    uri: mongodb://mongod_dbuser_iam:mongod_dbpwd_iam@localhost:27018/iam?connectTimeoutMS=2000
+    uri: mongodb://mongod_dbuser_iam:mongod_dbpwd_iam@localhost:27018/iam?connectTimeoutMS=2000&appName=vitamui-iam
 
 discussion:
   mongodb:
-    uri: mongodb://mongod_dbuser_discussions:mongod_dbpwd_discussions@localhost:27018/discussions?connectTimeoutMS=2000
+    uri: mongodb://mongod_dbuser_discussions:mongod_dbpwd_discussions@localhost:27018/discussions?connectTimeoutMS=2000&appName=vitamui-iam-discussions
   scheduling:
     purgeTransactionDiscussions:
       enabled: true

api/api-security/security/src/main/resources/application-dev.yml

@@ -8,7 +8,7 @@ spring:
         enabled: false
         register: false
   mongodb:
-    uri: mongodb://mongod_dbuser_security:mongod_dbpwd_security@localhost:27018/security?connectTimeoutMS=2000
+    uri: mongodb://mongod_dbuser_security:mongod_dbpwd_security@localhost:27018/security?connectTimeoutMS=2000&appName=vitamui-security
 
   web:
     error:

cas/cas-server/src/main/config/application-dev.yml

@@ -65,7 +65,7 @@ cas:
     signing.key: =oebPIRe18A0cAeBdZCHkVlLPa_Kbthxo70iRpAhbk84dQGQj_8AOEMvEg3y7GAKYxtpF5nn6nx7vj5iU-eHStg
     encryption.key: 3FzNquczUjhmeJyqu251Ow
   service-registry.mongo:
-    client-uri: mongodb://mongod_dbuser_cas:mongod_dbpwd_cas@localhost:27018/cas
+    client-uri: mongodb://mongod_dbuser_cas:mongod_dbpwd_cas@localhost:27018/cas?appName=vitamui-cas
   authn:
     # Authent with x509 certificate :
     # x509:

cas/cas-server/src/main/config/application-recette.yml

@@ -46,7 +46,7 @@ management:
 cas:
   server.prefix: https://dev.vitamui.com:8080/cas
   service-registry.mongo:
-    client-uri: mongodb://mongod_dbuser_cas:mongod_dbpwd_cas@localhost:27018/cas
+    client-uri: mongodb://mongod_dbuser_cas:mongod_dbpwd_cas@localhost:27018/cas?appName=vitamui-cas
 
 # Root: iam-client - Internal VITAM-UI IAM service client
 iam-client:

deployment/README.rst

@@ -205,6 +205,28 @@ ATTENTION: il faut avoir déployer aussi les extras Vitam, sinon le déploiement
 
 en l'absence ce cette extra vars, le comportement par defaut est extra=no
 
+Exploitation
+=============
+
+Diagnostic des sessions logiques MongoDB
+----------------------------------------
+
+Le script ``scripts/diagnose_mongo_logical_sessions.js`` répond à la question « le *reaper* de sessions logiques tourne-t-il sur ce nœud ? », posée par le message ``Unable to add session ID ... into the cache because the number of active sessions is too high`` (bug #15294). Il est en lecture seule et peut être lancé sur un primaire de production.
+
+*Script* ::
+
+   mongosh "mongodb://<hôte>:<port>/admin?replicaSet=<replicaset>" \
+       --username <admin> --password \
+       --quiet --file scripts/diagnose_mongo_logical_sessions.js
+
+Il sort en 1 dès qu'un ``PROBLEM`` est relevé et en 0 sinon, ce qui permet de l'appeler depuis une supervision. Un ``WARNING`` demande une seconde passe : relancer le script un cycle de rafraîchissement plus tard et comparer ``activeSessionsCount`` distingue un *reaper* bloqué d'un nœud simplement peu sollicité.
+
+Le *reaper* ne tournant que sur le primaire, c'est là que le paramètre ``replicaSet`` amène la connexion, quel que soit le membre indiqué en amorce. C'est aussi ce qui rend cette URI inutilisable pour comparer les membres entre eux : elle lirait le primaire à chaque fois. Pour interroger un membre précis, il faut une connexion directe ::
+
+   mongosh "mongodb://<membre>:<port>/admin?directConnection=true" \
+       --username <admin> --password \
+       --quiet --file scripts/diagnose_mongo_logical_sessions.js
+
 Désinstallation
 =================
 

deployment/roles/mongo/handlers/main.yml

@@ -1,11 +1,5 @@
 ---
 
-- name: "mongo - restart service"
-  systemd:
-    name: vitamui-mongod
-    state: restarted
-    daemon_reload: yes
-
 - name: "mongo - reload consul configuration"
   systemd:
     name: "{{ consul.service_name | default('vitam-consul') }}"

deployment/roles/mongo/tasks/main.yml

@@ -5,11 +5,10 @@
     name:
       - mongodb-org-server
     state: present
-  register: result
+  register: mongod_package
   retries: "{{ packages_install_retries_number }}"
-  until: result is succeeded
+  until: mongod_package is succeeded
   delay: "{{ packages_install_retries_delay }}"
-  notify: "mongo - restart service"
   when: install_mode != "container"
 
 - name: Disable mongod default service
@@ -31,7 +30,7 @@
     owner: root
     group: root
     mode: "0644"
-  notify: "mongo - restart service"
+  register: mongod_service_file
 
 #### Configuration ####
 
@@ -51,7 +50,7 @@
     - log
     - script
     - tmp
-  notify: "mongo - restart service"
+  register: mongod_directories
   tags: update_mongodb_configuration
 
 - name: Create db directory
@@ -61,7 +60,72 @@
     owner: "{{ vitamui_defaults.users.vitamuidb | default('vitamuidb') }}"
     group: "{{ vitamui_defaults.users.group | default('vitamui') }}"
     mode: "{{ '0777' if install_mode == 'container' else vitam_defaults.folder.folder_permission }}"
-  notify: mongo - restart service
+  register: mongod_db_directory
+  tags: update_mongodb_configuration
+
+# The configuration written below deliberately declares no sharding.clusterRole
+# (bug #15294). mongod refuses to start when it finds a shardIdentity document
+# in admin.system.version without that role, and it only finds out at restart
+# time, once the new configuration has already replaced the old one. VitamUI
+# never registers its replica set into a sharded cluster, so the document should
+# not be there, but a node carrying one has to be caught before that point.
+- name: Check whether this node already holds MongoDB data
+  stat:
+    path: "{{ mongod_effective_db_path }}/WiredTiger"
+  register: mongod_data_directory
+  tags: update_mongodb_configuration
+
+# An arbiter is left out of the whole check: it replicates no admin.system.users,
+# so the authenticated lookup below can only be refused on one, and it holds no
+# collection to shard, so it carries no shardIdentity document to begin with.
+# Without this the guard further down would abort every deployment of an
+# inventory declaring an arbiter.
+- name: Look for a leftover shardIdentity document
+  command: >-
+    mongosh {{ ip_service }}:{{ mongodb.mongod_port }}/admin
+    -u {{ mongodb.admin.user }} -p {{ mongodb.admin.password }} --quiet
+    --eval 'print("VITAMUI_SHARD_IDENTITY=" + (db.system.version.findOne({_id: "shardIdentity"}) ? 1 : 0))'
+  register: mongod_shard_identity
+  no_log: "{{ hide_passwords_during_deploy }}"
+  failed_when: false
+  changed_when: false
+  when: not (mongo_arbiter | default(false) | bool)
+  tags: update_mongodb_configuration
+
+- name: Fail when a shardIdentity document would keep mongod from restarting
+  fail:
+    msg: >-
+      admin.system.version holds a shardIdentity document on this node, and the
+      mongod configuration deployed by VitamUI declares no sharding.clusterRole:
+      mongod would refuse to start on the next restart. Check first that this
+      replica set really is not a shard of a sharded cluster, then drop the
+      document with: mongosh {{ ip_service }}:{{ mongodb.mongod_port }}/admin
+      -u <admin> --eval 'db.system.version.deleteOne({_id: "shardIdentity"})'
+  when: mongod_shard_identity.stdout | default('') is search('VITAMUI_SHARD_IDENTITY=[1-9]')
+  tags: update_mongodb_configuration
+
+# Reading admin.system.version needs a running mongod and the admin credentials.
+# A node being installed has neither, and no sharding history to carry either,
+# so it goes through. A node that already holds data and still cannot be read is
+# the dangerous case: it may well carry the document, and letting it through
+# would only find that out when it refuses to restart.
+- name: Fail when the shardIdentity check could not run on a node holding data
+  fail:
+    msg: >-
+      This node already holds MongoDB data but admin.system.version could not be
+      read, so the shardIdentity check did not run: mongod is down, or the admin
+      credentials were refused. The configuration deployed by VitamUI declares no
+      sharding.clusterRole, and a node carrying a shardIdentity document would
+      only refuse to start once that configuration is in place. Start mongod and
+      run the deployment again, or check the node by hand with: mongosh
+      {{ ip_service }}:{{ mongodb.mongod_port }}/admin -u <admin> --eval
+      'db.system.version.findOne({_id: "shardIdentity"})' then re-run with
+      -e mongod_skip_shard_identity_check=true once it answers null.
+  when:
+    - not (mongo_arbiter | default(false) | bool)
+    - mongod_data_directory.stat.exists
+    - mongod_shard_identity.stdout | default('') is not search('VITAMUI_SHARD_IDENTITY=')
+    - not (mongod_skip_shard_identity_check | default(false) | bool)
   tags: update_mongodb_configuration
 
 - name: Create the mongod configuration server file
@@ -71,7 +135,7 @@
     owner: "{{ vitamui_defaults.users.vitamuidb | default('vitamuidb') }}"
     group: "{{ vitamui_defaults.users.group | default('vitamui') }}"
     mode: "{{ '0444' if install_mode == 'container' else vitam_defaults.folder.conf_permission }}"
-  notify: mongo - restart service
+  register: mongod_configuration
   tags: update_mongodb_configuration
 
 - name: Enable logrotate for vitamui-mongod
@@ -120,6 +184,46 @@
     - consul_conf
     - update_mongodb_configuration
 
+#### Rolling restart ####
+# mongod must never be restarted on every member at once: with the default
+# parallelism (forks = 20 in ansible.cfg) the whole replica set goes down within
+# the same second and the quorum is lost, where a member by member restart is
+# just a failover. The loop below is what serialises it, driven from a single
+# host so that each member is taken down, waited for, and only then followed by
+# the next one.
+
+# The package installation and the systemd unit carry no tag, so a run limited
+# to --tags update_mongodb_configuration never registers their variables. The
+# "changed" test raises on an undefined value rather than answering false, so
+# both are guarded: in such a run neither can have changed anyway, since neither
+# ran.
+- name: Check whether vitamui-mongod must be restarted
+  set_fact:
+    mongod_restart_required: >-
+      {{ (mongod_package is defined and mongod_package is changed)
+         or (mongod_service_file is defined and mongod_service_file is changed)
+         or mongod_directories is changed
+         or mongod_db_directory is changed
+         or mongod_configuration is changed }}
+  tags: update_mongodb_configuration
+
+- name: Restart vitamui-mongod, one replica set member at a time
+  include_tasks:
+    file: rolling_restart.yml
+    # Without apply, the tag below only gates the include itself: the tasks it
+    # brings in carry no tag of their own and a run limited to
+    # --tags update_mongodb_configuration would skip every restart and every
+    # wait, leaving the configuration it just wrote inactive.
+    apply:
+      tags: update_mongodb_configuration
+  vars:
+    mongod_member_restart_required: "{{ hostvars[mongod_member]['mongod_restart_required'] | default(false) | bool }}"
+  loop: "{{ groups['hosts_vitamui_mongod'] }}"
+  loop_control:
+    loop_var: mongod_member
+  run_once: true
+  tags: update_mongodb_configuration
+
 - name: Ensure vitamui-mongod is started
   service:
     name: vitamui-mongod

deployment/roles/mongo/tasks/rolling_restart.yml

@@ -0,0 +1,68 @@
+---
+
+# One member of the replica set, included once per member from main.yml. The
+# loop is what makes the sequence sequential, and it has to be: "throttle: 1"
+# on a block only serialises each task across the hosts, so every member would
+# be restarted before the first wait had even started. Measured on Ansible
+# 2.9.27, three hosts, a block of restart then sleep 3: with throttle the three
+# sequences all ran between t and t+9.8s, with this loop they ran back to back.
+
+# The include loops over every member of hosts_vitamui_mongod, including the
+# ones the current play does not cover: a run under --limit, or a member that
+# errored out earlier, leaves no mongod_restart_required fact behind. Defaulting
+# that to "nothing to do" is the only safe choice here, but it must not pass
+# unnoticed: such a member keeps running the configuration it had, which is the
+# very one this fix exists to replace.
+- name: "Report that the restart state of {{ mongod_member }} is unknown"
+  debug:
+    msg: >-
+      WARNING: {{ mongod_member }} is not part of this run, so whether it needs a
+      restart could not be determined and it is left untouched. It may still be
+      running with the previous mongod configuration, sharding.clusterRole
+      included. Run the mongo role against it before considering the replica set
+      up to date.
+  when: hostvars[mongod_member]['mongod_restart_required'] is not defined
+
+- name: "Restart vitamui-mongod on {{ mongod_member }}"
+  # Restarting the primary needs no explicit rs.stepDown(): a clean SIGTERM
+  # makes mongod hand the primary role over before exiting.
+  systemd:
+    name: vitamui-mongod
+    state: restarted
+    daemon_reload: yes
+  delegate_to: "{{ mongod_member }}"
+  when: mongod_member_restart_required | bool
+
+- name: "Wait for the vitamui-mongod port to be open on {{ mongod_member }}"
+  wait_for:
+    host: "{{ hostvars[mongod_member]['ip_service'] }}"
+    port: "{{ mongodb.mongod_port }}"
+    timeout: "{{ vitamui_defaults.services.start_timeout }}"
+  delegate_to: "{{ mongod_member }}"
+  when: mongod_member_restart_required | bool
+
+- name: "Wait for {{ mongod_member }} to be back in the replica set"
+  # db.hello() is answered before authentication, so this holds whether or not
+  # the security block has already been added to mongod.conf. A member with no
+  # setName belongs to no replica set yet and has nothing to rejoin, which is
+  # the case on a first installation since the set is only initiated further
+  # down the role: it answers at once instead of waiting.
+  #
+  # The state is read here, after the restart, and never before it: a probe run
+  # beforehand cannot tell "belongs to no replica set" from "could not be
+  # reached", and taking the second for the first would move on to the next
+  # member while this one is still recovering - two members down at once on a
+  # three member set, which is exactly the quorum loss the rolling restart is
+  # there to avoid. Read afterwards, an unreachable member simply keeps the
+  # retries going and eventually stops the play on itself.
+  command: >-
+    mongosh --host {{ hostvars[mongod_member]['ip_service'] }} --port {{ mongodb.mongod_port }}
+    --quiet --eval 'const member = db.hello();
+    quit(!member.setName || member.isWritablePrimary || member.secondary || member.arbiterOnly ? 0 : 1)'
+  delegate_to: "{{ mongod_member }}"
+  register: mongod_member_state
+  until: mongod_member_state.rc == 0
+  retries: "{{ vitamui_defaults.services.status_retries_number | default(60) }}"
+  delay: "{{ vitamui_defaults.services.status_retries_delay | default(5) }}"
+  changed_when: false
+  when: mongod_member_restart_required | bool

deployment/roles/mongo/templates/mongod.conf.j2

@@ -43,8 +43,18 @@ replication:
   replSetName: shard{{ mongo_shard_id | default(0) }} # name of the replica set
   enableMajorityReadConcern: true
 
-sharding:
-  clusterRole: shardsvr # role du shard
+# NOTE: do NOT declare "sharding.clusterRole: shardsvr" here.
+# VitamUI deploys a single replica set and no mongos / config server (the
+# inventory only defines hosts_vitamui_mongod). A mongod declaring itself a
+# shard server delegates the creation of config.system.sessions to the cluster,
+# which therefore never happens: the logical session cache is never reaped and
+# grows until it hits maxSessions, at which point MongoDB rejects every new
+# session until the process is restarted (bug #15294).
+# The "shard0" replica set name above comes from the same VITAM heritage, but
+# unlike the cluster role it is load bearing: it is the replicaSet= of every
+# application URI, the _id init-replica.js.j2 initiates the set with, and what
+# mongo_init connects through. Renaming it would mean rewriting all of them for
+# a purely cosmetic gain.
 
 # During (re)initialization, authentication is temporarily disabled to ensure replica-set & user creation/update.
 # Authentication will be (re)enabled during mongo_configure role execution