feat(monitoring): generated Zabbix templates with validation

.github/workflows/zabbix.yml

@@ -0,0 +1,30 @@
+# Imports the generated Zabbix templates into real Zabbix 6.0 and 7.0 servers and checks items and
+# triggers against recorded responses (`make zabbix-validate`). Slow, so it runs only when the
+# monitoring code or the templates change, and it is not required to merge.
+name: Zabbix
+
+on:
+  pull_request:
+    paths:
+      - 'orchestrator/monitoring/**'
+      - 'deploy/zabbix/**'
+      - 'deploy/zabbix-test/**'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: zabbix-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validate:
+    name: make zabbix-validate
+    runs-on: ubuntu-24.04
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - run: make zabbix-validate

Makefile

@@ -19,11 +19,12 @@ PY_PACKAGES = protocol worker receiver orchestrator
 
 FRONTEND_DEPS = frontend/node_modules/.package-lock.json
 
-.PHONY: help dev-image lock lint fmt test test-postgres check frontend-install frontend-lint frontend-test frontend-build \
+.PHONY: help dev-image lock lint fmt test test-postgres check zabbix-template zabbix-validate frontend-install frontend-lint frontend-test frontend-build \
 	images version set-version dev-up dev-down dev-start dev-stop dev-status dev-reset dev-ps dev-logs dev-seed dev-approve
 
 help:
 	@echo "Quality:   lint fmt test test-postgres frontend-lint frontend-test check"
+	@echo "Zabbix:    zabbix-template zabbix-validate"
 	@echo "Deps:      lock dev-image frontend-install"
 	@echo "Build:     frontend-build images (IMAGE_TAG=$(IMAGE_TAG))"
 	@echo "Release:   version set-version V=X.Y.Z"
@@ -69,6 +70,36 @@ test-postgres: dev-image
 
 check: lint test test-postgres frontend-lint frontend-test frontend-build
 
+# The generated Zabbix templates (monitoring/zabbix.py), committed under deploy/zabbix/; a test keeps them in step.
+ZABBIX_VERSIONS = 6.0 7.0
+ZABBIX_TEST = DEV_IMAGE=$(DEV_IMAGE) docker compose -f deploy/zabbix-test/compose.yaml
+zabbix-template: dev-image
+	$(DOCKER_RUN) -e DJANGO_SETTINGS_MODULE=devicevault.test_settings -w /workspace/orchestrator $(DEV_IMAGE) \
+		sh -c 'set -e; for v in $(ZABBIX_VERSIONS); do \
+			python manage.py zabbix_template --version $$v --output ../deploy/zabbix/devicevault_zabbix_$$v.yaml; done'
+
+# Import the committed templates into real Zabbix servers (each version in turn, deploy/zabbix-test/) and check
+# them against recorded responses. Slow, so not part of `check`; zabbix.yml runs it on monitoring changes.
+# The images each version runs, pinned: <version>:<zabbix image tag>.
+ZABBIX_IMAGES = 6.0:alpine-6.0.49 7.0:alpine-7.0.32
+ZABBIX_POSTGRES_IMAGE = postgres:16-alpine
+zabbix-validate: dev-image
+	@status=0; for pair in $(ZABBIX_IMAGES); do \
+		v=$${pair%%:*}; export ZABBIX_TAG=$${pair#*:} POSTGRES_IMAGE=$(ZABBIX_POSTGRES_IMAGE); \
+		echo "== Zabbix $$v ($$ZABBIX_TAG)"; \
+		$(ZABBIX_TEST) up -d --quiet-pull postgres server web stub \
+			&& $(ZABBIX_TEST) run --rm validate --version $$v --phase import \
+			&& $(ZABBIX_TEST) exec -T postgres psql -q -U zabbix -d zabbix -c 'VACUUM ANALYZE' \
+			&& $(ZABBIX_TEST) run --rm validate --version $$v --phase check \
+			|| { status=1; $(ZABBIX_TEST) logs --no-color server web postgres \
+				| grep -viE 'Invalid token|became not supported|PHP Deprecated|POST /api_jsonrpc' | tail -200; \
+				$(ZABBIX_TEST) exec -T postgres psql -U zabbix -d zabbix -c \
+					"select pid, state, now() - xact_start as open_for, left(query, 200) as query \
+					 from pg_stat_activity where datname = 'zabbix' order by xact_start"; }; \
+		$(ZABBIX_TEST) down --volumes --remove-orphans --timeout 5; \
+		[ $$status -eq 0 ] || break; \
+	done; exit $$status
+
 # Reinstall only when the lockfile changes.
 $(FRONTEND_DEPS): frontend/package.json frontend/package-lock.json
 	$(NODE) npm ci --no-audit --no-fund

deploy/zabbix-test/compose.yaml

@@ -0,0 +1,63 @@
+# A throwaway Zabbix (server, web/API, PostgreSQL) for `make zabbix-validate`, which imports the
+# generated templates (deploy/zabbix/) and checks them against a stub serving recorded responses.
+# One version at a time: `make zabbix-validate` sets ZABBIX_TAG and POSTGRES_IMAGE (ZABBIX_IMAGES in the Makefile).
+# Nothing is published on the host; the validator reaches the API over the compose network.
+name: devicevault-zabbix-test
+
+x-db: &db
+  POSTGRES_USER: zabbix
+  POSTGRES_PASSWORD: zabbix-test-only
+  POSTGRES_DB: zabbix
+
+services:
+  postgres:
+    image: ${POSTGRES_IMAGE:?set by make zabbix-validate}
+    environment: *db
+    # Lock waits show up in the logs `make zabbix-validate` prints when it fails.
+    command: [postgres, -c, log_lock_waits=on, -c, deadlock_timeout=2s]
+    tmpfs:
+      - /var/lib/postgresql/data
+
+  server:
+    image: zabbix/zabbix-server-pgsql:${ZABBIX_TAG:?set by make zabbix-validate}
+    depends_on: [postgres]
+    environment:
+      <<: *db
+      DB_SERVER_HOST: postgres
+      # Pick up new hosts and items quickly.
+      ZBX_CACHEUPDATEFREQUENCY: '10'
+      ZBX_STARTHTTPPOLLERS: '5'
+      ZBX_LOGSLOWQUERIES: '2000'
+
+  web:
+    image: zabbix/zabbix-web-nginx-pgsql:${ZABBIX_TAG}
+    depends_on: [postgres, server]
+    environment:
+      <<: *db
+      DB_SERVER_HOST: postgres
+      ZBX_SERVER_HOST: server
+      PHP_TZ: UTC
+      # host.create links templates with hundreds of items; give it time on small CI runners (validate.py API_TIMEOUT).
+      ZBX_MAXEXECUTIONTIME: '600'
+
+  # Recorded orchestrator and relay answers (stdlib only), at http://stub:8000.
+  stub:
+    image: ${DEV_IMAGE:-devicevault-dev}
+    user: '${DV_UID:-1000}:${DV_GID:-1000}'
+    volumes:
+      - ../zabbix-test:/zabbix-test:ro
+    command: [python, -I, /zabbix-test/stub.py, /zabbix-test/responses]
+
+  validate:
+    image: ${DEV_IMAGE:-devicevault-dev}
+    profiles: [validate]
+    user: '${DV_UID:-1000}:${DV_GID:-1000}'
+    depends_on: [web, stub]
+    environment:
+      HOME: /tmp
+      ZABBIX_URL: http://web:8080/api_jsonrpc.php
+      STUB_URL: http://stub:8000
+    volumes:
+      - ../zabbix-test:/zabbix-test:ro
+      - ../zabbix:/templates:ro
+    entrypoint: [python, -I, /zabbix-test/validate.py]

deploy/zabbix-test/responses/device-groups.json

@@ -0,0 +1,192 @@
+{
+  "data": [
+    {
+      "id": "3",
+      "name": "Core",
+      "stored_1h": 2,
+      "failed_1h": 1,
+      "timed_out_1h": 0,
+      "finished_1h": 3,
+      "success_ratio_1h": 66.67,
+      "failure_ratio_1h": 33.33,
+      "retried_1h": 0,
+      "changed_1h": 1,
+      "bytes_stored_1h": 36426,
+      "lines_added_1h": 4,
+      "lines_removed_1h": 1,
+      "lines_modified_1h": 2,
+      "stored_24h": 40,
+      "failed_24h": 8,
+      "timed_out_24h": 2,
+      "finished_24h": 50,
+      "success_ratio_24h": 80.0,
+      "failure_ratio_24h": 20.0,
+      "retried_24h": 3,
+      "changed_24h": 6,
+      "bytes_stored_24h": 728520,
+      "lines_added_24h": 31,
+      "lines_removed_24h": 12,
+      "lines_modified_24h": 9,
+      "stored_7d": 290,
+      "failed_7d": 45,
+      "timed_out_7d": 5,
+      "finished_7d": 340,
+      "success_ratio_7d": 85.29,
+      "failure_ratio_7d": 14.71,
+      "retried_7d": 17,
+      "changed_7d": 41,
+      "bytes_stored_7d": 5281770,
+      "lines_added_7d": 204,
+      "lines_removed_7d": 88,
+      "lines_modified_7d": 63,
+      "stored_total": 5120,
+      "failed_total": 410,
+      "timed_out_total": 37,
+      "failures_connection_24h": 3,
+      "failures_connection_total": 150,
+      "failures_auth_24h": 4,
+      "failures_auth_total": 61,
+      "failures_timeout_24h": 1,
+      "failures_timeout_total": 40,
+      "failures_command_24h": 0,
+      "failures_command_total": 12,
+      "failures_verification_24h": 0,
+      "failures_verification_total": 0,
+      "failures_storage_24h": 0,
+      "failures_storage_total": 3,
+      "failures_integrity_24h": 0,
+      "failures_integrity_total": 1,
+      "failures_push_24h": 0,
+      "failures_push_total": 0,
+      "failures_orphaned_24h": 2,
+      "failures_orphaned_total": 37,
+      "failures_unknown_24h": 0,
+      "failures_unknown_total": 143,
+      "queued": 1,
+      "running": 2,
+      "duration_collect_avg": 6.2,
+      "duration_collect_p50": 4.0,
+      "duration_collect_p95": 21.5,
+      "duration_collect_max": 48.0,
+      "duration_store_avg": 0.8,
+      "duration_store_p50": 0.5,
+      "duration_store_p95": 2.1,
+      "duration_store_max": 6.4,
+      "duration_wait_avg": 35.0,
+      "duration_wait_p50": 2.0,
+      "duration_wait_p95": 870.0,
+      "duration_wait_max": 1810.0,
+      "duration_total_avg": 42.0,
+      "duration_total_p50": 7.0,
+      "duration_total_p95": 900.0,
+      "duration_total_max": 1830.0,
+      "devices_total": 40,
+      "devices_enabled": 39,
+      "devices_disabled": 1,
+      "devices_inactive": 2,
+      "devices_never": 2,
+      "devices_overdue": 4,
+      "devices_failing_1": 6,
+      "devices_failing_3": 2,
+      "devices_failing_5": 1,
+      "oldest_success_age": 259200,
+      "oldest_success_device": "Edge 01",
+      "backups_held": 1480,
+      "bytes_held": 26954000
+    },
+    {
+      "stored_1h": 0,
+      "failed_1h": 0,
+      "timed_out_1h": 0,
+      "finished_1h": 0,
+      "success_ratio_1h": null,
+      "failure_ratio_1h": null,
+      "retried_1h": 0,
+      "changed_1h": 0,
+      "bytes_stored_1h": 0,
+      "lines_added_1h": 0,
+      "lines_removed_1h": 0,
+      "lines_modified_1h": 0,
+      "stored_24h": 0,
+      "failed_24h": 0,
+      "timed_out_24h": 0,
+      "finished_24h": 0,
+      "success_ratio_24h": null,
+      "failure_ratio_24h": null,
+      "retried_24h": 0,
+      "changed_24h": 0,
+      "bytes_stored_24h": 0,
+      "lines_added_24h": 0,
+      "lines_removed_24h": 0,
+      "lines_modified_24h": 0,
+      "stored_7d": 0,
+      "failed_7d": 0,
+      "timed_out_7d": 0,
+      "finished_7d": 0,
+      "success_ratio_7d": null,
+      "failure_ratio_7d": null,
+      "retried_7d": 0,
+      "changed_7d": 0,
+      "bytes_stored_7d": 0,
+      "lines_added_7d": 0,
+      "lines_removed_7d": 0,
+      "lines_modified_7d": 0,
+      "stored_total": 0,
+      "failed_total": 0,
+      "timed_out_total": 0,
+      "failures_connection_24h": 0,
+      "failures_connection_total": 0,
+      "failures_auth_24h": 0,
+      "failures_auth_total": 0,
+      "failures_timeout_24h": 0,
+      "failures_timeout_total": 0,
+      "failures_command_24h": 0,
+      "failures_command_total": 0,
+      "failures_verification_24h": 0,
+      "failures_verification_total": 0,
+      "failures_storage_24h": 0,
+      "failures_storage_total": 0,
+      "failures_integrity_24h": 0,
+      "failures_integrity_total": 0,
+      "failures_push_24h": 0,
+      "failures_push_total": 0,
+      "failures_orphaned_24h": 0,
+      "failures_orphaned_total": 0,
+      "failures_unknown_24h": 0,
+      "failures_unknown_total": 0,
+      "queued": 0,
+      "running": 0,
+      "duration_collect_avg": null,
+      "duration_collect_p50": null,
+      "duration_collect_p95": null,
+      "duration_collect_max": null,
+      "duration_store_avg": null,
+      "duration_store_p50": null,
+      "duration_store_p95": null,
+      "duration_store_max": null,
+      "duration_wait_avg": null,
+      "duration_wait_p50": null,
+      "duration_wait_p95": null,
+      "duration_wait_max": null,
+      "duration_total_avg": null,
+      "duration_total_p50": null,
+      "duration_total_p95": null,
+      "duration_total_max": null,
+      "devices_total": 0,
+      "devices_enabled": 0,
+      "devices_disabled": 0,
+      "devices_inactive": 0,
+      "devices_never": 0,
+      "devices_overdue": 0,
+      "devices_failing_1": 0,
+      "devices_failing_3": 0,
+      "devices_failing_5": 0,
+      "oldest_success_age": 0,
+      "backups_held": 0,
+      "bytes_held": 0,
+      "id": "ungrouped",
+      "name": "Ungrouped",
+      "oldest_success_device": ""
+    }
+  ]
+}

deploy/zabbix-test/responses/device.json

@@ -0,0 +1,34 @@
+{
+  "found": 1,
+  "id": "12",
+  "name": "Edge 01",
+  "group": "Core",
+  "ip": "192.0.2.10",
+  "enabled": 1,
+  "lifecycle_code": 0,
+  "status_code": 2,
+  "last_attempt_at": 1792996400,
+  "last_success_at": 1792737200,
+  "last_success_age": 259200,
+  "last_failure_at": 1792996400,
+  "failure_reason_code": 2,
+  "consecutive_failures": 3,
+  "expected_interval": 86400,
+  "next_due_at": 1792866800,
+  "overdue": 1,
+  "last_collect_duration": 2.5,
+  "last_store_duration": 0.4,
+  "last_duration": 4.1,
+  "last_size": 18213,
+  "last_changed": 1,
+  "last_lines_added": 4,
+  "last_lines_removed": 1,
+  "last_lines_modified": 2,
+  "backups_held": 30,
+  "pinned": 1,
+  "push_mode": "off",
+  "last_push_at": 0,
+  "success_ratio_7d": 33.33,
+  "published_at": "2026-10-26T05:00:00+00:00",
+  "published_age": 3000
+}

deploy/zabbix-test/responses/receivers.json

@@ -0,0 +1,12 @@
+{
+  "data": [
+    {
+      "id": "5",
+      "name": "receiver-1",
+      "online": 1,
+      "accepted_24h": 5,
+      "refused_24h": 3,
+      "last_push_at": 1792996400
+    }
+  ]
+}