diff --git a/.github/workflows/zabbix.yml b/.github/workflows/zabbix.yml new file mode 100644 index 0000000..97e6043 --- /dev/null +++ b/.github/workflows/zabbix.yml @@ -0,0 +1,30 @@ +# Imports the generated Zabbix templates into real Zabbix 6.0 and 7.0 servers and checks items and +# triggers against recorded responses (`make zabbix-validate`). Slow, so it runs only when the +# monitoring code or the templates change, and it is not required to merge. +name: Zabbix + +on: + pull_request: + paths: + - 'orchestrator/monitoring/**' + - 'deploy/zabbix/**' + - 'deploy/zabbix-test/**' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: zabbix-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + name: make zabbix-validate + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - run: make zabbix-validate diff --git a/Makefile b/Makefile index db1fc43..99ec091 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,12 @@ PY_PACKAGES = protocol worker receiver orchestrator FRONTEND_DEPS = frontend/node_modules/.package-lock.json -.PHONY: help dev-image lock lint fmt test test-postgres check frontend-install frontend-lint frontend-test frontend-build \ +.PHONY: help dev-image lock lint fmt test test-postgres check zabbix-template zabbix-validate frontend-install frontend-lint frontend-test frontend-build \ images version set-version dev-up dev-down dev-start dev-stop dev-status dev-reset dev-ps dev-logs dev-seed dev-approve help: @echo "Quality: lint fmt test test-postgres frontend-lint frontend-test check" + @echo "Zabbix: zabbix-template zabbix-validate" @echo "Deps: lock dev-image frontend-install" @echo "Build: frontend-build images (IMAGE_TAG=$(IMAGE_TAG))" @echo "Release: version set-version V=X.Y.Z" @@ -69,6 +70,36 @@ test-postgres: dev-image check: lint test test-postgres frontend-lint frontend-test frontend-build +# The generated Zabbix templates (monitoring/zabbix.py), committed under deploy/zabbix/; a test keeps them in step. +ZABBIX_VERSIONS = 6.0 7.0 +ZABBIX_TEST = DEV_IMAGE=$(DEV_IMAGE) docker compose -f deploy/zabbix-test/compose.yaml +zabbix-template: dev-image + $(DOCKER_RUN) -e DJANGO_SETTINGS_MODULE=devicevault.test_settings -w /workspace/orchestrator $(DEV_IMAGE) \ + sh -c 'set -e; for v in $(ZABBIX_VERSIONS); do \ + python manage.py zabbix_template --version $$v --output ../deploy/zabbix/devicevault_zabbix_$$v.yaml; done' + +# Import the committed templates into real Zabbix servers (each version in turn, deploy/zabbix-test/) and check +# them against recorded responses. Slow, so not part of `check`; zabbix.yml runs it on monitoring changes. +# The images each version runs, pinned: :. +ZABBIX_IMAGES = 6.0:alpine-6.0.49 7.0:alpine-7.0.32 +ZABBIX_POSTGRES_IMAGE = postgres:16-alpine +zabbix-validate: dev-image + @status=0; for pair in $(ZABBIX_IMAGES); do \ + v=$${pair%%:*}; export ZABBIX_TAG=$${pair#*:} POSTGRES_IMAGE=$(ZABBIX_POSTGRES_IMAGE); \ + echo "== Zabbix $$v ($$ZABBIX_TAG)"; \ + $(ZABBIX_TEST) up -d --quiet-pull postgres server web stub \ + && $(ZABBIX_TEST) run --rm validate --version $$v --phase import \ + && $(ZABBIX_TEST) exec -T postgres psql -q -U zabbix -d zabbix -c 'VACUUM ANALYZE' \ + && $(ZABBIX_TEST) run --rm validate --version $$v --phase check \ + || { status=1; $(ZABBIX_TEST) logs --no-color server web postgres \ + | grep -viE 'Invalid token|became not supported|PHP Deprecated|POST /api_jsonrpc' | tail -200; \ + $(ZABBIX_TEST) exec -T postgres psql -U zabbix -d zabbix -c \ + "select pid, state, now() - xact_start as open_for, left(query, 200) as query \ + from pg_stat_activity where datname = 'zabbix' order by xact_start"; }; \ + $(ZABBIX_TEST) down --volumes --remove-orphans --timeout 5; \ + [ $$status -eq 0 ] || break; \ + done; exit $$status + # Reinstall only when the lockfile changes. $(FRONTEND_DEPS): frontend/package.json frontend/package-lock.json $(NODE) npm ci --no-audit --no-fund diff --git a/deploy/zabbix-test/compose.yaml b/deploy/zabbix-test/compose.yaml new file mode 100644 index 0000000..0ceda16 --- /dev/null +++ b/deploy/zabbix-test/compose.yaml @@ -0,0 +1,63 @@ +# A throwaway Zabbix (server, web/API, PostgreSQL) for `make zabbix-validate`, which imports the +# generated templates (deploy/zabbix/) and checks them against a stub serving recorded responses. +# One version at a time: `make zabbix-validate` sets ZABBIX_TAG and POSTGRES_IMAGE (ZABBIX_IMAGES in the Makefile). +# Nothing is published on the host; the validator reaches the API over the compose network. +name: devicevault-zabbix-test + +x-db: &db + POSTGRES_USER: zabbix + POSTGRES_PASSWORD: zabbix-test-only + POSTGRES_DB: zabbix + +services: + postgres: + image: ${POSTGRES_IMAGE:?set by make zabbix-validate} + environment: *db + # Lock waits show up in the logs `make zabbix-validate` prints when it fails. + command: [postgres, -c, log_lock_waits=on, -c, deadlock_timeout=2s] + tmpfs: + - /var/lib/postgresql/data + + server: + image: zabbix/zabbix-server-pgsql:${ZABBIX_TAG:?set by make zabbix-validate} + depends_on: [postgres] + environment: + <<: *db + DB_SERVER_HOST: postgres + # Pick up new hosts and items quickly. + ZBX_CACHEUPDATEFREQUENCY: '10' + ZBX_STARTHTTPPOLLERS: '5' + ZBX_LOGSLOWQUERIES: '2000' + + web: + image: zabbix/zabbix-web-nginx-pgsql:${ZABBIX_TAG} + depends_on: [postgres, server] + environment: + <<: *db + DB_SERVER_HOST: postgres + ZBX_SERVER_HOST: server + PHP_TZ: UTC + # host.create links templates with hundreds of items; give it time on small CI runners (validate.py API_TIMEOUT). + ZBX_MAXEXECUTIONTIME: '600' + + # Recorded orchestrator and relay answers (stdlib only), at http://stub:8000. + stub: + image: ${DEV_IMAGE:-devicevault-dev} + user: '${DV_UID:-1000}:${DV_GID:-1000}' + volumes: + - ../zabbix-test:/zabbix-test:ro + command: [python, -I, /zabbix-test/stub.py, /zabbix-test/responses] + + validate: + image: ${DEV_IMAGE:-devicevault-dev} + profiles: [validate] + user: '${DV_UID:-1000}:${DV_GID:-1000}' + depends_on: [web, stub] + environment: + HOME: /tmp + ZABBIX_URL: http://web:8080/api_jsonrpc.php + STUB_URL: http://stub:8000 + volumes: + - ../zabbix-test:/zabbix-test:ro + - ../zabbix:/templates:ro + entrypoint: [python, -I, /zabbix-test/validate.py] diff --git a/deploy/zabbix-test/responses/device-groups.json b/deploy/zabbix-test/responses/device-groups.json new file mode 100644 index 0000000..8e625ca --- /dev/null +++ b/deploy/zabbix-test/responses/device-groups.json @@ -0,0 +1,192 @@ +{ + "data": [ + { + "id": "3", + "name": "Core", + "stored_1h": 2, + "failed_1h": 1, + "timed_out_1h": 0, + "finished_1h": 3, + "success_ratio_1h": 66.67, + "failure_ratio_1h": 33.33, + "retried_1h": 0, + "changed_1h": 1, + "bytes_stored_1h": 36426, + "lines_added_1h": 4, + "lines_removed_1h": 1, + "lines_modified_1h": 2, + "stored_24h": 40, + "failed_24h": 8, + "timed_out_24h": 2, + "finished_24h": 50, + "success_ratio_24h": 80.0, + "failure_ratio_24h": 20.0, + "retried_24h": 3, + "changed_24h": 6, + "bytes_stored_24h": 728520, + "lines_added_24h": 31, + "lines_removed_24h": 12, + "lines_modified_24h": 9, + "stored_7d": 290, + "failed_7d": 45, + "timed_out_7d": 5, + "finished_7d": 340, + "success_ratio_7d": 85.29, + "failure_ratio_7d": 14.71, + "retried_7d": 17, + "changed_7d": 41, + "bytes_stored_7d": 5281770, + "lines_added_7d": 204, + "lines_removed_7d": 88, + "lines_modified_7d": 63, + "stored_total": 5120, + "failed_total": 410, + "timed_out_total": 37, + "failures_connection_24h": 3, + "failures_connection_total": 150, + "failures_auth_24h": 4, + "failures_auth_total": 61, + "failures_timeout_24h": 1, + "failures_timeout_total": 40, + "failures_command_24h": 0, + "failures_command_total": 12, + "failures_verification_24h": 0, + "failures_verification_total": 0, + "failures_storage_24h": 0, + "failures_storage_total": 3, + "failures_integrity_24h": 0, + "failures_integrity_total": 1, + "failures_push_24h": 0, + "failures_push_total": 0, + "failures_orphaned_24h": 2, + "failures_orphaned_total": 37, + "failures_unknown_24h": 0, + "failures_unknown_total": 143, + "queued": 1, + "running": 2, + "duration_collect_avg": 6.2, + "duration_collect_p50": 4.0, + "duration_collect_p95": 21.5, + "duration_collect_max": 48.0, + "duration_store_avg": 0.8, + "duration_store_p50": 0.5, + "duration_store_p95": 2.1, + "duration_store_max": 6.4, + "duration_wait_avg": 35.0, + "duration_wait_p50": 2.0, + "duration_wait_p95": 870.0, + "duration_wait_max": 1810.0, + "duration_total_avg": 42.0, + "duration_total_p50": 7.0, + "duration_total_p95": 900.0, + "duration_total_max": 1830.0, + "devices_total": 40, + "devices_enabled": 39, + "devices_disabled": 1, + "devices_inactive": 2, + "devices_never": 2, + "devices_overdue": 4, + "devices_failing_1": 6, + "devices_failing_3": 2, + "devices_failing_5": 1, + "oldest_success_age": 259200, + "oldest_success_device": "Edge 01", + "backups_held": 1480, + "bytes_held": 26954000 + }, + { + "stored_1h": 0, + "failed_1h": 0, + "timed_out_1h": 0, + "finished_1h": 0, + "success_ratio_1h": null, + "failure_ratio_1h": null, + "retried_1h": 0, + "changed_1h": 0, + "bytes_stored_1h": 0, + "lines_added_1h": 0, + "lines_removed_1h": 0, + "lines_modified_1h": 0, + "stored_24h": 0, + "failed_24h": 0, + "timed_out_24h": 0, + "finished_24h": 0, + "success_ratio_24h": null, + "failure_ratio_24h": null, + "retried_24h": 0, + "changed_24h": 0, + "bytes_stored_24h": 0, + "lines_added_24h": 0, + "lines_removed_24h": 0, + "lines_modified_24h": 0, + "stored_7d": 0, + "failed_7d": 0, + "timed_out_7d": 0, + "finished_7d": 0, + "success_ratio_7d": null, + "failure_ratio_7d": null, + "retried_7d": 0, + "changed_7d": 0, + "bytes_stored_7d": 0, + "lines_added_7d": 0, + "lines_removed_7d": 0, + "lines_modified_7d": 0, + "stored_total": 0, + "failed_total": 0, + "timed_out_total": 0, + "failures_connection_24h": 0, + "failures_connection_total": 0, + "failures_auth_24h": 0, + "failures_auth_total": 0, + "failures_timeout_24h": 0, + "failures_timeout_total": 0, + "failures_command_24h": 0, + "failures_command_total": 0, + "failures_verification_24h": 0, + "failures_verification_total": 0, + "failures_storage_24h": 0, + "failures_storage_total": 0, + "failures_integrity_24h": 0, + "failures_integrity_total": 0, + "failures_push_24h": 0, + "failures_push_total": 0, + "failures_orphaned_24h": 0, + "failures_orphaned_total": 0, + "failures_unknown_24h": 0, + "failures_unknown_total": 0, + "queued": 0, + "running": 0, + "duration_collect_avg": null, + "duration_collect_p50": null, + "duration_collect_p95": null, + "duration_collect_max": null, + "duration_store_avg": null, + "duration_store_p50": null, + "duration_store_p95": null, + "duration_store_max": null, + "duration_wait_avg": null, + "duration_wait_p50": null, + "duration_wait_p95": null, + "duration_wait_max": null, + "duration_total_avg": null, + "duration_total_p50": null, + "duration_total_p95": null, + "duration_total_max": null, + "devices_total": 0, + "devices_enabled": 0, + "devices_disabled": 0, + "devices_inactive": 0, + "devices_never": 0, + "devices_overdue": 0, + "devices_failing_1": 0, + "devices_failing_3": 0, + "devices_failing_5": 0, + "oldest_success_age": 0, + "backups_held": 0, + "bytes_held": 0, + "id": "ungrouped", + "name": "Ungrouped", + "oldest_success_device": "" + } + ] +} \ No newline at end of file diff --git a/deploy/zabbix-test/responses/device.json b/deploy/zabbix-test/responses/device.json new file mode 100644 index 0000000..fa0a1ec --- /dev/null +++ b/deploy/zabbix-test/responses/device.json @@ -0,0 +1,34 @@ +{ + "found": 1, + "id": "12", + "name": "Edge 01", + "group": "Core", + "ip": "192.0.2.10", + "enabled": 1, + "lifecycle_code": 0, + "status_code": 2, + "last_attempt_at": 1792996400, + "last_success_at": 1792737200, + "last_success_age": 259200, + "last_failure_at": 1792996400, + "failure_reason_code": 2, + "consecutive_failures": 3, + "expected_interval": 86400, + "next_due_at": 1792866800, + "overdue": 1, + "last_collect_duration": 2.5, + "last_store_duration": 0.4, + "last_duration": 4.1, + "last_size": 18213, + "last_changed": 1, + "last_lines_added": 4, + "last_lines_removed": 1, + "last_lines_modified": 2, + "backups_held": 30, + "pinned": 1, + "push_mode": "off", + "last_push_at": 0, + "success_ratio_7d": 33.33, + "published_at": "2026-10-26T05:00:00+00:00", + "published_age": 3000 +} diff --git a/deploy/zabbix-test/responses/receivers.json b/deploy/zabbix-test/responses/receivers.json new file mode 100644 index 0000000..d929d8b --- /dev/null +++ b/deploy/zabbix-test/responses/receivers.json @@ -0,0 +1,12 @@ +{ + "data": [ + { + "id": "5", + "name": "receiver-1", + "online": 1, + "accepted_24h": 5, + "refused_24h": 3, + "last_push_at": 1792996400 + } + ] +} \ No newline at end of file diff --git a/deploy/zabbix-test/responses/storage-targets.json b/deploy/zabbix-test/responses/storage-targets.json new file mode 100644 index 0000000..205ae14 --- /dev/null +++ b/deploy/zabbix-test/responses/storage-targets.json @@ -0,0 +1,15 @@ +{ + "data": [ + { + "id": "1", + "name": "local", + "driver": "local", + "workers_online": 0, + "last_stored_at": 1792996400, + "storage_failures_24h": 0, + "backups_held": 1480, + "bytes_held": 26954000, + "bytes_stored_24h": 728520 + } + ] +} \ No newline at end of file diff --git a/deploy/zabbix-test/responses/summary.json b/deploy/zabbix-test/responses/summary.json new file mode 100644 index 0000000..9837753 --- /dev/null +++ b/deploy/zabbix-test/responses/summary.json @@ -0,0 +1,116 @@ +{ + "stored_1h": 2, + "failed_1h": 1, + "timed_out_1h": 0, + "finished_1h": 3, + "success_ratio_1h": 66.67, + "failure_ratio_1h": 33.33, + "retried_1h": 0, + "changed_1h": 1, + "bytes_stored_1h": 36426, + "lines_added_1h": 4, + "lines_removed_1h": 1, + "lines_modified_1h": 2, + "stored_24h": 40, + "failed_24h": 8, + "timed_out_24h": 2, + "finished_24h": 50, + "success_ratio_24h": 80.0, + "failure_ratio_24h": 20.0, + "retried_24h": 3, + "changed_24h": 6, + "bytes_stored_24h": 728520, + "lines_added_24h": 31, + "lines_removed_24h": 12, + "lines_modified_24h": 9, + "stored_7d": 290, + "failed_7d": 45, + "timed_out_7d": 5, + "finished_7d": 340, + "success_ratio_7d": 85.29, + "failure_ratio_7d": 14.71, + "retried_7d": 17, + "changed_7d": 41, + "bytes_stored_7d": 5281770, + "lines_added_7d": 204, + "lines_removed_7d": 88, + "lines_modified_7d": 63, + "stored_total": 5120, + "failed_total": 410, + "timed_out_total": 37, + "failures_connection_24h": 3, + "failures_connection_total": 150, + "failures_auth_24h": 4, + "failures_auth_total": 61, + "failures_timeout_24h": 1, + "failures_timeout_total": 40, + "failures_command_24h": 0, + "failures_command_total": 12, + "failures_verification_24h": 0, + "failures_verification_total": 0, + "failures_storage_24h": 0, + "failures_storage_total": 3, + "failures_integrity_24h": 0, + "failures_integrity_total": 1, + "failures_push_24h": 0, + "failures_push_total": 0, + "failures_orphaned_24h": 2, + "failures_orphaned_total": 37, + "failures_unknown_24h": 0, + "failures_unknown_total": 143, + "queued": 1, + "running": 2, + "duration_collect_avg": 6.2, + "duration_collect_p50": 4.0, + "duration_collect_p95": 21.5, + "duration_collect_max": 48.0, + "duration_store_avg": 0.8, + "duration_store_p50": 0.5, + "duration_store_p95": 2.1, + "duration_store_max": 6.4, + "duration_wait_avg": 35.0, + "duration_wait_p50": 2.0, + "duration_wait_p95": 870.0, + "duration_wait_max": 1810.0, + "duration_total_avg": 42.0, + "duration_total_p50": 7.0, + "duration_total_p95": 900.0, + "duration_total_max": 1830.0, + "devices_total": 52, + "devices_enabled": 50, + "devices_disabled": 2, + "devices_inactive": 3, + "devices_never": 2, + "devices_overdue": 4, + "devices_failing_1": 6, + "devices_failing_3": 2, + "devices_failing_5": 1, + "oldest_success_age": 259200, + "oldest_success_device": "Edge 01", + "backups_held": 1480, + "bytes_held": 26954000, + "push_accepted_24h": 5, + "push_refused_24h": 3, + "push_refused_unknown_source_24h": 2, + "push_refused_auth_failed_24h": 1, + "push_refused_wrong_protocol_24h": 0, + "push_refused_filename_24h": 0, + "push_refused_too_large_24h": 0, + "push_refused_device_changed_24h": 0, + "push_refused_error_24h": 0, + "push_last_at": 1792996400, + "push_silent_7d": 1, + "version": "0.1.0", + "payload_encryption": 1, + "server_time": 1793000000, + "computed_at": 1792999990, + "computed_age": 10, + "workers_online": 2, + "workers_offline": 1, + "workers_pending": 1, + "workers_revoked": 0, + "workers_behind": 1, + "scheduler_leader": 1, + "scheduler_lease_ttl": 25, + "scheduler_tick_age": 300 +} diff --git a/deploy/zabbix-test/responses/workers.json b/deploy/zabbix-test/responses/workers.json new file mode 100644 index 0000000..9cf4124 --- /dev/null +++ b/deploy/zabbix-test/responses/workers.json @@ -0,0 +1,37 @@ +{ + "data": [ + { + "id": "1", + "name": "collector-1", + "role": "collector", + "status_code": 1, + "online": 1, + "heartbeat_age": 12, + "pending_age": 0, + "version": "0.1.0", + "assignment_behind": 0 + }, + { + "id": "4", + "name": "storage-1", + "role": "storage", + "status_code": 1, + "online": 0, + "heartbeat_age": 900, + "pending_age": 0, + "version": "0.1.0", + "assignment_behind": 1 + }, + { + "id": "7", + "name": "collector-new", + "role": "collector", + "status_code": 0, + "online": 0, + "heartbeat_age": -1, + "pending_age": 7200, + "version": "0.1.0", + "assignment_behind": 0 + } + ] +} \ No newline at end of file diff --git a/deploy/zabbix-test/stub.py b/deploy/zabbix-test/stub.py new file mode 100644 index 0000000..228e064 --- /dev/null +++ b/deploy/zabbix-test/stub.py @@ -0,0 +1,72 @@ +"""A stand-in for DeviceVault's monitoring endpoints and a receiver relay, for ``make zabbix-validate``. + +Serves the recorded answers in ``responses/`` (standard library only): + +- ``/api/monitoring/{summary,device-groups,workers,storage-targets,receivers}/``; +- ``/device/?ip=&id=`` (a relay) and ``/api/monitoring/device/``: ``device.json`` for + 192.0.2.10 or id 12, 503 ``stale`` for 192.0.2.53, and ``found: 0`` for anything else; +- 401 for a missing or wrong bearer token, and for every path under ``/refused/``. + +The clock-dependent fields are set at answer time: the summary's ``server_time`` runs ten +minutes slow (so the clock-skew trigger fires) and ``computed_at`` is ten seconds old. +""" + +import json +import sys +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +TOKEN = 'dvt_zabbixte_validation-only-token' +FOUND_IP, FOUND_ID, STALE_IP = '192.0.2.10', '12', '192.0.2.53' +LISTS = {'device-groups', 'workers', 'storage-targets', 'receivers'} + +responses = Path(sys.argv[1] if len(sys.argv) > 1 else Path(__file__).parent / 'responses') + + +def recorded(name: str) -> dict: + return json.loads((responses / f'{name}.json').read_text()) + + +class Handler(BaseHTTPRequestHandler): + def answer(self, status: int, body: dict) -> None: + data = json.dumps(body).encode() + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): # noqa: N802 (http.server's name) + url = urlsplit(self.path) + parts = [part for part in url.path.split('/') if part] + if self.headers.get('Authorization') != f'Bearer {TOKEN}' or 'refused' in parts: + return self.answer(401, {'detail': 'Invalid token.'}) + if not parts: + return self.answer(404, {'detail': 'Not found.'}) + name = parts[-1] + now = int(time.time()) + if name == 'summary': + body = recorded('summary') + body.update(server_time=now - 600, computed_at=now - 10, computed_age=10) + return self.answer(200, body) + if name in LISTS: + return self.answer(200, recorded(name)) + if name == 'device': + query = {key: values[0] for key, values in parse_qs(url.query).items()} + ip, device_id = query.get('ip', ''), query.get('id', '') + if ip == STALE_IP and not device_id: + return self.answer(503, {'error': 'stale'}) + if device_id == FOUND_ID or (not device_id and ip == FOUND_IP): + return self.answer(200, recorded('device')) + device = recorded('device') + return self.answer(200, {'found': 0, 'published_at': device['published_at'], 'published_age': 120}) + return self.answer(404, {'detail': 'Not found.'}) + + def log_message(self, format, *args): # noqa: A002 + sys.stderr.write(f'stub: {self.command} {self.path} {args[1] if len(args) > 1 else ""}\n') + + +if __name__ == '__main__': + ThreadingHTTPServer(('0.0.0.0', 8000), Handler).serve_forever() diff --git a/deploy/zabbix-test/validate.py b/deploy/zabbix-test/validate.py new file mode 100644 index 0000000..b77025b --- /dev/null +++ b/deploy/zabbix-test/validate.py @@ -0,0 +1,330 @@ +"""Check the generated Zabbix templates against a real Zabbix server (``make zabbix-validate``). + +Runs in the dev image on the ``deploy/zabbix-test`` compose network, standard library only: + +1. imports ``/templates/devicevault_zabbix_.yaml`` through the API (``configuration.import``); +2. creates hosts pointing at the stub (``stub.py``): the DeviceVault host, one whose token is + refused, and three device hosts (found, unknown, relay without a snapshot); +3. waits until every item is supported and holds the recorded value (nulls and absent fields + discarded, not stored), and until exactly the expected triggers are in problem state; +4. prints a summary and exits non-zero on any difference. ``make`` tears the stack down. +""" + +import argparse +import json +import math +import os +import socket +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ZABBIX_URL = os.environ.get('ZABBIX_URL', 'http://web:8080/api_jsonrpc.php') +STUB_URL = os.environ.get('STUB_URL', 'http://stub:8000') +TOKEN = 'dvt_zabbixte_validation-only-token' +HOST_TEMPLATE, DEVICE_TEMPLATE = 'DeviceVault by HTTP', 'DeviceVault Device by HTTP' +RESPONSES = Path(__file__).resolve().parent / 'responses' +LISTS = {'group': 'device-groups', 'worker': 'workers', 'target': 'storage-targets', 'receiver': 'receivers'} +DYNAMIC = {'dv.server_time', 'dv.computed_at', 'dv.computed_age'} # set by the stub at answer time +TIMEOUT = 420 +SETTLE = 75 # after everything matches, hold on for longer than the no-data periods and check again + +HOST_MACROS = {'{$DV.URL}': STUB_URL, '{$DV.TOKEN}': TOKEN, '{$DV.INTERVAL}': '10s', '{$DV.NODATA}': '1m'} +DEVICE_MACROS = { + '{$DV.RELAY.URL}': f'{STUB_URL}/relay', + '{$DV.TOKEN}': TOKEN, + '{$DV.DEVICE.INTERVAL}': '10s', + '{$DV.DEVICE.NODATA}': '1m', + '{$DV.DEVICE.STALE.MAX}': '1d', +} +HOSTS = { + # host: (template, interface address, macros) + 'DeviceVault': (HOST_TEMPLATE, None, HOST_MACROS), + 'DeviceVault refused': (HOST_TEMPLATE, None, {**HOST_MACROS, '{$DV.URL}': f'{STUB_URL}/refused'}), + 'Router found': (DEVICE_TEMPLATE, '192.0.2.10', DEVICE_MACROS), + 'Router unknown': (DEVICE_TEMPLATE, '192.0.2.99', DEVICE_MACROS), + 'Router stale': (DEVICE_TEMPLATE, '192.0.2.53', DEVICE_MACROS), +} +EXPECTED_TRIGGERS = { + 'DeviceVault': { + 'DeviceVault: success ratio below 95% (24h)', + 'DeviceVault: devices never backed up', + 'DeviceVault: backups are slow (p95 over 10m)', + 'DeviceVault: clock differs from Zabbix by more than 60s', + 'DeviceVault: scheduler is not ticking', + 'DeviceVault: group Core: success ratio below 95% (24h)', + 'DeviceVault: group Core: a device has not backed up for over 2d', + 'DeviceVault: group Core: devices overdue', + 'DeviceVault: group Core: devices failing 3+ times in a row', + 'DeviceVault: worker storage-1: no heartbeat for 3m', + 'DeviceVault: worker collector-new: pending approval for over 1h', + 'DeviceVault: storage target local: no storage worker online', + }, + # Everything else depends on the endpoint trigger, so one problem, not dozens. + 'DeviceVault refused': {'DeviceVault: monitoring API unreachable or refusing'}, + 'Router found': { + 'DeviceVault: backup overdue', + 'DeviceVault: backup failing', + 'DeviceVault: backup authentication failing', + 'DeviceVault: no successful backup for 1d', + 'DeviceVault: DeviceVault data is old', + }, + 'Router unknown': {'DeviceVault: not backed up by DeviceVault'}, + 'Router stale': {'DeviceVault: no data from DeviceVault'}, +} +# Hosts whose HTTP items are meant to fail (the token is refused). +FAILING_HOSTS = {'DeviceVault refused'} +# Seconds one API call may take (within the web image's PHP limit, ZBX_MAXEXECUTIONTIME). Linking the templates +# in host.create builds hundreds of items, which is slow on a small CI runner. +API_TIMEOUT = 180 +HOST_CREATE_ATTEMPTS = 3 +# The server's trapper port: it listens only once its start-up database work is done. +ZABBIX_SERVER = os.environ.get('ZABBIX_SERVER', 'server:10051') + + +class Api: + def __init__(self, url: str): + self.url, self.auth = url, None + + def call(self, method: str, params): + body = {'jsonrpc': '2.0', 'method': method, 'params': params, 'id': 1} + if self.auth and method not in ('apiinfo.version', 'user.login'): + body['auth'] = self.auth + request = urllib.request.Request(self.url, json.dumps(body).encode(), {'Content-Type': 'application/json-rpc'}) + with urllib.request.urlopen(request, timeout=API_TIMEOUT) as response: + answer = json.load(response) + if 'error' in answer: + raise RuntimeError(f'{method}: {answer["error"].get("data") or answer["error"]}') + return answer['result'] + + +def connect() -> tuple[Api, str]: + api = Api(ZABBIX_URL) + deadline = time.monotonic() + 300 + while True: + try: + version = api.call('apiinfo.version', []) + api.auth = api.call('user.login', {'username': 'Admin', 'password': 'zabbix'}) + return api, version + except (OSError, RuntimeError, ValueError) as exc: + if time.monotonic() > deadline: + raise SystemExit(f'Zabbix API not ready: {exc}') from exc + time.sleep(3) + + +def wait_for_server() -> None: + """Wait until the Zabbix server listens. Until then it may hold database locks that make API writes hang.""" + host, _, port = ZABBIX_SERVER.rpartition(':') + deadline = time.monotonic() + 300 + while True: + try: + with socket.create_connection((host, int(port)), timeout=5): + return + except OSError as exc: + if time.monotonic() > deadline: + raise SystemExit(f'Zabbix server not listening on {ZABBIX_SERVER}: {exc}') from exc + time.sleep(3) + + +def import_templates(api: Api, version: str) -> None: + source = Path(f'/templates/devicevault_zabbix_{version}.yaml').read_text() + both = {'createMissing': True, 'updateExisting': True} + rules = { + 'templates': both, + 'items': both, + 'discoveryRules': both, + 'triggers': both, + 'graphs': both, + 'valueMaps': both, + 'templateDashboards': both, + 'templateLinkage': {'createMissing': True}, + ('groups' if version == '6.0' else 'template_groups'): {'createMissing': True}, + } + api.call('configuration.import', {'format': 'yaml', 'rules': rules, 'source': source}) + + +def create_hosts(api: Api) -> dict[str, str]: + group = api.call('hostgroup.create', {'name': 'DeviceVault validation'})['groupids'][0] + templates = { + row['host']: row['templateid'] + for row in api.call('template.get', {'output': ['host'], 'filter': {'host': [HOST_TEMPLATE, DEVICE_TEMPLATE]}}) + } + if set(templates) != {HOST_TEMPLATE, DEVICE_TEMPLATE}: + raise SystemExit(f'imported templates missing: found {sorted(templates)}') + ids = {} + for host, (template, address, macros) in HOSTS.items(): + params = { + 'host': host, + 'groups': [{'groupid': group}], + 'templates': [{'templateid': templates[template]}], + 'macros': [{'macro': macro, 'value': value} for macro, value in macros.items()], + } + if address: + params['interfaces'] = [ + {'type': 1, 'main': 1, 'useip': 1, 'ip': address, 'dns': '', 'port': '10050'} + ] # fmt: skip + ids[host] = _create_host(api, params) + return ids + + +def _create_host(api: Api, params: dict) -> str: + """host.create, surviving a lost answer: a timed-out call may still have created the host.""" + for attempt in range(1, HOST_CREATE_ATTEMPTS + 1): + started = time.monotonic() + try: + hostid = api.call('host.create', params)['hostids'][0] + print(f'host.create {params["host"]!r}: {time.monotonic() - started:.1f}s', flush=True) + return hostid + except TimeoutError: + print(f'host.create {params["host"]!r} timed out (attempt {attempt})', flush=True) + found = api.call('host.get', {'output': ['hostid'], 'filter': {'host': [params['host']]}}) + if found: + return found[0]['hostid'] + raise SystemExit(f'could not create host {params["host"]!r}') + + +def _recorded(name: str) -> dict: + return json.loads((RESPONSES / f'{name}.json').read_text()) + + +def expected_values(host: str) -> dict[str, object]: + """Item key -> the value it should hold (None: no value stored), for the items checked.""" + template = HOSTS[host][0] + if host in FAILING_HOSTS: + return {'dv.http.status': 401} + if template == HOST_TEMPLATE: + out = {f'dv.{key}': value for key, value in _recorded('summary').items()} + for slug, name in LISTS.items(): + for entry in _recorded(name)['data']: + for key, value in entry.items(): + if key not in ('id', 'name'): + out[f'dv.{slug}.{key}[{entry["id"]}]'] = value + out['dv.http.status'] = 200 + return out + device = _recorded('device') + keys = [*device, 'error'] + if host == 'Router found': + return {f'dv.device.{key}': device.get(key, 'none') for key in keys} + if host == 'Router unknown': + answer = {'found': 0, 'published_at': device['published_at'], 'published_age': 120, 'error': 'none'} + else: + answer = {'error': 'stale'} + return {f'dv.device.{key}': answer.get(key) for key in keys} + + +def _matches(expected, actual: str) -> bool: + if isinstance(expected, bool | int | float): + try: + return math.isclose(float(actual), float(expected), rel_tol=1e-6, abs_tol=0.01) + except ValueError: + return False + return actual == str(expected) + + +def check(api: Api, hosts: dict[str, str]) -> tuple[list[str], dict]: + """Every difference from the expected state, and counts for the summary.""" + problems: list[str] = [] + counts = {'items': 0, 'unsupported': 0, 'expected_unsupported': 0, 'values': 0, 'fired': 0, 'rules': 0} + for host, hostid in hosts.items(): + items = api.call( + 'item.get', + {'hostids': hostid, 'output': ['key_', 'state', 'error', 'lastclock', 'lastvalue', 'type', 'history']}, + ) + rules = api.call('discoveryrule.get', {'hostids': hostid, 'output': ['key_', 'state', 'error']}) + by_key = {item['key_']: item for item in items} + counts['items'] += len(items) + counts['rules'] += len(rules) + for item in [*items, *rules]: + if item['state'] != '1': + continue + if host in FAILING_HOSTS: + counts['expected_unsupported'] += 1 + else: + counts['unsupported'] += 1 + problems.append(f'{host}: {item["key_"]} unsupported: {item["error"]}') + for key, value in expected_values(host).items(): + item = by_key.get(key) + if item is None: + problems.append(f'{host}: no item {key} (discovery not done?)') + continue + has_value = item['lastclock'] != '0' + if value is None: + if has_value: + problems.append(f'{host}: {key} should have no value (null or absent), has {item["lastvalue"]!r}') + elif not has_value: + problems.append(f'{host}: {key} has no value yet (expected {value!r})') + elif key not in DYNAMIC and not _matches(value, item['lastvalue']): + problems.append(f'{host}: {key} = {item["lastvalue"]!r}, expected {value!r}') + counts['values'] += 1 + triggers = api.call( + 'trigger.get', + {'hostids': hostid, 'output': ['description', 'value', 'state', 'error'], 'expandDescription': True}, + ) + for trigger in triggers: + # Unknown is expected where the items fail on purpose, and for triggers comparing with values + # older than the test has run (an hour ago, ten minutes ago). + if trigger['state'] == '1' and host not in FAILING_HOSTS and 'not enough data' not in trigger['error']: + problems.append(f'{host}: trigger {trigger["description"]!r} unknown: {trigger["error"]}') + fired = {trigger['description'] for trigger in triggers if trigger['value'] == '1'} + counts['fired'] += len(fired) + for name in sorted(EXPECTED_TRIGGERS[host] - fired): + problems.append(f'{host}: trigger {name!r} did not fire') + for name in sorted(fired - EXPECTED_TRIGGERS[host]): + problems.append(f'{host}: trigger {name!r} fired unexpectedly') + return problems, counts + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('--version', required=True, choices=['6.0', '7.0']) + # `make zabbix-validate` runs `import`, analyzes the freshly loaded database (without planner statistics a + # slow runner's server crawls while holding locks that host.create waits on), then runs `check`. + parser.add_argument('--phase', choices=['all', 'import', 'check'], default='all') + args = parser.parse_args() + + api, server = connect() + if not server.startswith(f'{args.version}.'): + raise SystemExit(f'expected Zabbix {args.version}, the API is {server}') + wait_for_server() + if args.phase in ('all', 'import'): + import_templates(api, args.version) + print(f'Zabbix {server}: imported {HOST_TEMPLATE!r} and {DEVICE_TEMPLATE!r}', flush=True) + if args.phase == 'import': + return 0 + hosts = create_hosts(api) + print(f'Created {len(hosts)} hosts; waiting for items and triggers (up to {TIMEOUT}s)', flush=True) + + started = time.monotonic() + settled_at = None + while True: + problems, counts = check(api, hosts) + elapsed = int(time.monotonic() - started) + if not problems: + settled_at = settled_at or time.monotonic() + if time.monotonic() - settled_at >= SETTLE: + break + else: + settled_at = None + if elapsed > TIMEOUT: + break + time.sleep(10) + + expected = sum(len(names) for names in EXPECTED_TRIGGERS.values()) + print( + f'Zabbix {server}: {counts["items"]} items and {counts["rules"]} discovery rules on {len(hosts)} hosts, ' + f'{counts["unsupported"]} unsupported ({counts["expected_unsupported"]} on the refused-token host, ' + f'as intended); {counts["values"]} values checked; {counts["fired"]} triggers fired, {expected} expected' + ) + if problems: + print(f'FAILED after {elapsed}s:') + for problem in problems: + print(f' - {problem}') + return 1 + print(f'OK: every item supported with its recorded value, exactly the expected triggers fired ({elapsed}s)') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/deploy/zabbix/devicevault_zabbix_6.0.yaml b/deploy/zabbix/devicevault_zabbix_6.0.yaml new file mode 100644 index 0000000..acabe63 --- /dev/null +++ b/deploy/zabbix/devicevault_zabbix_6.0.yaml @@ -0,0 +1,6098 @@ +zabbix_export: + version: '6.0' + groups: + - uuid: e6e21a0925d9472b929b6e4039bdc474 + name: Templates/Applications + templates: + - uuid: aa670518836c4d5cb9a3d9772f613cb8 + template: DeviceVault by HTTP + name: DeviceVault by HTTP + description: |- + Backup health of a DeviceVault installation, polled from its read-only monitoring API. + + Link it to one host standing for DeviceVault, monitored by the Zabbix server or a proxy that can reach the web tier. + Set {$DV.URL} (the web address, without /api) and {$DV.TOKEN} (a service account's token with the monitoring scope, + plus monitoring_platform for workers, the scheduler, storage targets and receivers) as global macros, or on the host. + + Generated by DeviceVault from its metric catalogue; see DEPLOYMENT.md, section Monitoring. + groups: + - name: Templates/Applications + items: + - uuid: 7d7afd6b15984fb281a39f1d6c2b15c8 + name: 'DeviceVault: Get summary' + type: HTTP_AGENT + key: dv.summary.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: Estate outcomes, timing, freshness, change, volume, push and self, and platform health with that scope. + timeout: 15s + url: '{$DV.URL}/api/monitoring/summary/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: 68a85968b2444b9fb74b925696c567bc + name: 'DeviceVault: API status' + type: HTTP_AGENT + key: dv.http.status + delay: '{$DV.INTERVAL}' + description: The HTTP status the summary endpoint answers with (200 when the token is accepted). + timeout: 15s + url: '{$DV.URL}/api/monitoring/summary/' + status_codes: '' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + retrieve_mode: HEADERS + preprocessing: + - type: REGEX + parameters: + - ^HTTP\/[\d.]+\s+(\d+) + - \1 + tags: + - tag: component + value: api + triggers: + - uuid: ce94da43a52f45239320610816fcc11b + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + name: 'DeviceVault: monitoring API unreachable or refusing' + opdata: 'HTTP status: {ITEM.LASTVALUE1}' + priority: HIGH + description: 'The summary endpoint did not answer for {$DV.NODATA}, or answered with an error (401 or 403: the token is unknown, revoked, expired, out of its networks or lacks the monitoring scope; 5xx: DeviceVault is failing).' + tags: + - tag: component + value: api + - tag: scope + value: availability + - tag: group + value: all + - uuid: 45f498cf445a4118b28b11978cdfde4b + name: 'DeviceVault: Get device groups' + type: HTTP_AGENT + key: dv.group.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: The device groups list (device-groups/), read by discovery. + timeout: 15s + url: '{$DV.URL}/api/monitoring/device-groups/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: 491432725d07479fa406471c23c82fcb + name: 'DeviceVault: Get workers' + type: HTTP_AGENT + key: dv.worker.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: The workers list (workers/), read by discovery. + timeout: 15s + url: '{$DV.URL}/api/monitoring/workers/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: df9789dd3b3d4467beccf13fa52c06be + name: 'DeviceVault: Get storage targets' + type: HTTP_AGENT + key: dv.target.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: The storage targets list (storage-targets/), read by discovery. + timeout: 15s + url: '{$DV.URL}/api/monitoring/storage-targets/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: df1a9eca7ff34596a7c228b2fdae35d2 + name: 'DeviceVault: Get receivers' + type: HTTP_AGENT + key: dv.receiver.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: The receivers list (receivers/), read by discovery. + timeout: 15s + url: '{$DV.URL}/api/monitoring/receivers/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: bf5d6a49b06a49d68688b2c4335bf0da + name: 'DeviceVault: Backups stored (1h)' + type: DEPENDENT + key: dv.stored_1h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.stored_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: d05ff201b40a4c3e9369ed9013a940a4 + name: 'DeviceVault: Backups failed (1h)' + type: DEPENDENT + key: dv.failed_1h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished failed in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.failed_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: b7441aa109d645caa090f8ac745bf4d4 + name: 'DeviceVault: Backups timed out (1h)' + type: DEPENDENT + key: dv.timed_out_1h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished timed out in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.timed_out_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: b36f5d7941f748069ee4837a0ed95deb + name: 'DeviceVault: Backups finished (1h)' + type: DEPENDENT + key: dv.finished_1h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished in the last 1h, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.finished_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 6de8ac90da2c4ba89bd1160339934112 + name: 'DeviceVault: Success ratio (1h)' + type: DEPENDENT + key: dv.success_ratio_1h + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 1h; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.success_ratio_1h + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 2796bf19e91b419b8a64894e0cb18948 + name: 'DeviceVault: Failure ratio (1h)' + type: DEPENDENT + key: dv.failure_ratio_1h + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.failure_ratio_1h + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 0459709f841d43d2a62fb7135d531760 + name: 'DeviceVault: Backups retried (1h)' + type: DEPENDENT + key: dv.retried_1h + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 1h that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.retried_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: f5e1de1d16d04daea5c8f5ead3d85443 + name: 'DeviceVault: Backups with changes (1h)' + type: DEPENDENT + key: dv.changed_1h + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 1h that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.changed_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: a80e4fa544da4c5fab14d2999d57753e + name: 'DeviceVault: Bytes stored (1h)' + type: DEPENDENT + key: dv.bytes_stored_1h + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.bytes_stored_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: b7b3bf1bd7fc40a8939c466672665919 + name: 'DeviceVault: Lines added (1h)' + type: DEPENDENT + key: dv.lines_added_1h + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_added_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 0a48e19e290245f0a963230cf9fcac34 + name: 'DeviceVault: Lines removed (1h)' + type: DEPENDENT + key: dv.lines_removed_1h + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_removed_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: e742ff9f30cf47bd88dd36a39a3d8abb + name: 'DeviceVault: Lines modified (1h)' + type: DEPENDENT + key: dv.lines_modified_1h + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_modified_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: c98c5a1973684dc3abf7c1b758d51557 + name: 'DeviceVault: Backups stored (24h)' + type: DEPENDENT + key: dv.stored_24h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.stored_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: a39dae0c0f354eac8ffbd51b29ff2569 + name: 'DeviceVault: Backups failed (24h)' + type: DEPENDENT + key: dv.failed_24h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished failed in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.failed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: d01a82fdae77489283622305e4988756 + name: 'DeviceVault: Backups timed out (24h)' + type: DEPENDENT + key: dv.timed_out_24h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished timed out in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.timed_out_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 0d0a895c8c314d1b98bf289ad25fec3b + name: 'DeviceVault: Backups finished (24h)' + type: DEPENDENT + key: dv.finished_24h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished in the last 24h, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.finished_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: ee3eef0c393b4480904f0bdc8e956fe9 + name: 'DeviceVault: Success ratio (24h)' + type: DEPENDENT + key: dv.success_ratio_24h + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 24h; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.success_ratio_24h + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: bff829b7da9842c09db731090d5c44f0 + name: 'DeviceVault: Failure ratio (24h)' + type: DEPENDENT + key: dv.failure_ratio_24h + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.failure_ratio_24h + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 2dd63b91c7e84378b064ddbfc65d8176 + name: 'DeviceVault: Backups retried (24h)' + type: DEPENDENT + key: dv.retried_24h + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 24h that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.retried_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: f034e175cc0a4704b0a7aaafeede4bf4 + name: 'DeviceVault: Backups with changes (24h)' + type: DEPENDENT + key: dv.changed_24h + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 24h that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.changed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 1f83a71fd87a486fb2b0573426596d3a + name: 'DeviceVault: Bytes stored (24h)' + type: DEPENDENT + key: dv.bytes_stored_24h + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.bytes_stored_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: e7e8e91b24cc40379746e71c3ab2edf6 + name: 'DeviceVault: Lines added (24h)' + type: DEPENDENT + key: dv.lines_added_24h + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_added_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: efff3fc1bfae4d91b69dda3341aa019c + name: 'DeviceVault: Lines removed (24h)' + type: DEPENDENT + key: dv.lines_removed_24h + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_removed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 3ad32f07f80d44e0944b69b79357b301 + name: 'DeviceVault: Lines modified (24h)' + type: DEPENDENT + key: dv.lines_modified_24h + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_modified_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: ab59662da07c40faaf06a5b38c381779 + name: 'DeviceVault: Backups stored (7d)' + type: DEPENDENT + key: dv.stored_7d + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.stored_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 11ed5cab803945a48a6093526338b782 + name: 'DeviceVault: Backups failed (7d)' + type: DEPENDENT + key: dv.failed_7d + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished failed in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.failed_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: ac1178a794824faea469ecf148c0854c + name: 'DeviceVault: Backups timed out (7d)' + type: DEPENDENT + key: dv.timed_out_7d + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished timed out in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.timed_out_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 87cda6948b524c00bad3530d5708d3e1 + name: 'DeviceVault: Backups finished (7d)' + type: DEPENDENT + key: dv.finished_7d + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished in the last 7d, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.finished_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: a1c3004eec7d4e628c64759f6fea2921 + name: 'DeviceVault: Success ratio (7d)' + type: DEPENDENT + key: dv.success_ratio_7d + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 7d; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.success_ratio_7d + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 773fb814bc2c41ea98e00fb541b5012f + name: 'DeviceVault: Failure ratio (7d)' + type: DEPENDENT + key: dv.failure_ratio_7d + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.failure_ratio_7d + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: d0f21018cac04045a17e9b5bc82add10 + name: 'DeviceVault: Backups retried (7d)' + type: DEPENDENT + key: dv.retried_7d + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 7d that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.retried_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 47c4e068063d4008864c09a5cce09963 + name: 'DeviceVault: Backups with changes (7d)' + type: DEPENDENT + key: dv.changed_7d + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 7d that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.changed_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 5c305fa9175b4352bd20982566c5d58b + name: 'DeviceVault: Bytes stored (7d)' + type: DEPENDENT + key: dv.bytes_stored_7d + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.bytes_stored_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: c5e046f6dad145418c414e4e18b0a02d + name: 'DeviceVault: Lines added (7d)' + type: DEPENDENT + key: dv.lines_added_7d + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.lines_added_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 4dd08de61f7f4c09b05db1881c979837 + name: 'DeviceVault: Lines removed (7d)' + type: DEPENDENT + key: dv.lines_removed_7d + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.lines_removed_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 61db668aaae4490f9083571ad2832445 + name: 'DeviceVault: Lines modified (7d)' + type: DEPENDENT + key: dv.lines_modified_7d + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.lines_modified_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: f4831046c8574711804c150144dbe906 + name: 'DeviceVault: Backups stored (all time)' + type: DEPENDENT + key: dv.stored_total + delay: '0' + value_type: UNSIGNED + description: Every job of the estate that finished stored, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.stored_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 04bc79f736f54cd8b433c4a611cd4814 + name: 'DeviceVault: Backups failed (all time)' + type: DEPENDENT + key: dv.failed_total + delay: '0' + value_type: UNSIGNED + description: Every job of the estate that finished failed, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.failed_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: d7c6ebcf694d4ea4918f20911e4f5c38 + name: 'DeviceVault: Backups timed out (all time)' + type: DEPENDENT + key: dv.timed_out_total + delay: '0' + value_type: UNSIGNED + description: Every job of the estate that finished timed out, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.timed_out_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: ca8148c3b9cf4c3382d3d4530329f2e0 + name: 'DeviceVault: Failures: connection (24h)' + type: DEPENDENT + key: dv.failures_connection_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "connection" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_connection_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 9a45beed5b544ae298aed1f8d0b8527a + name: 'DeviceVault: Failures: connection (all time)' + type: DEPENDENT + key: dv.failures_connection_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "connection" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_connection_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 2c1c9dcc6276477d89daa1517a4eb889 + name: 'DeviceVault: Failures: auth (24h)' + type: DEPENDENT + key: dv.failures_auth_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "auth" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_auth_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: eb2bbc3549e048c8b554b071fae875d3 + name: 'DeviceVault: Failures: auth (all time)' + type: DEPENDENT + key: dv.failures_auth_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "auth" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_auth_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 4bf5d3a853654634ad0112893a578d00 + name: 'DeviceVault: Failures: timeout (24h)' + type: DEPENDENT + key: dv.failures_timeout_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "timeout" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_timeout_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 6743e88c2a024be181e7695556430897 + name: 'DeviceVault: Failures: timeout (all time)' + type: DEPENDENT + key: dv.failures_timeout_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "timeout" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_timeout_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: ac50b44e87bb4b389a2dc18e1a405023 + name: 'DeviceVault: Failures: command (24h)' + type: DEPENDENT + key: dv.failures_command_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "command" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_command_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 413607c919e340b0a47bb58431439fb6 + name: 'DeviceVault: Failures: command (all time)' + type: DEPENDENT + key: dv.failures_command_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "command" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_command_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: d793f46a07584550a80ae643fb894767 + name: 'DeviceVault: Failures: verification (24h)' + type: DEPENDENT + key: dv.failures_verification_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "verification" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_verification_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 4d58d0c2e3b24c79b554db33dcb929fa + name: 'DeviceVault: Failures: verification (all time)' + type: DEPENDENT + key: dv.failures_verification_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "verification" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_verification_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 3838d117b128435aa43a50acc8f768f6 + name: 'DeviceVault: Failures: storage (24h)' + type: DEPENDENT + key: dv.failures_storage_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "storage" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_storage_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 03b0892403bf45b5838bc8728049e64b + name: 'DeviceVault: Failures: storage (all time)' + type: DEPENDENT + key: dv.failures_storage_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "storage" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_storage_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 47dfc2bad7de492b9c4c00776be2a2b5 + name: 'DeviceVault: Failures: integrity (24h)' + type: DEPENDENT + key: dv.failures_integrity_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "integrity" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_integrity_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 3daacf291cd94d4a895542fd5b2b0f72 + name: 'DeviceVault: Failures: integrity (all time)' + type: DEPENDENT + key: dv.failures_integrity_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "integrity" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_integrity_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 11662cba7b444b9181602ed163d70800 + name: 'DeviceVault: Failures: push (24h)' + type: DEPENDENT + key: dv.failures_push_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "push" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_push_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 32cb4286c36b4a7a9c5fa2a1febca3fa + name: 'DeviceVault: Failures: push (all time)' + type: DEPENDENT + key: dv.failures_push_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "push" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_push_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: a8d3c9b187874159ac4e721bb7dadf47 + name: 'DeviceVault: Failures: orphaned (24h)' + type: DEPENDENT + key: dv.failures_orphaned_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "orphaned" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_orphaned_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: c0fc4ba8f0db4a3fb78cb48afe9d3703 + name: 'DeviceVault: Failures: orphaned (all time)' + type: DEPENDENT + key: dv.failures_orphaned_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "orphaned" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_orphaned_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 2f3c82e212bb44cabb1587500fbaa05a + name: 'DeviceVault: Failures: unknown (24h)' + type: DEPENDENT + key: dv.failures_unknown_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "unknown" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_unknown_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 0f7895ce88d04f0094f67cb4afe72eff + name: 'DeviceVault: Failures: unknown (all time)' + type: DEPENDENT + key: dv.failures_unknown_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "unknown" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_unknown_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 0cfa53fd39154a22954d4e77448093b8 + name: 'DeviceVault: Backups queued' + type: DEPENDENT + key: dv.queued + delay: '0' + value_type: UNSIGNED + description: Jobs waiting for a collector now + preprocessing: + - type: JSONPATH + parameters: + - $.queued + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 8cc9286e82d144b8a9b89678349c1b9f + name: 'DeviceVault: Backups running' + type: DEPENDENT + key: dv.running + delay: '0' + value_type: UNSIGNED + description: Jobs being collected or stored now + preprocessing: + - type: JSONPATH + parameters: + - $.running + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: b0cd9f9abde44f7a8a28517ecc437cde + name: 'DeviceVault: Duration of collection, average (24h)' + type: DEPENDENT + key: dv.duration_collect_avg + delay: '0' + value_type: FLOAT + units: s + description: The average collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_collect_avg + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 1d35db9e53a048769868aa3e02f89482 + name: 'DeviceVault: Duration of collection, median (24h)' + type: DEPENDENT + key: dv.duration_collect_p50 + delay: '0' + value_type: FLOAT + units: s + description: The median collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_collect_p50 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 6b74c612653048b0b67d7fd643221e8c + name: 'DeviceVault: Duration of collection, 95th percentile (24h)' + type: DEPENDENT + key: dv.duration_collect_p95 + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_collect_p95 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: fe431e627a4844358469bc2e47846fef + name: 'DeviceVault: Duration of collection, maximum (24h)' + type: DEPENDENT + key: dv.duration_collect_max + delay: '0' + value_type: FLOAT + units: s + description: The maximum collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_collect_max + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: eb48681932be41348117cbddd2ca0ae5 + name: 'DeviceVault: Duration of storage, average (24h)' + type: DEPENDENT + key: dv.duration_store_avg + delay: '0' + value_type: FLOAT + units: s + description: The average storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_store_avg + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: ad6efe14e72f4b98b7701cc901f89fad + name: 'DeviceVault: Duration of storage, median (24h)' + type: DEPENDENT + key: dv.duration_store_p50 + delay: '0' + value_type: FLOAT + units: s + description: The median storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_store_p50 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: b12a616fdf364399827773ade3855d23 + name: 'DeviceVault: Duration of storage, 95th percentile (24h)' + type: DEPENDENT + key: dv.duration_store_p95 + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_store_p95 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: fb29667a9c6d443c9b3a7ea113f536c1 + name: 'DeviceVault: Duration of storage, maximum (24h)' + type: DEPENDENT + key: dv.duration_store_max + delay: '0' + value_type: FLOAT + units: s + description: The maximum storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_store_max + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 3f2a0d3dc461408dbe216b5b2a9176e4 + name: 'DeviceVault: Duration of queue wait, average (24h)' + type: DEPENDENT + key: dv.duration_wait_avg + delay: '0' + value_type: FLOAT + units: s + description: The average queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_wait_avg + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 548cae0d8e2846fb8b56768f4bc08205 + name: 'DeviceVault: Duration of queue wait, median (24h)' + type: DEPENDENT + key: dv.duration_wait_p50 + delay: '0' + value_type: FLOAT + units: s + description: The median queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_wait_p50 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 84887283402e45cf94f576519bf29f20 + name: 'DeviceVault: Duration of queue wait, 95th percentile (24h)' + type: DEPENDENT + key: dv.duration_wait_p95 + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_wait_p95 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: a55eb0351b364fedb22be7d3875a1f09 + name: 'DeviceVault: Duration of queue wait, maximum (24h)' + type: DEPENDENT + key: dv.duration_wait_max + delay: '0' + value_type: FLOAT + units: s + description: The maximum queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_wait_max + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: a1fafac13440474fb2406c886e516991 + name: 'DeviceVault: Duration of request to stored, average (24h)' + type: DEPENDENT + key: dv.duration_total_avg + delay: '0' + value_type: FLOAT + units: s + description: The average request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_total_avg + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: c031d2a665184f109512c4b1ac6153b3 + name: 'DeviceVault: Duration of request to stored, median (24h)' + type: DEPENDENT + key: dv.duration_total_p50 + delay: '0' + value_type: FLOAT + units: s + description: The median request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_total_p50 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: f579da68fc83427a86dd4999b225c455 + name: 'DeviceVault: Duration of request to stored, 95th percentile (24h)' + type: DEPENDENT + key: dv.duration_total_p95 + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_total_p95 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: cfb1a08a1f9646888bf4696c81fbf16e + name: 'DeviceVault: Duration of request to stored, maximum (24h)' + type: DEPENDENT + key: dv.duration_total_max + delay: '0' + value_type: FLOAT + units: s + description: The maximum request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_total_max + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: da454db68db947988e8579d923f5130b + name: 'DeviceVault: Devices' + type: DEPENDENT + key: dv.devices_total + delay: '0' + value_type: UNSIGNED + description: Active devices + preprocessing: + - type: JSONPATH + parameters: + - $.devices_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: e7975daf14454e889458a4ec3ee2460b + name: 'DeviceVault: Devices enabled' + type: DEPENDENT + key: dv.devices_enabled + delay: '0' + value_type: UNSIGNED + description: Active devices that are not paused + preprocessing: + - type: JSONPATH + parameters: + - $.devices_enabled + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: a8d69f8bf46343c69f833e7baf840ec2 + name: 'DeviceVault: Devices disabled' + type: DEPENDENT + key: dv.devices_disabled + delay: '0' + value_type: UNSIGNED + description: Active devices that are paused + preprocessing: + - type: JSONPATH + parameters: + - $.devices_disabled + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 2d2bbbcbcdb34215a7225f606e35da29 + name: 'DeviceVault: Devices inactive' + type: DEPENDENT + key: dv.devices_inactive + delay: '0' + value_type: UNSIGNED + description: Devices marked inactive (retired, held) + preprocessing: + - type: JSONPATH + parameters: + - $.devices_inactive + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 46fa513b9b014aa3b44a49e8c6ab6c61 + name: 'DeviceVault: Devices never backed up' + type: DEPENDENT + key: dv.devices_never + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices with no stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.devices_never + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + triggers: + - uuid: e6a6c08a27f34731a7e03b63ff386f25 + expression: last(/DeviceVault by HTTP/dv.devices_never)>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.devices_never)=0 + name: 'DeviceVault: devices never backed up' + opdata: 'Devices: {ITEM.LASTVALUE1}' + priority: WARNING + description: Active, enabled devices have no stored backup yet. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: freshness + - tag: scope + value: notice + - tag: group + value: all + - uuid: a7bb31e86ed947f3b15df09eb237427d + name: 'DeviceVault: Devices overdue' + type: DEPENDENT + key: dv.devices_overdue + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last success is older than their schedule allows + preprocessing: + - type: JSONPATH + parameters: + - $.devices_overdue + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 42f1d2f48a6141fdbe987841f59f08a5 + name: 'DeviceVault: Devices failing 1+ times in a row' + type: DEPENDENT + key: dv.devices_failing_1 + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 1 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.devices_failing_1 + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: e17dcdfead6e462cb352e781ba65765b + name: 'DeviceVault: Devices failing 3+ times in a row' + type: DEPENDENT + key: dv.devices_failing_3 + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 3 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.devices_failing_3 + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: bd01fc742cfc4f239b8bc5cfc6e1b04e + name: 'DeviceVault: Devices failing 5+ times in a row' + type: DEPENDENT + key: dv.devices_failing_5 + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 5 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.devices_failing_5 + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 51fa145a62b74a5c89dcb94307a8b98a + name: 'DeviceVault: Oldest last success, age' + type: DEPENDENT + key: dv.oldest_success_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since the stalest active, enabled device last backed up (0 with none backed up) + preprocessing: + - type: JSONPATH + parameters: + - $.oldest_success_age + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: e0994af2c3134b129a2dacde6ffc2b47 + name: 'DeviceVault: Oldest last success, device' + type: DEPENDENT + key: dv.oldest_success_device + delay: '0' + value_type: CHAR + trends: '0' + description: Name of that device + preprocessing: + - type: JSONPATH + parameters: + - $.oldest_success_device + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 0f577e1812b74f94a6d35c40d15a53c8 + name: 'DeviceVault: Backups held' + type: DEPENDENT + key: dv.backups_held + delay: '0' + value_type: UNSIGNED + description: Stored backups not pruned by retention + preprocessing: + - type: JSONPATH + parameters: + - $.backups_held + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: e665b41c4f6f44bd85baec23db678d7f + name: 'DeviceVault: Bytes held' + type: DEPENDENT + key: dv.bytes_held + delay: '0' + value_type: UNSIGNED + units: B + description: Size of those backups (as the collectors reported it) + preprocessing: + - type: JSONPATH + parameters: + - $.bytes_held + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: a5acd80391b7405d8aa08843414e9773 + name: 'DeviceVault: Pushes accepted (24h)' + type: DEPENDENT + key: dv.push_accepted_24h + delay: '0' + value_type: UNSIGNED + description: Files devices pushed in the last 24h that became backups + preprocessing: + - type: JSONPATH + parameters: + - $.push_accepted_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 7fea1bc7dcc846b5a97e654b458aefb0 + name: 'DeviceVault: Pushes refused (24h)' + type: DEPENDENT + key: dv.push_refused_24h + delay: '0' + value_type: UNSIGNED + description: Refused pushes in the last 24h (receivers rate-limit reports, so a lower bound) + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + triggers: + - uuid: ce2584cdbefe455983a4d3f18d8738ff + expression: last(/DeviceVault by HTTP/dv.push_refused_24h)-last(/DeviceVault by HTTP/dv.push_refused_24h,#1:now-1h)>{$DV.PUSH.REFUSED.MAX} + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.push_refused_24h)-last(/DeviceVault by HTTP/dv.push_refused_24h,#1:now-1h)<={$DV.PUSH.REFUSED.MAX} + name: 'DeviceVault: push refusals spiking' + priority: WARNING + description: Receivers refused more than {$DV.PUSH.REFUSED.MAX} pushes in the last hour (unknown sources, failed authentication, wrong protocol...). See the Push refusals page. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: push + - tag: scope + value: security + - tag: group + value: all + - uuid: 03f1a943066b4130ba14334fc5a61c78 + name: 'DeviceVault: Pushes refused: unknown_source (24h)' + type: DEPENDENT + key: dv.push_refused_unknown_source_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "unknown_source" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_unknown_source_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: e153c62f08d949b297bf746562184c33 + name: 'DeviceVault: Pushes refused: auth_failed (24h)' + type: DEPENDENT + key: dv.push_refused_auth_failed_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "auth_failed" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_auth_failed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 767cd5dbc43b490381bf0b4602af9b05 + name: 'DeviceVault: Pushes refused: wrong_protocol (24h)' + type: DEPENDENT + key: dv.push_refused_wrong_protocol_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "wrong_protocol" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_wrong_protocol_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 2254e228de7748a3903285b63b9c2d88 + name: 'DeviceVault: Pushes refused: filename (24h)' + type: DEPENDENT + key: dv.push_refused_filename_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "filename" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_filename_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: cf848a92b86d4b78bd47698a0e3a6e6f + name: 'DeviceVault: Pushes refused: too_large (24h)' + type: DEPENDENT + key: dv.push_refused_too_large_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "too_large" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_too_large_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: b9442c3e887e491ca2115b86c5bf33dd + name: 'DeviceVault: Pushes refused: device_changed (24h)' + type: DEPENDENT + key: dv.push_refused_device_changed_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "device_changed" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_device_changed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: ddb3dce602414bb3befb08c2be70a172 + name: 'DeviceVault: Pushes refused: error (24h)' + type: DEPENDENT + key: dv.push_refused_error_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "error" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_error_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 18b15115c27a42408b5a6735c4aed57b + name: 'DeviceVault: Last push' + type: DEPENDENT + key: dv.push_last_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When a device last pushed a file that was accepted (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.push_last_at + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: a0e54c7c2eec4f8988b9fac3a078ed51 + name: 'DeviceVault: Push devices silent (7d)' + type: DEPENDENT + key: dv.push_silent_7d + delay: '0' + value_type: UNSIGNED + description: Active, enabled push devices with no accepted push in the last 7 days + preprocessing: + - type: JSONPATH + parameters: + - $.push_silent_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 67f49bedceed4741a898ba9ff7de7ac4 + name: 'DeviceVault: DeviceVault version' + type: DEPENDENT + key: dv.version + delay: '0' + value_type: CHAR + trends: '0' + description: The orchestrator version + preprocessing: + - type: JSONPATH + parameters: + - $.version + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + - uuid: cad58f9a198c4a5baf52fc9836edbc0e + name: 'DeviceVault: Payload encryption' + type: DEPENDENT + key: dv.payload_encryption + delay: '0' + value_type: UNSIGNED + description: Whether backups and secrets sent to workers are encrypted (DV_PAYLOAD_ENCRYPTION) + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.payload_encryption + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + - uuid: 735c6ab5d700422db048c7b21d03ffaa + name: 'DeviceVault: Server time' + type: DEPENDENT + key: dv.server_time + delay: '0' + value_type: UNSIGNED + units: unixtime + description: The orchestrator clock, to compare with the monitoring server (clock skew) + preprocessing: + - type: JSONPATH + parameters: + - $.server_time + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + triggers: + - uuid: c69620d17b864ce4a17647de6be22acd + expression: fuzzytime(/DeviceVault by HTTP/dv.server_time,{$DV.SKEW.MAX})=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: fuzzytime(/DeviceVault by HTTP/dv.server_time,{$DV.SKEW.MAX})=1 + name: 'DeviceVault: clock differs from Zabbix by more than {$DV.SKEW.MAX}s' + priority: WARNING + description: The orchestrator's clock and the Zabbix server's differ; schedules and ages will be off. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: system + - tag: scope + value: availability + - tag: group + value: all + - uuid: d96a16c28a2441ec936b0c670effa8af + name: 'DeviceVault: Figures computed at' + type: DEPENDENT + key: dv.computed_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When these figures were computed (they are cached briefly) + preprocessing: + - type: JSONPATH + parameters: + - $.computed_at + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + - uuid: 459e6977d31e4402b01ec5804459c1cf + name: 'DeviceVault: Figures age' + type: DEPENDENT + key: dv.computed_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since these figures were computed + preprocessing: + - type: JSONPATH + parameters: + - $.computed_age + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + - uuid: e0907f452d0a46f7ada01137b04d5b50 + name: 'DeviceVault: Workers online' + type: DEPENDENT + key: dv.workers_online + delay: '0' + value_type: UNSIGNED + description: Approved workers that sent a heartbeat recently + preprocessing: + - type: JSONPATH + parameters: + - $.workers_online + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 70ed2e5d33764a56ac15724f464a2cc9 + name: 'DeviceVault: Workers offline' + type: DEPENDENT + key: dv.workers_offline + delay: '0' + value_type: UNSIGNED + description: Approved workers with no recent heartbeat + preprocessing: + - type: JSONPATH + parameters: + - $.workers_offline + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 1a8a7f9863fb4fc0a76877cf561d46ee + name: 'DeviceVault: Workers pending approval' + type: DEPENDENT + key: dv.workers_pending + delay: '0' + value_type: UNSIGNED + description: Workers waiting for approval + preprocessing: + - type: JSONPATH + parameters: + - $.workers_pending + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 88c0d3bb8ee44636972ed108a892ae49 + name: 'DeviceVault: Workers revoked' + type: DEPENDENT + key: dv.workers_revoked + delay: '0' + value_type: UNSIGNED + description: Revoked workers + preprocessing: + - type: JSONPATH + parameters: + - $.workers_revoked + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 145842d376a74619b442f1776cdf1742 + name: 'DeviceVault: Workers behind on their assignment' + type: DEPENDENT + key: dv.workers_behind + delay: '0' + value_type: UNSIGNED + description: Approved workers whose applied assignment version is older than the current one + preprocessing: + - type: JSONPATH + parameters: + - $.workers_behind + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: f98ee35e560a4f11bc87ce34a4801b6b + name: 'DeviceVault: Scheduler has a leader' + type: DEPENDENT + key: dv.scheduler_leader + delay: '0' + value_type: UNSIGNED + description: Whether a scheduler node holds the lease + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.scheduler_leader + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + triggers: + - uuid: 08ce456e0dbf42dfab0272a8e615dffc + expression: last(/DeviceVault by HTTP/dv.scheduler_leader)=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.scheduler_leader)=1 + name: 'DeviceVault: scheduler has no leader' + priority: HIGH + description: No scheduler node holds the lease, so no scheduled backup starts. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: scheduler + - tag: scope + value: availability + - tag: group + value: all + - uuid: b56f370402974734b2c398f97e7b4372 + name: 'DeviceVault: Scheduler lease remaining' + type: DEPENDENT + key: dv.scheduler_lease_ttl + delay: '0' + value_type: FLOAT + units: s + description: Seconds left on the leader's lease (0 without a leader) + preprocessing: + - type: JSONPATH + parameters: + - $.scheduler_lease_ttl + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 4ab08602bc5e4ab28b73ed9449946919 + name: 'DeviceVault: Scheduler last tick, age' + type: DEPENDENT + key: dv.scheduler_tick_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since the leader last ran its tick (-1 if it never has) + preprocessing: + - type: JSONPATH + parameters: + - $.scheduler_tick_age + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + triggers: + - uuid: 0ba886097e6a4a2baac33c52bbde0969 + expression: last(/DeviceVault by HTTP/dv.scheduler_tick_age)>2m or last(/DeviceVault by HTTP/dv.scheduler_tick_age)<0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.scheduler_tick_age)>=0 and last(/DeviceVault by HTTP/dv.scheduler_tick_age)<=2m + name: 'DeviceVault: scheduler is not ticking' + priority: AVERAGE + description: The scheduler leader has not run its tick for over 2 minutes. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + - name: 'DeviceVault: scheduler has no leader' + expression: last(/DeviceVault by HTTP/dv.scheduler_leader)=0 + recovery_expression: last(/DeviceVault by HTTP/dv.scheduler_leader)=1 + tags: + - tag: component + value: scheduler + - tag: scope + value: availability + - tag: group + value: all + discovery_rules: + - uuid: 8ee5ed976cd24231b5aa9df6a263a488 + name: 'DeviceVault: Device groups discovery' + type: DEPENDENT + key: dv.group.discovery + delay: '0' + lifetime: 7d + description: Device groups from device-groups/; resources gone for 7 days are deleted. + item_prototypes: + - uuid: 4a1743a9f04b4189b51f9b7c67bcb040 + name: 'Group {#NAME}: Backups stored (1h)' + type: DEPENDENT + key: dv.group.stored_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].stored_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 2ed654c82eac4450a1b2c4c02a44799d + name: 'Group {#NAME}: Backups failed (1h)' + type: DEPENDENT + key: dv.group.failed_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished failed in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failed_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: b869fe46ea79420980eadfe98d853b5c + name: 'Group {#NAME}: Backups timed out (1h)' + type: DEPENDENT + key: dv.group.timed_out_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished timed out in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].timed_out_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 51ee81aa30304f88a44dfdd6757f1464 + name: 'Group {#NAME}: Backups finished (1h)' + type: DEPENDENT + key: dv.group.finished_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished in the last 1h, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].finished_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 48c25c537c584907856b846208fe0c1a + name: 'Group {#NAME}: Success ratio (1h)' + type: DEPENDENT + key: dv.group.success_ratio_1h[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 1h; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].success_ratio_1h.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 351b05bebd3c4163983df610d31e0497 + name: 'Group {#NAME}: Failure ratio (1h)' + type: DEPENDENT + key: dv.group.failure_ratio_1h[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failure_ratio_1h.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: d87c52054bdc4ddfbf4d0d2ec9c3d35a + name: 'Group {#NAME}: Backups retried (1h)' + type: DEPENDENT + key: dv.group.retried_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 1h that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].retried_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 6f2a8663787e4ac797c156ea763c5f26 + name: 'Group {#NAME}: Backups with changes (1h)' + type: DEPENDENT + key: dv.group.changed_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 1h that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].changed_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 117017daab604ad7ab378d85f5bd0ad1 + name: 'Group {#NAME}: Bytes stored (1h)' + type: DEPENDENT + key: dv.group.bytes_stored_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_stored_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + - uuid: ba3f06adaaa046aa971e9ad56dda3fbc + name: 'Group {#NAME}: Lines added (1h)' + type: DEPENDENT + key: dv.group.lines_added_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_added_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 88a3c5b2665541fe972d246e773d0869 + name: 'Group {#NAME}: Lines removed (1h)' + type: DEPENDENT + key: dv.group.lines_removed_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_removed_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 756ca5ce409a41f78acdad1ba7250cfb + name: 'Group {#NAME}: Lines modified (1h)' + type: DEPENDENT + key: dv.group.lines_modified_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_modified_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: abd8cd753fc646499ff15120b9a422e6 + name: 'Group {#NAME}: Backups stored (24h)' + type: DEPENDENT + key: dv.group.stored_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].stored_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 535fbd84bdd646548321735f8ac4c666 + name: 'Group {#NAME}: Backups failed (24h)' + type: DEPENDENT + key: dv.group.failed_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished failed in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failed_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 372673b6afeb4fd9b5287097bf0b27d3 + name: 'Group {#NAME}: Backups timed out (24h)' + type: DEPENDENT + key: dv.group.timed_out_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished timed out in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].timed_out_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 49fada23260244ee881849417c6e0d48 + name: 'Group {#NAME}: Backups finished (24h)' + type: DEPENDENT + key: dv.group.finished_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished in the last 24h, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].finished_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: a8943a7fb4b0431e8fae7f7da3f8fedf + name: 'Group {#NAME}: Success ratio (24h)' + type: DEPENDENT + key: dv.group.success_ratio_24h[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 24h; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].success_ratio_24h.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: e21cb82adad74ff197e492786fcc139f + name: 'Group {#NAME}: Failure ratio (24h)' + type: DEPENDENT + key: dv.group.failure_ratio_24h[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failure_ratio_24h.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: fcdc21a78c8f42fdb5ec1e086bfd223d + name: 'Group {#NAME}: Backups retried (24h)' + type: DEPENDENT + key: dv.group.retried_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 24h that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].retried_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 5b71e8727ce644e9b75f93b471ee1bce + name: 'Group {#NAME}: Backups with changes (24h)' + type: DEPENDENT + key: dv.group.changed_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 24h that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].changed_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: dbd8934338114dd592dc9324962fc3c5 + name: 'Group {#NAME}: Bytes stored (24h)' + type: DEPENDENT + key: dv.group.bytes_stored_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_stored_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + - uuid: 5af8b515f70f451a9d1a919bfbe4d9ba + name: 'Group {#NAME}: Lines added (24h)' + type: DEPENDENT + key: dv.group.lines_added_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_added_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 7262dfbec4de4246bdc2442ace00ea11 + name: 'Group {#NAME}: Lines removed (24h)' + type: DEPENDENT + key: dv.group.lines_removed_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_removed_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: b6456fc4cb2841bb88b93c6318c34ed9 + name: 'Group {#NAME}: Lines modified (24h)' + type: DEPENDENT + key: dv.group.lines_modified_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_modified_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: e006e153a9c24b58980bc73dae785d58 + name: 'Group {#NAME}: Backups stored (7d)' + type: DEPENDENT + key: dv.group.stored_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].stored_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 280948d4a30f4c499ad8a3be9c5fca45 + name: 'Group {#NAME}: Backups failed (7d)' + type: DEPENDENT + key: dv.group.failed_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished failed in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failed_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 7d35f9af7edc402baa1e8dfa431d6edd + name: 'Group {#NAME}: Backups timed out (7d)' + type: DEPENDENT + key: dv.group.timed_out_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished timed out in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].timed_out_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 51a45579fb5d4ea6819857e7c25c63b1 + name: 'Group {#NAME}: Backups finished (7d)' + type: DEPENDENT + key: dv.group.finished_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished in the last 7d, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].finished_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: db511c28c41c4f3bbd22fc746dfc4090 + name: 'Group {#NAME}: Success ratio (7d)' + type: DEPENDENT + key: dv.group.success_ratio_7d[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 7d; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].success_ratio_7d.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 6f8ed993f11b4fdd9beecec26509428d + name: 'Group {#NAME}: Failure ratio (7d)' + type: DEPENDENT + key: dv.group.failure_ratio_7d[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failure_ratio_7d.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: a8d524bfa90b453fbf30c35f9936cb9b + name: 'Group {#NAME}: Backups retried (7d)' + type: DEPENDENT + key: dv.group.retried_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 7d that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].retried_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: df916b4a2a0146888d9fd7d8e6727ddb + name: 'Group {#NAME}: Backups with changes (7d)' + type: DEPENDENT + key: dv.group.changed_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 7d that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].changed_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 9d36a29f9177405cbce8d326f78620d7 + name: 'Group {#NAME}: Bytes stored (7d)' + type: DEPENDENT + key: dv.group.bytes_stored_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_stored_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + - uuid: 095bc292c2f84aef84c3efcd4f5ebf05 + name: 'Group {#NAME}: Lines added (7d)' + type: DEPENDENT + key: dv.group.lines_added_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_added_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 0cf3fe47de744be3b9b14841c06bd54a + name: 'Group {#NAME}: Lines removed (7d)' + type: DEPENDENT + key: dv.group.lines_removed_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_removed_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: e8f9024e2a9d4c52b46ebf4fde49b578 + name: 'Group {#NAME}: Lines modified (7d)' + type: DEPENDENT + key: dv.group.lines_modified_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_modified_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: a1c4874145ff48a29e92606df214b84c + name: 'Group {#NAME}: Backups stored (all time)' + type: DEPENDENT + key: dv.group.stored_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every job of the device group that finished stored, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].stored_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 50f3a982eaab4bc487588fc87d0fdbb3 + name: 'Group {#NAME}: Backups failed (all time)' + type: DEPENDENT + key: dv.group.failed_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every job of the device group that finished failed, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failed_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 7f3ec882d62e4e2f9416cf950553ea88 + name: 'Group {#NAME}: Backups timed out (all time)' + type: DEPENDENT + key: dv.group.timed_out_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every job of the device group that finished timed out, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].timed_out_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 4f08a3e8e0d74b96a7dd24a39e332b49 + name: 'Group {#NAME}: Failures: connection (24h)' + type: DEPENDENT + key: dv.group.failures_connection_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "connection" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_connection_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 879984579275444592ca94183297c357 + name: 'Group {#NAME}: Failures: connection (all time)' + type: DEPENDENT + key: dv.group.failures_connection_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "connection" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_connection_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 2306998cc35a40c89c84feb86fc122b5 + name: 'Group {#NAME}: Failures: auth (24h)' + type: DEPENDENT + key: dv.group.failures_auth_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "auth" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_auth_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 5f12e0ec836946aba11b3434062d4ebd + name: 'Group {#NAME}: Failures: auth (all time)' + type: DEPENDENT + key: dv.group.failures_auth_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "auth" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_auth_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 8384eb8ba2714385958276aa9dbf90a9 + name: 'Group {#NAME}: Failures: timeout (24h)' + type: DEPENDENT + key: dv.group.failures_timeout_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "timeout" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_timeout_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 8c7bd16d1c994c41b8429b39ac571694 + name: 'Group {#NAME}: Failures: timeout (all time)' + type: DEPENDENT + key: dv.group.failures_timeout_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "timeout" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_timeout_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: f557e4ac67a24a41b8e59c352dc517fb + name: 'Group {#NAME}: Failures: command (24h)' + type: DEPENDENT + key: dv.group.failures_command_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "command" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_command_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 43653f2190f64d0eb8d91d52c8a98e94 + name: 'Group {#NAME}: Failures: command (all time)' + type: DEPENDENT + key: dv.group.failures_command_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "command" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_command_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: aaf8e8d28cf9461298c54b2ff0c62b44 + name: 'Group {#NAME}: Failures: verification (24h)' + type: DEPENDENT + key: dv.group.failures_verification_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "verification" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_verification_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: d00f86ac2d154282be9a601b842ae2d2 + name: 'Group {#NAME}: Failures: verification (all time)' + type: DEPENDENT + key: dv.group.failures_verification_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "verification" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_verification_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: e33934d57a3d48f6a54594f61e83ef58 + name: 'Group {#NAME}: Failures: storage (24h)' + type: DEPENDENT + key: dv.group.failures_storage_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "storage" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_storage_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 172085b4d3ae4280b03a2b88371f9191 + name: 'Group {#NAME}: Failures: storage (all time)' + type: DEPENDENT + key: dv.group.failures_storage_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "storage" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_storage_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 167d76c2d55b49138f0cba38f6d39e08 + name: 'Group {#NAME}: Failures: integrity (24h)' + type: DEPENDENT + key: dv.group.failures_integrity_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "integrity" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_integrity_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: e35e9bb3cd304b7bb7826563e92e429f + name: 'Group {#NAME}: Failures: integrity (all time)' + type: DEPENDENT + key: dv.group.failures_integrity_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "integrity" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_integrity_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 060485436b3744069abb69ba8c0775f0 + name: 'Group {#NAME}: Failures: push (24h)' + type: DEPENDENT + key: dv.group.failures_push_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "push" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_push_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 91421731449b4be4a10ead71b101808c + name: 'Group {#NAME}: Failures: push (all time)' + type: DEPENDENT + key: dv.group.failures_push_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "push" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_push_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 1cc1c70a147747ad9e51e2bf067b2618 + name: 'Group {#NAME}: Failures: orphaned (24h)' + type: DEPENDENT + key: dv.group.failures_orphaned_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "orphaned" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_orphaned_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 50f3e45fe3bf4fd4b57cd58f535570c3 + name: 'Group {#NAME}: Failures: orphaned (all time)' + type: DEPENDENT + key: dv.group.failures_orphaned_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "orphaned" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_orphaned_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 3f88976874c641f8b98b85e0431b1926 + name: 'Group {#NAME}: Failures: unknown (24h)' + type: DEPENDENT + key: dv.group.failures_unknown_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "unknown" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_unknown_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 691354533cfc4c9ba347b2a30ee6c8e3 + name: 'Group {#NAME}: Failures: unknown (all time)' + type: DEPENDENT + key: dv.group.failures_unknown_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "unknown" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_unknown_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 92332234b7104d6d8c406d012fd168d5 + name: 'Group {#NAME}: Backups queued' + type: DEPENDENT + key: dv.group.queued[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs waiting for a collector now + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].queued.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 1233d8e96cf4479383800b9e69f16dbe + name: 'Group {#NAME}: Backups running' + type: DEPENDENT + key: dv.group.running[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs being collected or stored now + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].running.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: ef3f8ffa77884bceaebdc996c819a6bc + name: 'Group {#NAME}: Duration of collection, average (24h)' + type: DEPENDENT + key: dv.group.duration_collect_avg[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The average collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_collect_avg.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: d03c4fc0cb284c109c7fe1a3ee1118b0 + name: 'Group {#NAME}: Duration of collection, median (24h)' + type: DEPENDENT + key: dv.group.duration_collect_p50[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The median collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_collect_p50.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 4553c05d602a4cb1bb1ba145b34ddd43 + name: 'Group {#NAME}: Duration of collection, 95th percentile (24h)' + type: DEPENDENT + key: dv.group.duration_collect_p95[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_collect_p95.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: e752315be6954eab88d2194825179283 + name: 'Group {#NAME}: Duration of collection, maximum (24h)' + type: DEPENDENT + key: dv.group.duration_collect_max[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The maximum collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_collect_max.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 58311ef021a5495286a58fbda935053e + name: 'Group {#NAME}: Duration of storage, average (24h)' + type: DEPENDENT + key: dv.group.duration_store_avg[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The average storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_store_avg.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 23070bed8d104c8b9ab6b224bd8ec6af + name: 'Group {#NAME}: Duration of storage, median (24h)' + type: DEPENDENT + key: dv.group.duration_store_p50[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The median storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_store_p50.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 37b51cbf9cb7424a8a5d65807bb5b18c + name: 'Group {#NAME}: Duration of storage, 95th percentile (24h)' + type: DEPENDENT + key: dv.group.duration_store_p95[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_store_p95.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: aab4ce82a85e41d6afbfe5d8d3780a23 + name: 'Group {#NAME}: Duration of storage, maximum (24h)' + type: DEPENDENT + key: dv.group.duration_store_max[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The maximum storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_store_max.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 8343582210d34439a2a3be8790f513fd + name: 'Group {#NAME}: Duration of queue wait, average (24h)' + type: DEPENDENT + key: dv.group.duration_wait_avg[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The average queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_wait_avg.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: de72ebd870aa42458dd10f8da8912531 + name: 'Group {#NAME}: Duration of queue wait, median (24h)' + type: DEPENDENT + key: dv.group.duration_wait_p50[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The median queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_wait_p50.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: b7a180d088e3444ba2ddb2ee1421e718 + name: 'Group {#NAME}: Duration of queue wait, 95th percentile (24h)' + type: DEPENDENT + key: dv.group.duration_wait_p95[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_wait_p95.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 74983e6b6f0041a890dfdbf3eed770d9 + name: 'Group {#NAME}: Duration of queue wait, maximum (24h)' + type: DEPENDENT + key: dv.group.duration_wait_max[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The maximum queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_wait_max.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 06d0154fa5b946b1add2b628392bce2b + name: 'Group {#NAME}: Duration of request to stored, average (24h)' + type: DEPENDENT + key: dv.group.duration_total_avg[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The average request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_total_avg.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 09633c4fc9dd4c169bb5688bc0a3c24c + name: 'Group {#NAME}: Duration of request to stored, median (24h)' + type: DEPENDENT + key: dv.group.duration_total_p50[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The median request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_total_p50.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 8f9b2f8294e445b28783167c2e7d232f + name: 'Group {#NAME}: Duration of request to stored, 95th percentile (24h)' + type: DEPENDENT + key: dv.group.duration_total_p95[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_total_p95.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 57eb899d55924ff2870fef96f6630361 + name: 'Group {#NAME}: Duration of request to stored, maximum (24h)' + type: DEPENDENT + key: dv.group.duration_total_max[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The maximum request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_total_max.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 90e8afac4b0d4096bf681ca76594280d + name: 'Group {#NAME}: Devices' + type: DEPENDENT + key: dv.group.devices_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active devices + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: b9c0c4d705df4853af9442714eb0547d + name: 'Group {#NAME}: Devices enabled' + type: DEPENDENT + key: dv.group.devices_enabled[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active devices that are not paused + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_enabled.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 7bcb7ede9459410a8bc51589a6354c19 + name: 'Group {#NAME}: Devices disabled' + type: DEPENDENT + key: dv.group.devices_disabled[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active devices that are paused + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_disabled.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: b70e01993cf94ced98a94172e3e4e326 + name: 'Group {#NAME}: Devices inactive' + type: DEPENDENT + key: dv.group.devices_inactive[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Devices marked inactive (retired, held) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_inactive.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: a4ad4efe2ff64bf285acea75e8b8f4d1 + name: 'Group {#NAME}: Devices never backed up' + type: DEPENDENT + key: dv.group.devices_never[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices with no stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_never.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 83889d57f3e843fbb88dabda15f8a869 + name: 'Group {#NAME}: Devices overdue' + type: DEPENDENT + key: dv.group.devices_overdue[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last success is older than their schedule allows + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_overdue.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + trigger_prototypes: + - uuid: ef999962267a453c93c7d59f33aa149a + expression: last(/DeviceVault by HTTP/dv.group.devices_overdue[{#ID}])>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.group.devices_overdue[{#ID}])=0 + name: 'DeviceVault: group {#NAME}: devices overdue' + opdata: 'Devices: {ITEM.LASTVALUE1}' + priority: WARNING + description: Devices of the group have no successful backup as recent as their schedule allows. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: freshness + - tag: scope + value: notice + - tag: group + value: '{#NAME}' + - uuid: a5b69d65d2864a0494dad950e0ad8a41 + name: 'Group {#NAME}: Devices failing 1+ times in a row' + type: DEPENDENT + key: dv.group.devices_failing_1[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 1 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_failing_1.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 7bcf29776f354fff99ff8962538f70c3 + name: 'Group {#NAME}: Devices failing 3+ times in a row' + type: DEPENDENT + key: dv.group.devices_failing_3[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 3 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_failing_3.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: b882b69d7599467faafc9638b0557742 + name: 'Group {#NAME}: Devices failing 5+ times in a row' + type: DEPENDENT + key: dv.group.devices_failing_5[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 5 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_failing_5.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 3be22af8cf43425c9ebce3757bc9fd6f + name: 'Group {#NAME}: Oldest last success, age' + type: DEPENDENT + key: dv.group.oldest_success_age[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: Seconds since the stalest active, enabled device last backed up (0 with none backed up) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].oldest_success_age.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + trigger_prototypes: + - uuid: b5ed63d32cbb4908aa87101d1bf517b6 + expression: last(/DeviceVault by HTTP/dv.group.oldest_success_age[{#ID}])>{$DV.STALE.MAX} + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.group.oldest_success_age[{#ID}])<={$DV.STALE.MAX} + name: 'DeviceVault: group {#NAME}: a device has not backed up for over {$DV.STALE.MAX}' + opdata: 'Age: {ITEM.LASTVALUE1}' + priority: AVERAGE + description: The stalest active, enabled device of the group last backed up longer ago than {$DV.STALE.MAX}. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: freshness + - tag: scope + value: notice + - tag: group + value: '{#NAME}' + - uuid: a43b52b8c5574bb6abb61ac006f6bfa7 + name: 'Group {#NAME}: Oldest last success, device' + type: DEPENDENT + key: dv.group.oldest_success_device[{#ID}] + delay: '0' + value_type: CHAR + trends: '0' + description: Name of that device + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].oldest_success_device.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 59084e7b2dd444fa830cbfd5f324e0e2 + name: 'Group {#NAME}: Backups held' + type: DEPENDENT + key: dv.group.backups_held[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups not pruned by retention + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].backups_held.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + - uuid: 326d0c0421094876b09aafe5e8a8f686 + name: 'Group {#NAME}: Bytes held' + type: DEPENDENT + key: dv.group.bytes_held[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of those backups (as the collectors reported it) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_held.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + trigger_prototypes: + - uuid: 5f59b736655e4699aeef3e108e377c3b + expression: last(/DeviceVault by HTTP/dv.group.success_ratio_24h[{#ID}])<{$DV.SUCCESS.MIN} and last(/DeviceVault by HTTP/dv.group.finished_24h[{#ID}])>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.group.success_ratio_24h[{#ID}])>={$DV.SUCCESS.MIN} or last(/DeviceVault by HTTP/dv.group.finished_24h[{#ID}])=0 + name: 'DeviceVault: group {#NAME}: success ratio below {$DV.SUCCESS.MIN}% (24h)' + opdata: 'Success ratio: {ITEM.LASTVALUE1}' + priority: WARNING + description: Fewer backups than {$DV.SUCCESS.MIN}% of those that finished in the last 24h were stored. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: backups + - tag: scope + value: notice + - tag: group + value: '{#NAME}' + - uuid: 772a2579e246490ba7fcbebd9d5d2e96 + expression: ({$DV.FAIL.CONSEC}<3 and last(/DeviceVault by HTTP/dv.group.devices_failing_1[{#ID}])>0) or ({$DV.FAIL.CONSEC}>=3 and {$DV.FAIL.CONSEC}<5 and last(/DeviceVault by HTTP/dv.group.devices_failing_3[{#ID}])>0) or ({$DV.FAIL.CONSEC}>=5 and last(/DeviceVault by HTTP/dv.group.devices_failing_5[{#ID}])>0) + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: ({$DV.FAIL.CONSEC}<3 and last(/DeviceVault by HTTP/dv.group.devices_failing_1[{#ID}])=0) or ({$DV.FAIL.CONSEC}>=3 and {$DV.FAIL.CONSEC}<5 and last(/DeviceVault by HTTP/dv.group.devices_failing_3[{#ID}])=0) or ({$DV.FAIL.CONSEC}>=5 and last(/DeviceVault by HTTP/dv.group.devices_failing_5[{#ID}])=0) + name: 'DeviceVault: group {#NAME}: devices failing {$DV.FAIL.CONSEC}+ times in a row' + priority: WARNING + description: Devices of the group failed their last {$DV.FAIL.CONSEC} or more backups (counted at 1, 3 or 5). + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: failures + - tag: scope + value: notice + - tag: group + value: '{#NAME}' + graph_prototypes: + - uuid: b15469bb49da47abb0105cf87e3d4008 + name: 'Group {#NAME}: Device freshness' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.group.devices_overdue[{#ID}] + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.group.devices_never[{#ID}] + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.group.devices_failing_1[{#ID}] + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.group.devices_failing_3[{#ID}] + - sortorder: '4' + color: FC6EA3 + item: + host: DeviceVault by HTTP + key: dv.group.devices_failing_5[{#ID}] + - uuid: 07225627607e4585bf2edd94dd42be74 + name: 'Group {#NAME}: Success ratio' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.group.success_ratio_1h[{#ID}] + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.group.success_ratio_24h[{#ID}] + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.group.success_ratio_7d[{#ID}] + master_item: + key: dv.group.get + lld_macro_paths: + - lld_macro: '{#ID}' + path: $.id + - lld_macro: '{#NAME}' + path: $.name + preprocessing: + - type: JSONPATH + parameters: + - $.data + - uuid: bae2bee53cb14a42beb2570a01fa4bc2 + name: 'DeviceVault: Workers discovery' + type: DEPENDENT + key: dv.worker.discovery + delay: '0' + lifetime: 7d + description: Workers from workers/; resources gone for 7 days are deleted. + item_prototypes: + - uuid: e955765ba1224f88ad43a32a28fe202d + name: 'Worker {#NAME}: Role' + type: DEPENDENT + key: dv.worker.role[{#ID}] + delay: '0' + value_type: CHAR + trends: '0' + description: collector, storage or receiver + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].role.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: 8aadecc7e3804a13b5983b03df760205 + name: 'Worker {#NAME}: Status' + type: DEPENDENT + key: dv.worker.status_code[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Approval status + valuemap: + name: DeviceVault worker status + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].status_code.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: 11ffb095bd704547a65927072382e35b + name: 'Worker {#NAME}: Online' + type: DEPENDENT + key: dv.worker.online[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Sent a heartbeat recently + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].online.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: f163f1fc30d54e02887660b7703741ce + name: 'Worker {#NAME}: Heartbeat age' + type: DEPENDENT + key: dv.worker.heartbeat_age[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: Seconds since the last heartbeat (-1 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].heartbeat_age.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: ff011da8cf8345709d03ce890a8145ca + name: 'Worker {#NAME}: Pending approval for' + type: DEPENDENT + key: dv.worker.pending_age[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: Seconds since a pending worker registered (0 once decided) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].pending_age.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + trigger_prototypes: + - uuid: 24937a04ea184c9eb2b509b68a29beb8 + expression: last(/DeviceVault by HTTP/dv.worker.pending_age[{#ID}])>{$DV.PENDING.MAX} + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.worker.pending_age[{#ID}])<={$DV.PENDING.MAX} + name: 'DeviceVault: worker {#NAME}: pending approval for over {$DV.PENDING.MAX}' + priority: WARNING + description: A worker registered and is waiting for an administrator to approve or revoke it (Workers page). + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: worker + - tag: scope + value: notice + - tag: group + value: all + - uuid: 74195fd317f94c4f9dd10e9330c3e9f8 + name: 'Worker {#NAME}: Version' + type: DEPENDENT + key: dv.worker.version[{#ID}] + delay: '0' + value_type: CHAR + trends: '0' + description: The DeviceVault version it runs + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].version.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: b519b1cd45fa4b9291b46e13aafe7f19 + name: 'Worker {#NAME}: Assignment behind' + type: DEPENDENT + key: dv.worker.assignment_behind[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Applied assignment version is older than the current one + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].assignment_behind.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + trigger_prototypes: + - uuid: 755a31b876ef400fbb3d39602ab3c91d + expression: min(/DeviceVault by HTTP/dv.worker.assignment_behind[{#ID}],10m)=1 and last(/DeviceVault by HTTP/dv.worker.assignment_behind[{#ID}],#1:now-10m)=1 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.worker.assignment_behind[{#ID}])=0 + name: 'DeviceVault: worker {#NAME}: assignment behind for 10 minutes' + priority: WARNING + description: The worker has not applied its current assignment for 10 minutes. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: worker + - tag: scope + value: notice + - tag: group + value: all + trigger_prototypes: + - uuid: e88297272d484bb3aba01988e234fd68 + expression: last(/DeviceVault by HTTP/dv.worker.status_code[{#ID}])=1 and (last(/DeviceVault by HTTP/dv.worker.heartbeat_age[{#ID}])>{$DV.WORKER.HEARTBEAT.MAX} or last(/DeviceVault by HTTP/dv.worker.heartbeat_age[{#ID}])<0) + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.worker.status_code[{#ID}])<>1 or (last(/DeviceVault by HTTP/dv.worker.heartbeat_age[{#ID}])>=0 and last(/DeviceVault by HTTP/dv.worker.heartbeat_age[{#ID}])<={$DV.WORKER.HEARTBEAT.MAX}) + name: 'DeviceVault: worker {#NAME}: no heartbeat for {$DV.WORKER.HEARTBEAT.MAX}' + opdata: 'Heartbeat age: {ITEM.LASTVALUE2}' + priority: AVERAGE + description: 'An approved worker has not sent a heartbeat recently: it is down or cannot reach the broker.' + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: worker + - tag: scope + value: availability + - tag: group + value: all + master_item: + key: dv.worker.get + lld_macro_paths: + - lld_macro: '{#ID}' + path: $.id + - lld_macro: '{#NAME}' + path: $.name + preprocessing: + - type: JSONPATH + parameters: + - $.data + - uuid: d40a955469964db28f69ea210a9f9705 + name: 'DeviceVault: Storage targets discovery' + type: DEPENDENT + key: dv.target.discovery + delay: '0' + lifetime: 7d + description: Storage targets from storage-targets/; resources gone for 7 days are deleted. + item_prototypes: + - uuid: e979fd4876f24de9a666e3c1d6caa68a + name: 'Storage target {#NAME}: Driver' + type: DEPENDENT + key: dv.target.driver[{#ID}] + delay: '0' + value_type: CHAR + trends: '0' + description: Storage driver + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].driver.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: 8db9426b364f4cfcac71b9f467124c2c + name: 'Storage target {#NAME}: Workers online' + type: DEPENDENT + key: dv.target.workers_online[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Approved storage workers serving it that are online + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].workers_online.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + trigger_prototypes: + - uuid: 4a6d3f0ca9f44be1917c4106d0ad5e62 + expression: last(/DeviceVault by HTTP/dv.target.workers_online[{#ID}])=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.target.workers_online[{#ID}])>0 + name: 'DeviceVault: storage target {#NAME}: no storage worker online' + priority: HIGH + description: No approved, online storage worker serves the target, so backups for it cannot be stored. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: storage + - tag: scope + value: availability + - tag: group + value: all + - uuid: d6586e7c647b46d194ecbbafc528e0a1 + name: 'Storage target {#NAME}: Last stored' + type: DEPENDENT + key: dv.target.last_stored_at[{#ID}] + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When a backup was last stored there (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].last_stored_at.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: d34db5bbcfbf4715a9785b40d3fa4834 + name: 'Storage target {#NAME}: Storage failures (24h)' + type: DEPENDENT + key: dv.target.storage_failures_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs for it that failed at the storage step in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].storage_failures_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: 83d105687a954b309af7051bd23c1d9c + name: 'Storage target {#NAME}: Backups held' + type: DEPENDENT + key: dv.target.backups_held[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups not pruned by retention + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].backups_held.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: c93b2689278346aab31af5b6e192520f + name: 'Storage target {#NAME}: Bytes held' + type: DEPENDENT + key: dv.target.bytes_held[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of those backups + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_held.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: cee76a86f53a4e629282cc68203d92c3 + name: 'Storage target {#NAME}: Bytes stored (24h)' + type: DEPENDENT + key: dv.target.bytes_stored_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored there in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_stored_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + master_item: + key: dv.target.get + lld_macro_paths: + - lld_macro: '{#ID}' + path: $.id + - lld_macro: '{#NAME}' + path: $.name + preprocessing: + - type: JSONPATH + parameters: + - $.data + - uuid: f9a54bc1337d4596b693c1249c91b184 + name: 'DeviceVault: Receivers discovery' + type: DEPENDENT + key: dv.receiver.discovery + delay: '0' + lifetime: 7d + description: Receivers from receivers/; resources gone for 7 days are deleted. + item_prototypes: + - uuid: 71c074e5d2bb463dacf4b2f4d0df527c + name: 'Receiver {#NAME}: Online' + type: DEPENDENT + key: dv.receiver.online[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Sent a heartbeat recently + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].online.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.receiver.get + tags: + - tag: component + value: receiver + - tag: receiver + value: '{#NAME}' + - uuid: 224a5839f0964e0890e32f4ebaec2596 + name: 'Receiver {#NAME}: Pushes accepted (24h)' + type: DEPENDENT + key: dv.receiver.accepted_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Files it accepted in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].accepted_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.receiver.get + tags: + - tag: component + value: receiver + - tag: receiver + value: '{#NAME}' + - uuid: 03384f6d675b4e35bfc9b245767a6e73 + name: 'Receiver {#NAME}: Pushes refused (24h)' + type: DEPENDENT + key: dv.receiver.refused_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Pushes it refused in the last 24h (a lower bound) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].refused_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.receiver.get + tags: + - tag: component + value: receiver + - tag: receiver + value: '{#NAME}' + - uuid: d95067a88f354eceb09b5b7ecbe9cb25 + name: 'Receiver {#NAME}: Last push' + type: DEPENDENT + key: dv.receiver.last_push_at[{#ID}] + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When it last accepted a file (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].last_push_at.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.receiver.get + tags: + - tag: component + value: receiver + - tag: receiver + value: '{#NAME}' + master_item: + key: dv.receiver.get + lld_macro_paths: + - lld_macro: '{#ID}' + path: $.id + - lld_macro: '{#NAME}' + path: $.name + preprocessing: + - type: JSONPATH + parameters: + - $.data + tags: + - tag: class + value: software + - tag: target + value: devicevault + macros: + - macro: '{$DV.INTERVAL}' + value: 1m + description: How often the monitoring endpoints are polled. + - macro: '{$DV.NODATA}' + value: 5m + description: No answer from the monitoring API for this long raises the unreachable trigger. + - macro: '{$DV.SUCCESS.MIN}' + value: '95' + description: Lowest acceptable share of backups stored over 24h, in percent. + - macro: '{$DV.STALE.MAX}' + value: 2d + description: Longest acceptable age of the oldest last successful backup in a device group. + - macro: '{$DV.FAIL.CONSEC}' + value: '3' + description: Consecutive failures that count a device as failing (rounded down to 1, 3 or 5). + - macro: '{$DV.DURATION.MAX}' + value: 10m + description: Longest acceptable 95th percentile request-to-stored time over 24h. + - macro: '{$DV.WORKER.HEARTBEAT.MAX}' + value: 3m + description: Longest acceptable gap between an approved worker's heartbeats. + - macro: '{$DV.PENDING.MAX}' + value: 1h + description: Longest a worker may wait for approval before it is reported. + - macro: '{$DV.SKEW.MAX}' + value: '60' + description: Largest acceptable difference between the DeviceVault and Zabbix clocks, in seconds. + - macro: '{$DV.PUSH.REFUSED.MAX}' + value: '10' + description: More refused pushes than this within an hour raise a trigger. + dashboards: + - uuid: 1110baff42454613aea6d7bb5abb58f6 + name: DeviceVault overview + pages: + - name: Backups and platform + widgets: + - type: GRAPH_CLASSIC + width: '12' + height: '5' + fields: + - type: GRAPH + name: graphid + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Success ratio' + - type: GRAPH_CLASSIC + x: '12' + width: '12' + height: '5' + fields: + - type: GRAPH + name: graphid + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Request to stored time (24h)' + - type: GRAPH_CLASSIC + y: '5' + width: '12' + height: '5' + fields: + - type: GRAPH + name: graphid + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Failures by reason (24h)' + - type: GRAPH_CLASSIC + x: '12' + y: '5' + width: '12' + height: '5' + fields: + - type: GRAPH + name: graphid + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Device freshness' + - type: GRAPH_CLASSIC + y: '10' + width: '12' + height: '5' + fields: + - type: GRAPH + name: graphid + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Platform health' + - type: GRAPH_PROTOTYPE + x: '12' + y: '10' + width: '12' + height: '5' + fields: + - type: INTEGER + name: columns + value: '1' + - type: GRAPH_PROTOTYPE + name: graphid + value: + host: DeviceVault by HTTP + name: 'Group {#NAME}: Device freshness' + - type: GRAPH_PROTOTYPE + y: '15' + width: '12' + height: '5' + fields: + - type: INTEGER + name: columns + value: '1' + - type: GRAPH_PROTOTYPE + name: graphid + value: + host: DeviceVault by HTTP + name: 'Group {#NAME}: Success ratio' + valuemaps: + - uuid: ad56712d048b4e83a64ea74917e50cc8 + name: DeviceVault worker status + mappings: + - value: '0' + newvalue: pending + - value: '1' + newvalue: approved + - value: '2' + newvalue: revoked + - uuid: f3da0efc8f6148f2bba882f825995c2d + name: DeviceVault yes/no + mappings: + - value: '0' + newvalue: 'no' + - value: '1' + newvalue: 'yes' + - uuid: e5718ee14df14607bf11909779322638 + template: DeviceVault Device by HTTP + name: DeviceVault Device by HTTP + description: |- + DeviceVault backup status of this device, matched on its address ({HOST.IP}) or {$DV.DEVICE.ID}. + + Link it to the device hosts Zabbix already monitors. {$DV.RELAY.URL} is the monitoring relay on a DeviceVault receiver + in the zone of the proxy that monitors the host (set it once per zone on a site template that links this one), or + {$DV.URL}/api/monitoring where the proxy can reach DeviceVault directly. {$DV.TOKEN} (a service account's token with + the monitoring scope) is meant to be a global macro. + + Generated by DeviceVault from its metric catalogue; see DEPLOYMENT.md, section Monitoring. + groups: + - name: Templates/Applications + items: + - uuid: ca96cacc609e469094c7c3c06ff3ac51 + name: 'DeviceVault: Get device status' + type: HTTP_AGENT + key: dv.device.get + delay: '{$DV.DEVICE.INTERVAL}' + history: 1h + trends: '0' + value_type: TEXT + description: The device's answer from DeviceVault's published snapshot (503 when the snapshot is stale). + timeout: 15s + url: '{$DV.RELAY.URL}/device/?ip={HOST.IP}&id={$DV.DEVICE.ID}' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + status_codes: 200,503 + verify_peer: 'NO' + verify_host: 'NO' + - uuid: 0b705a2ff03d4b6abf9c05b3b87b0fce + name: 'DeviceVault: Answer error' + type: DEPENDENT + key: dv.device.error + delay: '0' + value_type: CHAR + trends: '0' + description: '"stale" when the relay has no current snapshot (503), otherwise "none".' + preprocessing: + - type: JSONPATH + parameters: + - $.error + error_handler: CUSTOM_VALUE + error_handler_params: none + master_item: + key: dv.device.get + tags: + - tag: component + value: api + - uuid: 2f38def1d45148ea81892889eb3fcd7c + name: 'DeviceVault: Found in DeviceVault' + type: DEPENDENT + key: dv.device.found + delay: '0' + value_type: UNSIGNED + description: 1 when the address or id matches a device the token may see + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.found + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + triggers: + - uuid: f6d1caf6791f45e0846e4df9b722fe64 + expression: last(/DeviceVault Device by HTTP/dv.device.found)=0 and {$DV.NOTFOUND.ALERT}=1 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.found)=1 or {$DV.NOTFOUND.ALERT}=0 + name: 'DeviceVault: not backed up by DeviceVault' + priority: INFO + description: No device DeviceVault backs up (and the token may see) has this address or id. Set {$DV.NOTFOUND.ALERT} to 0 on hosts that are not meant to be backed up. + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: notice + - tag: group + value: all + - uuid: 40a1efbd13964034b8c293dea346e521 + name: 'DeviceVault: Id' + type: DEPENDENT + key: dv.device.id + delay: '0' + value_type: CHAR + trends: '0' + description: Identifier, used by low-level discovery + preprocessing: + - type: JSONPATH + parameters: + - $.id + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 8467a2e893ce42ddb52235e697202258 + name: 'DeviceVault: Name' + type: DEPENDENT + key: dv.device.name + delay: '0' + value_type: CHAR + trends: '0' + description: Name + preprocessing: + - type: JSONPATH + parameters: + - $.name + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: bb53b5154f4e4bdf870bf672f5397b33 + name: 'DeviceVault: Device group' + type: DEPENDENT + key: dv.device.group + delay: '0' + value_type: CHAR + trends: '0' + description: Device group name (empty if none) + preprocessing: + - type: JSONPATH + parameters: + - $.group + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: db859423317a497e9676d26870d419fb + name: 'DeviceVault: Address' + type: DEPENDENT + key: dv.device.ip + delay: '0' + value_type: CHAR + trends: '0' + description: The address DeviceVault backs up + preprocessing: + - type: JSONPATH + parameters: + - $.ip + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 9db0f69fc35e4232b6f2383904ea2f2c + name: 'DeviceVault: Enabled' + type: DEPENDENT + key: dv.device.enabled + delay: '0' + value_type: UNSIGNED + description: Not paused + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.enabled + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: e0fa056945c349aa90d5c30c4e3164c3 + name: 'DeviceVault: Lifecycle' + type: DEPENDENT + key: dv.device.lifecycle_code + delay: '0' + value_type: UNSIGNED + description: Active, inactive (retired, held) or deletable + valuemap: + name: DeviceVault lifecycle + preprocessing: + - type: JSONPATH + parameters: + - $.lifecycle_code + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: fa7d2d9077b74297bc7b4342337122b6 + name: 'DeviceVault: Backup status' + type: DEPENDENT + key: dv.device.status_code + delay: '0' + value_type: UNSIGNED + description: 'The latest job: stored, failed, timed out, running, or never backed up' + valuemap: + name: DeviceVault job status + preprocessing: + - type: JSONPATH + parameters: + - $.status_code + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: cad221c62dd24a39b90c4b664c1f1d47 + name: 'DeviceVault: Last attempt' + type: DEPENDENT + key: dv.device.last_attempt_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When the latest job finished, or started if it is still running (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_attempt_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: f44841ebbfc044dcb6747ebe4dc38930 + name: 'DeviceVault: Last successful backup' + type: DEPENDENT + key: dv.device.last_success_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When the newest stored backup finished (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_success_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 1c65dfa53ed64c65959211fa327f0bb9 + name: 'DeviceVault: Last successful backup, age' + type: DEPENDENT + key: dv.device.last_success_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since then, at answer time (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_success_age + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 768084e7b85f4edea770786703b6a0b7 + name: 'DeviceVault: Last failure' + type: DEPENDENT + key: dv.device.last_failure_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When a job last failed or timed out (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_failure_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: b0e0321d217c46aea54c9d3f708d5c84 + name: 'DeviceVault: Failure reason' + type: DEPENDENT + key: dv.device.failure_reason_code + delay: '0' + value_type: UNSIGNED + description: Why the latest job failed (none when it did not) + valuemap: + name: DeviceVault failure reason + preprocessing: + - type: JSONPATH + parameters: + - $.failure_reason_code + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 33095c096c624dd09f226d2a5770c0ec + name: 'DeviceVault: Consecutive failures' + type: DEPENDENT + key: dv.device.consecutive_failures + delay: '0' + value_type: UNSIGNED + description: Finished jobs that failed since the last stored one + preprocessing: + - type: JSONPATH + parameters: + - $.consecutive_failures + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: f0440c89b95944408aa36391dd0bbf49 + name: 'DeviceVault: Expected interval' + type: DEPENDENT + key: dv.device.expected_interval + delay: '0' + value_type: FLOAT + units: s + description: The longest gap between the schedule's runs (0 without a schedule) + preprocessing: + - type: JSONPATH + parameters: + - $.expected_interval + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: b98a330701cd4dd19f1105464588fafa + name: 'DeviceVault: Next backup due by' + type: DEPENDENT + key: dv.device.next_due_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When a successful backup is due before the device counts as overdue (0 without a schedule) + preprocessing: + - type: JSONPATH + parameters: + - $.next_due_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 5d1320ecf6984edaaf73ee521c4f059e + name: 'DeviceVault: Overdue' + type: DEPENDENT + key: dv.device.overdue + delay: '0' + value_type: UNSIGNED + description: Its last success is older than its schedule allows + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.overdue + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: e2e60ebeabfe486ba4ef8a7cfa33f821 + name: 'DeviceVault: Last collection time' + type: DEPENDENT + key: dv.device.last_collect_duration + delay: '0' + value_type: FLOAT + units: s + description: Collection time of the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_collect_duration + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: afd4c90928d948f48f348ee27df39382 + name: 'DeviceVault: Last storage time' + type: DEPENDENT + key: dv.device.last_store_duration + delay: '0' + value_type: FLOAT + units: s + description: Storage time of the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_store_duration + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: dc616356820b4b7da3ff5704f9736c8b + name: 'DeviceVault: Last backup duration' + type: DEPENDENT + key: dv.device.last_duration + delay: '0' + value_type: FLOAT + units: s + description: Request-to-stored time of the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_duration + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: e6100b260b074f869daea2da5b032a60 + name: 'DeviceVault: Last backup size' + type: DEPENDENT + key: dv.device.last_size + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_size + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 7901f7bda8bd4bbfbd1ec02e56ce7d47 + name: 'DeviceVault: Last backup changed' + type: DEPENDENT + key: dv.device.last_changed + delay: '0' + value_type: UNSIGNED + description: The newest stored backup differs from the one before + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.last_changed + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: db66c12e466443a29fa4b20bfb032636 + name: 'DeviceVault: Lines added' + type: DEPENDENT + key: dv.device.last_lines_added + delay: '0' + value_type: UNSIGNED + description: By the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_lines_added + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: ef580c780a1c4398a4dab9ec72a9088c + name: 'DeviceVault: Lines removed' + type: DEPENDENT + key: dv.device.last_lines_removed + delay: '0' + value_type: UNSIGNED + description: By the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_lines_removed + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 326ddb83a0554307bc3f6f3bfce1978d + name: 'DeviceVault: Lines modified' + type: DEPENDENT + key: dv.device.last_lines_modified + delay: '0' + value_type: UNSIGNED + description: By the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_lines_modified + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 3d17344ee7df40b9ab4dc23a89c056e8 + name: 'DeviceVault: Backups held' + type: DEPENDENT + key: dv.device.backups_held + delay: '0' + value_type: UNSIGNED + description: Stored backups not pruned + preprocessing: + - type: JSONPATH + parameters: + - $.backups_held + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: c0cbd718251245bbb16143c842262d05 + name: 'DeviceVault: Pinned backups' + type: DEPENDENT + key: dv.device.pinned + delay: '0' + value_type: UNSIGNED + description: Backups pinned against retention + preprocessing: + - type: JSONPATH + parameters: + - $.pinned + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 23a76f004983476dae7f52b3c449dd7c + name: 'DeviceVault: Push mode' + type: DEPENDENT + key: dv.device.push_mode + delay: '0' + value_type: CHAR + trends: '0' + description: off, ssh or tftp + preprocessing: + - type: JSONPATH + parameters: + - $.push_mode + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: f2bd3455a4fc47e99d290344d55124a5 + name: 'DeviceVault: Last push' + type: DEPENDENT + key: dv.device.last_push_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When the device last pushed a file that was accepted (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_push_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 78a87c73abb04e1e9caff7d82e2aca00 + name: 'DeviceVault: Success ratio (7d)' + type: DEPENDENT + key: dv.device.success_ratio_7d + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 7 days; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.success_ratio_7d + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 6a1a3f1be3414722b8ff0fe93e85a0d3 + name: 'DeviceVault: Data published at' + type: DEPENDENT + key: dv.device.published_at + delay: '0' + value_type: CHAR + trends: '0' + description: When the orchestrator published the snapshot this answer comes from + preprocessing: + - type: JSONPATH + parameters: + - $.published_at + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 186b51512b1749d48a1e95c7c6610c39 + name: 'DeviceVault: Data age' + type: DEPENDENT + key: dv.device.published_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since that snapshot was published + preprocessing: + - type: JSONPATH + parameters: + - $.published_age + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + triggers: + - uuid: 0f0ee0d3ba0843139e2632e377aea3c4 + expression: last(/DeviceVault Device by HTTP/dv.device.published_age)>{$DV.DEVICE.DATA.MAX} + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.published_age)<={$DV.DEVICE.DATA.MAX} + name: 'DeviceVault: DeviceVault data is old' + priority: WARNING + description: 'The snapshot answering was published longer ago than {$DV.DEVICE.DATA.MAX}: the orchestrator has stopped publishing while the relay still serves.' + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: api + - tag: scope + value: availability + - tag: group + value: all + tags: + - tag: class + value: software + - tag: target + value: devicevault + macros: + - macro: '{$DV.RELAY.URL}' + value: https://receiver-site-a:8443 + description: The monitoring relay of a receiver in this zone (or {$DV.URL}/api/monitoring for the orchestrator). Set it once per zone on a site template. + - macro: '{$DV.DEVICE.ID}' + description: 'DeviceVault''s id for this device, for a host whose interface address is not the one DeviceVault backs up. Empty: match on {HOST.IP}.' + - macro: '{$DV.DEVICE.INTERVAL}' + value: 15m + description: How often the device status is polled (snapshots are published every 15m). + - macro: '{$DV.DEVICE.NODATA}' + value: 45m + description: No answer for this long (three poll intervals) raises the no-data trigger. + - macro: '{$DV.NOTFOUND.ALERT}' + value: '1' + description: 1 reports hosts DeviceVault does not back up; 0 switches that trigger off. + - macro: '{$DV.DEVICE.FAIL.CONSEC}' + value: '2' + description: Consecutive failed backups that raise the failing trigger. + - macro: '{$DV.DEVICE.STALE.MAX}' + value: '0' + description: Raise a trigger when the last successful backup is older than this (for example 2d); 0 switches it off in favour of the schedule-based overdue trigger. + - macro: '{$DV.DEVICE.DATA.MAX}' + value: 45m + description: Raise a trigger when the published snapshot answering is older than this. + valuemaps: + - uuid: e9df205963684b62aa3d2152dcef8d53 + name: DeviceVault job status + mappings: + - value: '0' + newvalue: never + - value: '1' + newvalue: stored + - value: '2' + newvalue: failed + - value: '3' + newvalue: timed_out + - value: '4' + newvalue: running + - uuid: fd31acf62a2043608afd3dfff4a5cf74 + name: DeviceVault failure reason + mappings: + - value: '0' + newvalue: none + - value: '1' + newvalue: connection + - value: '2' + newvalue: auth + - value: '3' + newvalue: timeout + - value: '4' + newvalue: command + - value: '5' + newvalue: verification + - value: '6' + newvalue: storage + - value: '7' + newvalue: integrity + - value: '8' + newvalue: push + - value: '9' + newvalue: orphaned + - value: '10' + newvalue: unknown + - uuid: 25b4248acc1947348e0c2a1f26787204 + name: DeviceVault lifecycle + mappings: + - value: '0' + newvalue: active + - value: '1' + newvalue: inactive + - value: '2' + newvalue: deletable + - uuid: 428caf8ffb724ce9bea7019a52add6e2 + name: DeviceVault yes/no + mappings: + - value: '0' + newvalue: 'no' + - value: '1' + newvalue: 'yes' + triggers: + - uuid: 5f7ec9745d7e4ec6805dac9604d8e610 + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + name: 'DeviceVault: monitoring figures are stale' + priority: WARNING + description: The figures are older than three poll intervals, or none arrived for {$DV.NODATA}. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + tags: + - tag: component + value: api + - tag: scope + value: availability + - tag: group + value: all + - uuid: 262d6d188d60400ba208aca504a25c50 + expression: last(/DeviceVault by HTTP/dv.success_ratio_24h)<{$DV.SUCCESS.MIN} and last(/DeviceVault by HTTP/dv.finished_24h)>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.success_ratio_24h)>={$DV.SUCCESS.MIN} or last(/DeviceVault by HTTP/dv.finished_24h)=0 + name: 'DeviceVault: success ratio below {$DV.SUCCESS.MIN}% (24h)' + opdata: 'Success ratio: {ITEM.LASTVALUE1}' + priority: WARNING + description: Fewer backups than {$DV.SUCCESS.MIN}% of those that finished in the last 24h were stored. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: backups + - tag: scope + value: notice + - tag: group + value: all + - uuid: d0f52a03ad3142c3810fd1ce8e0f0dba + expression: last(/DeviceVault by HTTP/dv.duration_total_p95)>{$DV.DURATION.MAX} and last(/DeviceVault by HTTP/dv.finished_24h)>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.duration_total_p95)<={$DV.DURATION.MAX} or last(/DeviceVault by HTTP/dv.finished_24h)=0 + name: 'DeviceVault: backups are slow (p95 over {$DV.DURATION.MAX})' + opdata: 'p95: {ITEM.LASTVALUE1}' + priority: WARNING + description: The 95th percentile request-to-stored time of the last 24h is above {$DV.DURATION.MAX}. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: timing + - tag: scope + value: performance + - tag: group + value: all + - uuid: fa2f7d0d147341349b66ca380fcd7468 + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + name: 'DeviceVault: no data from DeviceVault' + priority: WARNING + description: The relay at {$DV.RELAY.URL} did not answer for {$DV.DEVICE.NODATA}, refused the token, or has no current snapshot (503 stale). The other DeviceVault triggers of this host depend on this one. + tags: + - tag: component + value: api + - tag: scope + value: availability + - tag: group + value: all + - uuid: 5ccbaa48f4f34b5999d0c20e5d6e67d5 + expression: last(/DeviceVault Device by HTTP/dv.device.overdue)=1 and last(/DeviceVault Device by HTTP/dv.device.found)=1 and last(/DeviceVault Device by HTTP/dv.device.enabled)=1 and last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.overdue)=0 or last(/DeviceVault Device by HTTP/dv.device.found)<>1 or last(/DeviceVault Device by HTTP/dv.device.enabled)<>1 or last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)<>0 + name: 'DeviceVault: backup overdue' + priority: WARNING + description: The last successful backup is older than the schedule allows. + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: notice + - tag: group + value: all + - uuid: a9bfb97f075c4a22aa3121faeba05a68 + expression: last(/DeviceVault Device by HTTP/dv.device.consecutive_failures)>={$DV.DEVICE.FAIL.CONSEC} and last(/DeviceVault Device by HTTP/dv.device.found)=1 and last(/DeviceVault Device by HTTP/dv.device.enabled)=1 and last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.consecutive_failures)<{$DV.DEVICE.FAIL.CONSEC} or last(/DeviceVault Device by HTTP/dv.device.found)<>1 or last(/DeviceVault Device by HTTP/dv.device.enabled)<>1 or last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)<>0 + name: 'DeviceVault: backup failing' + opdata: 'Consecutive failures: {ITEM.LASTVALUE1}' + priority: WARNING + description: The last {$DV.DEVICE.FAIL.CONSEC} or more backups failed. + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: notice + - tag: group + value: all + - uuid: fb065cb052094f1c933cd4429ff84d0e + expression: last(/DeviceVault Device by HTTP/dv.device.failure_reason_code)=2 and last(/DeviceVault Device by HTTP/dv.device.found)=1 and last(/DeviceVault Device by HTTP/dv.device.enabled)=1 and last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.failure_reason_code)<>2 or last(/DeviceVault Device by HTTP/dv.device.found)<>1 or last(/DeviceVault Device by HTTP/dv.device.enabled)<>1 or last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)<>0 + name: 'DeviceVault: backup authentication failing' + priority: AVERAGE + description: The latest backup failed because the device refused the login or enable credentials. + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: security + - tag: group + value: all + - uuid: fd6e4d6fb2734e51a26db1a01ef65abd + expression: '{$DV.DEVICE.STALE.MAX}>0 and last(/DeviceVault Device by HTTP/dv.device.last_success_at)>0 and last(/DeviceVault Device by HTTP/dv.device.last_success_age)>{$DV.DEVICE.STALE.MAX} and last(/DeviceVault Device by HTTP/dv.device.found)=1 and last(/DeviceVault Device by HTTP/dv.device.enabled)=1 and last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)=0' + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: '{$DV.DEVICE.STALE.MAX}=0 or last(/DeviceVault Device by HTTP/dv.device.last_success_age)<={$DV.DEVICE.STALE.MAX} or last(/DeviceVault Device by HTTP/dv.device.found)<>1 or last(/DeviceVault Device by HTTP/dv.device.enabled)<>1 or last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)<>0' + name: 'DeviceVault: no successful backup for {$DV.DEVICE.STALE.MAX}' + priority: WARNING + description: A flat threshold on the age of the last successful backup ({$DV.DEVICE.STALE.MAX}, 0 = off). + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: notice + - tag: group + value: all + graphs: + - uuid: 1cec28292b5d42f68444bfb98be412d8 + name: 'DeviceVault: Success ratio' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.success_ratio_1h + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.success_ratio_24h + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.success_ratio_7d + - uuid: 7ed6c1ea807343d1b74434c108fa0a7a + name: 'DeviceVault: Request to stored time (24h)' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.duration_total_avg + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.duration_total_p50 + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.duration_total_p95 + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.duration_total_max + - uuid: f573a586b441450d9bf961cfca6506c8 + name: 'DeviceVault: Failures by reason (24h)' + type: STACKED + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.failures_connection_24h + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.failures_auth_24h + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.failures_timeout_24h + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.failures_command_24h + - sortorder: '4' + color: FC6EA3 + item: + host: DeviceVault by HTTP + key: dv.failures_verification_24h + - sortorder: '5' + color: 6C59DC + item: + host: DeviceVault by HTTP + key: dv.failures_storage_24h + - sortorder: '6' + color: C7A72D + item: + host: DeviceVault by HTTP + key: dv.failures_integrity_24h + - sortorder: '7' + color: BA2A5D + item: + host: DeviceVault by HTTP + key: dv.failures_push_24h + - sortorder: '8' + color: 0080FF + item: + host: DeviceVault by HTTP + key: dv.failures_orphaned_24h + - sortorder: '9' + color: '808080' + item: + host: DeviceVault by HTTP + key: dv.failures_unknown_24h + - uuid: c93c5351861d412da181216f86e3836f + name: 'DeviceVault: Device freshness' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.devices_overdue + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.devices_never + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.devices_failing_1 + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.devices_failing_3 + - sortorder: '4' + color: FC6EA3 + item: + host: DeviceVault by HTTP + key: dv.devices_failing_5 + - uuid: 166e916cdc594bdb97a1ed86b6da5652 + name: 'DeviceVault: Platform health' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.workers_online + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.workers_offline + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.workers_pending + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.workers_behind + - sortorder: '4' + color: FC6EA3 + item: + host: DeviceVault by HTTP + key: dv.scheduler_leader + - uuid: 5ea75b63f03e45e19a4555fdc8527e85 + name: 'DeviceVault: Last backup duration' + graph_items: + - color: 199C0D + item: + host: DeviceVault Device by HTTP + key: dv.device.last_duration + - sortorder: '1' + color: F63100 + item: + host: DeviceVault Device by HTTP + key: dv.device.last_collect_duration + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault Device by HTTP + key: dv.device.last_store_duration diff --git a/deploy/zabbix/devicevault_zabbix_7.0.yaml b/deploy/zabbix/devicevault_zabbix_7.0.yaml new file mode 100644 index 0000000..6585ba9 --- /dev/null +++ b/deploy/zabbix/devicevault_zabbix_7.0.yaml @@ -0,0 +1,6119 @@ +zabbix_export: + version: '7.0' + template_groups: + - uuid: e6e21a0925d9472b929b6e4039bdc474 + name: Templates/Applications + templates: + - uuid: aa670518836c4d5cb9a3d9772f613cb8 + template: DeviceVault by HTTP + name: DeviceVault by HTTP + description: |- + Backup health of a DeviceVault installation, polled from its read-only monitoring API. + + Link it to one host standing for DeviceVault, monitored by the Zabbix server or a proxy that can reach the web tier. + Set {$DV.URL} (the web address, without /api) and {$DV.TOKEN} (a service account's token with the monitoring scope, + plus monitoring_platform for workers, the scheduler, storage targets and receivers) as global macros, or on the host. + + Generated by DeviceVault from its metric catalogue; see DEPLOYMENT.md, section Monitoring. + groups: + - name: Templates/Applications + items: + - uuid: 7d7afd6b15984fb281a39f1d6c2b15c8 + name: 'DeviceVault: Get summary' + type: HTTP_AGENT + key: dv.summary.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: Estate outcomes, timing, freshness, change, volume, push and self, and platform health with that scope. + timeout: 15s + url: '{$DV.URL}/api/monitoring/summary/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: 68a85968b2444b9fb74b925696c567bc + name: 'DeviceVault: API status' + type: HTTP_AGENT + key: dv.http.status + delay: '{$DV.INTERVAL}' + description: The HTTP status the summary endpoint answers with (200 when the token is accepted). + timeout: 15s + url: '{$DV.URL}/api/monitoring/summary/' + status_codes: '' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + retrieve_mode: HEADERS + preprocessing: + - type: REGEX + parameters: + - ^HTTP\/[\d.]+\s+(\d+) + - \1 + tags: + - tag: component + value: api + triggers: + - uuid: ce94da43a52f45239320610816fcc11b + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + name: 'DeviceVault: monitoring API unreachable or refusing' + opdata: 'HTTP status: {ITEM.LASTVALUE1}' + priority: HIGH + description: 'The summary endpoint did not answer for {$DV.NODATA}, or answered with an error (401 or 403: the token is unknown, revoked, expired, out of its networks or lacks the monitoring scope; 5xx: DeviceVault is failing).' + tags: + - tag: component + value: api + - tag: scope + value: availability + - tag: group + value: all + - uuid: 45f498cf445a4118b28b11978cdfde4b + name: 'DeviceVault: Get device groups' + type: HTTP_AGENT + key: dv.group.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: The device groups list (device-groups/), read by discovery. + timeout: 15s + url: '{$DV.URL}/api/monitoring/device-groups/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: 491432725d07479fa406471c23c82fcb + name: 'DeviceVault: Get workers' + type: HTTP_AGENT + key: dv.worker.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: The workers list (workers/), read by discovery. + timeout: 15s + url: '{$DV.URL}/api/monitoring/workers/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: df9789dd3b3d4467beccf13fa52c06be + name: 'DeviceVault: Get storage targets' + type: HTTP_AGENT + key: dv.target.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: The storage targets list (storage-targets/), read by discovery. + timeout: 15s + url: '{$DV.URL}/api/monitoring/storage-targets/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: df1a9eca7ff34596a7c228b2fdae35d2 + name: 'DeviceVault: Get receivers' + type: HTTP_AGENT + key: dv.receiver.get + delay: '{$DV.INTERVAL}' + history: '0' + trends: '0' + value_type: TEXT + description: The receivers list (receivers/), read by discovery. + timeout: 15s + url: '{$DV.URL}/api/monitoring/receivers/' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + - uuid: bf5d6a49b06a49d68688b2c4335bf0da + name: 'DeviceVault: Backups stored (1h)' + type: DEPENDENT + key: dv.stored_1h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.stored_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: d05ff201b40a4c3e9369ed9013a940a4 + name: 'DeviceVault: Backups failed (1h)' + type: DEPENDENT + key: dv.failed_1h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished failed in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.failed_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: b7441aa109d645caa090f8ac745bf4d4 + name: 'DeviceVault: Backups timed out (1h)' + type: DEPENDENT + key: dv.timed_out_1h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished timed out in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.timed_out_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: b36f5d7941f748069ee4837a0ed95deb + name: 'DeviceVault: Backups finished (1h)' + type: DEPENDENT + key: dv.finished_1h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished in the last 1h, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.finished_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 6de8ac90da2c4ba89bd1160339934112 + name: 'DeviceVault: Success ratio (1h)' + type: DEPENDENT + key: dv.success_ratio_1h + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 1h; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.success_ratio_1h + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 2796bf19e91b419b8a64894e0cb18948 + name: 'DeviceVault: Failure ratio (1h)' + type: DEPENDENT + key: dv.failure_ratio_1h + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.failure_ratio_1h + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 0459709f841d43d2a62fb7135d531760 + name: 'DeviceVault: Backups retried (1h)' + type: DEPENDENT + key: dv.retried_1h + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 1h that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.retried_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: f5e1de1d16d04daea5c8f5ead3d85443 + name: 'DeviceVault: Backups with changes (1h)' + type: DEPENDENT + key: dv.changed_1h + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 1h that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.changed_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: a80e4fa544da4c5fab14d2999d57753e + name: 'DeviceVault: Bytes stored (1h)' + type: DEPENDENT + key: dv.bytes_stored_1h + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.bytes_stored_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: b7b3bf1bd7fc40a8939c466672665919 + name: 'DeviceVault: Lines added (1h)' + type: DEPENDENT + key: dv.lines_added_1h + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_added_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 0a48e19e290245f0a963230cf9fcac34 + name: 'DeviceVault: Lines removed (1h)' + type: DEPENDENT + key: dv.lines_removed_1h + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_removed_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: e742ff9f30cf47bd88dd36a39a3d8abb + name: 'DeviceVault: Lines modified (1h)' + type: DEPENDENT + key: dv.lines_modified_1h + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_modified_1h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: c98c5a1973684dc3abf7c1b758d51557 + name: 'DeviceVault: Backups stored (24h)' + type: DEPENDENT + key: dv.stored_24h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.stored_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: a39dae0c0f354eac8ffbd51b29ff2569 + name: 'DeviceVault: Backups failed (24h)' + type: DEPENDENT + key: dv.failed_24h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished failed in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.failed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: d01a82fdae77489283622305e4988756 + name: 'DeviceVault: Backups timed out (24h)' + type: DEPENDENT + key: dv.timed_out_24h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished timed out in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.timed_out_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 0d0a895c8c314d1b98bf289ad25fec3b + name: 'DeviceVault: Backups finished (24h)' + type: DEPENDENT + key: dv.finished_24h + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished in the last 24h, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.finished_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: ee3eef0c393b4480904f0bdc8e956fe9 + name: 'DeviceVault: Success ratio (24h)' + type: DEPENDENT + key: dv.success_ratio_24h + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 24h; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.success_ratio_24h + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: bff829b7da9842c09db731090d5c44f0 + name: 'DeviceVault: Failure ratio (24h)' + type: DEPENDENT + key: dv.failure_ratio_24h + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.failure_ratio_24h + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 2dd63b91c7e84378b064ddbfc65d8176 + name: 'DeviceVault: Backups retried (24h)' + type: DEPENDENT + key: dv.retried_24h + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 24h that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.retried_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: f034e175cc0a4704b0a7aaafeede4bf4 + name: 'DeviceVault: Backups with changes (24h)' + type: DEPENDENT + key: dv.changed_24h + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 24h that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.changed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 1f83a71fd87a486fb2b0573426596d3a + name: 'DeviceVault: Bytes stored (24h)' + type: DEPENDENT + key: dv.bytes_stored_24h + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.bytes_stored_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: e7e8e91b24cc40379746e71c3ab2edf6 + name: 'DeviceVault: Lines added (24h)' + type: DEPENDENT + key: dv.lines_added_24h + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_added_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: efff3fc1bfae4d91b69dda3341aa019c + name: 'DeviceVault: Lines removed (24h)' + type: DEPENDENT + key: dv.lines_removed_24h + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_removed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 3ad32f07f80d44e0944b69b79357b301 + name: 'DeviceVault: Lines modified (24h)' + type: DEPENDENT + key: dv.lines_modified_24h + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.lines_modified_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: ab59662da07c40faaf06a5b38c381779 + name: 'DeviceVault: Backups stored (7d)' + type: DEPENDENT + key: dv.stored_7d + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.stored_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 11ed5cab803945a48a6093526338b782 + name: 'DeviceVault: Backups failed (7d)' + type: DEPENDENT + key: dv.failed_7d + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished failed in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.failed_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: ac1178a794824faea469ecf148c0854c + name: 'DeviceVault: Backups timed out (7d)' + type: DEPENDENT + key: dv.timed_out_7d + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished timed out in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.timed_out_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 87cda6948b524c00bad3530d5708d3e1 + name: 'DeviceVault: Backups finished (7d)' + type: DEPENDENT + key: dv.finished_7d + delay: '0' + value_type: UNSIGNED + description: Jobs of the estate that finished in the last 7d, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.finished_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: a1c3004eec7d4e628c64759f6fea2921 + name: 'DeviceVault: Success ratio (7d)' + type: DEPENDENT + key: dv.success_ratio_7d + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 7d; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.success_ratio_7d + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 773fb814bc2c41ea98e00fb541b5012f + name: 'DeviceVault: Failure ratio (7d)' + type: DEPENDENT + key: dv.failure_ratio_7d + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.failure_ratio_7d + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: d0f21018cac04045a17e9b5bc82add10 + name: 'DeviceVault: Backups retried (7d)' + type: DEPENDENT + key: dv.retried_7d + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 7d that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.retried_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 47c4e068063d4008864c09a5cce09963 + name: 'DeviceVault: Backups with changes (7d)' + type: DEPENDENT + key: dv.changed_7d + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 7d that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.changed_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 5c305fa9175b4352bd20982566c5d58b + name: 'DeviceVault: Bytes stored (7d)' + type: DEPENDENT + key: dv.bytes_stored_7d + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.bytes_stored_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: c5e046f6dad145418c414e4e18b0a02d + name: 'DeviceVault: Lines added (7d)' + type: DEPENDENT + key: dv.lines_added_7d + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.lines_added_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 4dd08de61f7f4c09b05db1881c979837 + name: 'DeviceVault: Lines removed (7d)' + type: DEPENDENT + key: dv.lines_removed_7d + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.lines_removed_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: 61db668aaae4490f9083571ad2832445 + name: 'DeviceVault: Lines modified (7d)' + type: DEPENDENT + key: dv.lines_modified_7d + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.lines_modified_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: changes + - uuid: f4831046c8574711804c150144dbe906 + name: 'DeviceVault: Backups stored (all time)' + type: DEPENDENT + key: dv.stored_total + delay: '0' + value_type: UNSIGNED + description: Every job of the estate that finished stored, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.stored_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 04bc79f736f54cd8b433c4a611cd4814 + name: 'DeviceVault: Backups failed (all time)' + type: DEPENDENT + key: dv.failed_total + delay: '0' + value_type: UNSIGNED + description: Every job of the estate that finished failed, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.failed_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: d7c6ebcf694d4ea4918f20911e4f5c38 + name: 'DeviceVault: Backups timed out (all time)' + type: DEPENDENT + key: dv.timed_out_total + delay: '0' + value_type: UNSIGNED + description: Every job of the estate that finished timed out, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.timed_out_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: ca8148c3b9cf4c3382d3d4530329f2e0 + name: 'DeviceVault: Failures: connection (24h)' + type: DEPENDENT + key: dv.failures_connection_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "connection" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_connection_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 9a45beed5b544ae298aed1f8d0b8527a + name: 'DeviceVault: Failures: connection (all time)' + type: DEPENDENT + key: dv.failures_connection_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "connection" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_connection_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 2c1c9dcc6276477d89daa1517a4eb889 + name: 'DeviceVault: Failures: auth (24h)' + type: DEPENDENT + key: dv.failures_auth_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "auth" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_auth_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: eb2bbc3549e048c8b554b071fae875d3 + name: 'DeviceVault: Failures: auth (all time)' + type: DEPENDENT + key: dv.failures_auth_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "auth" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_auth_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 4bf5d3a853654634ad0112893a578d00 + name: 'DeviceVault: Failures: timeout (24h)' + type: DEPENDENT + key: dv.failures_timeout_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "timeout" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_timeout_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 6743e88c2a024be181e7695556430897 + name: 'DeviceVault: Failures: timeout (all time)' + type: DEPENDENT + key: dv.failures_timeout_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "timeout" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_timeout_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: ac50b44e87bb4b389a2dc18e1a405023 + name: 'DeviceVault: Failures: command (24h)' + type: DEPENDENT + key: dv.failures_command_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "command" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_command_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 413607c919e340b0a47bb58431439fb6 + name: 'DeviceVault: Failures: command (all time)' + type: DEPENDENT + key: dv.failures_command_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "command" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_command_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: d793f46a07584550a80ae643fb894767 + name: 'DeviceVault: Failures: verification (24h)' + type: DEPENDENT + key: dv.failures_verification_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "verification" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_verification_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 4d58d0c2e3b24c79b554db33dcb929fa + name: 'DeviceVault: Failures: verification (all time)' + type: DEPENDENT + key: dv.failures_verification_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "verification" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_verification_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 3838d117b128435aa43a50acc8f768f6 + name: 'DeviceVault: Failures: storage (24h)' + type: DEPENDENT + key: dv.failures_storage_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "storage" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_storage_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 03b0892403bf45b5838bc8728049e64b + name: 'DeviceVault: Failures: storage (all time)' + type: DEPENDENT + key: dv.failures_storage_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "storage" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_storage_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 47dfc2bad7de492b9c4c00776be2a2b5 + name: 'DeviceVault: Failures: integrity (24h)' + type: DEPENDENT + key: dv.failures_integrity_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "integrity" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_integrity_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 3daacf291cd94d4a895542fd5b2b0f72 + name: 'DeviceVault: Failures: integrity (all time)' + type: DEPENDENT + key: dv.failures_integrity_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "integrity" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_integrity_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 11662cba7b444b9181602ed163d70800 + name: 'DeviceVault: Failures: push (24h)' + type: DEPENDENT + key: dv.failures_push_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "push" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_push_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 32cb4286c36b4a7a9c5fa2a1febca3fa + name: 'DeviceVault: Failures: push (all time)' + type: DEPENDENT + key: dv.failures_push_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "push" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_push_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: a8d3c9b187874159ac4e721bb7dadf47 + name: 'DeviceVault: Failures: orphaned (24h)' + type: DEPENDENT + key: dv.failures_orphaned_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "orphaned" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_orphaned_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: c0fc4ba8f0db4a3fb78cb48afe9d3703 + name: 'DeviceVault: Failures: orphaned (all time)' + type: DEPENDENT + key: dv.failures_orphaned_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "orphaned" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_orphaned_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 2f3c82e212bb44cabb1587500fbaa05a + name: 'DeviceVault: Failures: unknown (24h)' + type: DEPENDENT + key: dv.failures_unknown_24h + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "unknown" + preprocessing: + - type: JSONPATH + parameters: + - $.failures_unknown_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 0f7895ce88d04f0094f67cb4afe72eff + name: 'DeviceVault: Failures: unknown (all time)' + type: DEPENDENT + key: dv.failures_unknown_total + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "unknown" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.failures_unknown_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: failures + - uuid: 0cfa53fd39154a22954d4e77448093b8 + name: 'DeviceVault: Backups queued' + type: DEPENDENT + key: dv.queued + delay: '0' + value_type: UNSIGNED + description: Jobs waiting for a collector now + preprocessing: + - type: JSONPATH + parameters: + - $.queued + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: 8cc9286e82d144b8a9b89678349c1b9f + name: 'DeviceVault: Backups running' + type: DEPENDENT + key: dv.running + delay: '0' + value_type: UNSIGNED + description: Jobs being collected or stored now + preprocessing: + - type: JSONPATH + parameters: + - $.running + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: backups + - uuid: b0cd9f9abde44f7a8a28517ecc437cde + name: 'DeviceVault: Duration of collection, average (24h)' + type: DEPENDENT + key: dv.duration_collect_avg + delay: '0' + value_type: FLOAT + units: s + description: The average collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_collect_avg + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 1d35db9e53a048769868aa3e02f89482 + name: 'DeviceVault: Duration of collection, median (24h)' + type: DEPENDENT + key: dv.duration_collect_p50 + delay: '0' + value_type: FLOAT + units: s + description: The median collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_collect_p50 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 6b74c612653048b0b67d7fd643221e8c + name: 'DeviceVault: Duration of collection, 95th percentile (24h)' + type: DEPENDENT + key: dv.duration_collect_p95 + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_collect_p95 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: fe431e627a4844358469bc2e47846fef + name: 'DeviceVault: Duration of collection, maximum (24h)' + type: DEPENDENT + key: dv.duration_collect_max + delay: '0' + value_type: FLOAT + units: s + description: The maximum collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_collect_max + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: eb48681932be41348117cbddd2ca0ae5 + name: 'DeviceVault: Duration of storage, average (24h)' + type: DEPENDENT + key: dv.duration_store_avg + delay: '0' + value_type: FLOAT + units: s + description: The average storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_store_avg + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: ad6efe14e72f4b98b7701cc901f89fad + name: 'DeviceVault: Duration of storage, median (24h)' + type: DEPENDENT + key: dv.duration_store_p50 + delay: '0' + value_type: FLOAT + units: s + description: The median storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_store_p50 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: b12a616fdf364399827773ade3855d23 + name: 'DeviceVault: Duration of storage, 95th percentile (24h)' + type: DEPENDENT + key: dv.duration_store_p95 + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_store_p95 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: fb29667a9c6d443c9b3a7ea113f536c1 + name: 'DeviceVault: Duration of storage, maximum (24h)' + type: DEPENDENT + key: dv.duration_store_max + delay: '0' + value_type: FLOAT + units: s + description: The maximum storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_store_max + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 3f2a0d3dc461408dbe216b5b2a9176e4 + name: 'DeviceVault: Duration of queue wait, average (24h)' + type: DEPENDENT + key: dv.duration_wait_avg + delay: '0' + value_type: FLOAT + units: s + description: The average queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_wait_avg + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 548cae0d8e2846fb8b56768f4bc08205 + name: 'DeviceVault: Duration of queue wait, median (24h)' + type: DEPENDENT + key: dv.duration_wait_p50 + delay: '0' + value_type: FLOAT + units: s + description: The median queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_wait_p50 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: 84887283402e45cf94f576519bf29f20 + name: 'DeviceVault: Duration of queue wait, 95th percentile (24h)' + type: DEPENDENT + key: dv.duration_wait_p95 + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_wait_p95 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: a55eb0351b364fedb22be7d3875a1f09 + name: 'DeviceVault: Duration of queue wait, maximum (24h)' + type: DEPENDENT + key: dv.duration_wait_max + delay: '0' + value_type: FLOAT + units: s + description: The maximum queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_wait_max + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: a1fafac13440474fb2406c886e516991 + name: 'DeviceVault: Duration of request to stored, average (24h)' + type: DEPENDENT + key: dv.duration_total_avg + delay: '0' + value_type: FLOAT + units: s + description: The average request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_total_avg + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: c031d2a665184f109512c4b1ac6153b3 + name: 'DeviceVault: Duration of request to stored, median (24h)' + type: DEPENDENT + key: dv.duration_total_p50 + delay: '0' + value_type: FLOAT + units: s + description: The median request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_total_p50 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: f579da68fc83427a86dd4999b225c455 + name: 'DeviceVault: Duration of request to stored, 95th percentile (24h)' + type: DEPENDENT + key: dv.duration_total_p95 + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_total_p95 + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: cfb1a08a1f9646888bf4696c81fbf16e + name: 'DeviceVault: Duration of request to stored, maximum (24h)' + type: DEPENDENT + key: dv.duration_total_max + delay: '0' + value_type: FLOAT + units: s + description: The maximum request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.duration_total_max + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: timing + - uuid: da454db68db947988e8579d923f5130b + name: 'DeviceVault: Devices' + type: DEPENDENT + key: dv.devices_total + delay: '0' + value_type: UNSIGNED + description: Active devices + preprocessing: + - type: JSONPATH + parameters: + - $.devices_total + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: e7975daf14454e889458a4ec3ee2460b + name: 'DeviceVault: Devices enabled' + type: DEPENDENT + key: dv.devices_enabled + delay: '0' + value_type: UNSIGNED + description: Active devices that are not paused + preprocessing: + - type: JSONPATH + parameters: + - $.devices_enabled + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: a8d69f8bf46343c69f833e7baf840ec2 + name: 'DeviceVault: Devices disabled' + type: DEPENDENT + key: dv.devices_disabled + delay: '0' + value_type: UNSIGNED + description: Active devices that are paused + preprocessing: + - type: JSONPATH + parameters: + - $.devices_disabled + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 2d2bbbcbcdb34215a7225f606e35da29 + name: 'DeviceVault: Devices inactive' + type: DEPENDENT + key: dv.devices_inactive + delay: '0' + value_type: UNSIGNED + description: Devices marked inactive (retired, held) + preprocessing: + - type: JSONPATH + parameters: + - $.devices_inactive + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 46fa513b9b014aa3b44a49e8c6ab6c61 + name: 'DeviceVault: Devices never backed up' + type: DEPENDENT + key: dv.devices_never + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices with no stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.devices_never + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + triggers: + - uuid: e6a6c08a27f34731a7e03b63ff386f25 + expression: last(/DeviceVault by HTTP/dv.devices_never)>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.devices_never)=0 + name: 'DeviceVault: devices never backed up' + opdata: 'Devices: {ITEM.LASTVALUE1}' + priority: WARNING + description: Active, enabled devices have no stored backup yet. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: freshness + - tag: scope + value: notice + - tag: group + value: all + - uuid: a7bb31e86ed947f3b15df09eb237427d + name: 'DeviceVault: Devices overdue' + type: DEPENDENT + key: dv.devices_overdue + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last success is older than their schedule allows + preprocessing: + - type: JSONPATH + parameters: + - $.devices_overdue + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 42f1d2f48a6141fdbe987841f59f08a5 + name: 'DeviceVault: Devices failing 1+ times in a row' + type: DEPENDENT + key: dv.devices_failing_1 + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 1 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.devices_failing_1 + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: e17dcdfead6e462cb352e781ba65765b + name: 'DeviceVault: Devices failing 3+ times in a row' + type: DEPENDENT + key: dv.devices_failing_3 + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 3 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.devices_failing_3 + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: bd01fc742cfc4f239b8bc5cfc6e1b04e + name: 'DeviceVault: Devices failing 5+ times in a row' + type: DEPENDENT + key: dv.devices_failing_5 + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 5 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.devices_failing_5 + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 51fa145a62b74a5c89dcb94307a8b98a + name: 'DeviceVault: Oldest last success, age' + type: DEPENDENT + key: dv.oldest_success_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since the stalest active, enabled device last backed up (0 with none backed up) + preprocessing: + - type: JSONPATH + parameters: + - $.oldest_success_age + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: e0994af2c3134b129a2dacde6ffc2b47 + name: 'DeviceVault: Oldest last success, device' + type: DEPENDENT + key: dv.oldest_success_device + delay: '0' + value_type: CHAR + trends: '0' + description: Name of that device + preprocessing: + - type: JSONPATH + parameters: + - $.oldest_success_device + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.summary.get + tags: + - tag: component + value: freshness + - uuid: 0f577e1812b74f94a6d35c40d15a53c8 + name: 'DeviceVault: Backups held' + type: DEPENDENT + key: dv.backups_held + delay: '0' + value_type: UNSIGNED + description: Stored backups not pruned by retention + preprocessing: + - type: JSONPATH + parameters: + - $.backups_held + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: e665b41c4f6f44bd85baec23db678d7f + name: 'DeviceVault: Bytes held' + type: DEPENDENT + key: dv.bytes_held + delay: '0' + value_type: UNSIGNED + units: B + description: Size of those backups (as the collectors reported it) + preprocessing: + - type: JSONPATH + parameters: + - $.bytes_held + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: volume + - uuid: a5acd80391b7405d8aa08843414e9773 + name: 'DeviceVault: Pushes accepted (24h)' + type: DEPENDENT + key: dv.push_accepted_24h + delay: '0' + value_type: UNSIGNED + description: Files devices pushed in the last 24h that became backups + preprocessing: + - type: JSONPATH + parameters: + - $.push_accepted_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 7fea1bc7dcc846b5a97e654b458aefb0 + name: 'DeviceVault: Pushes refused (24h)' + type: DEPENDENT + key: dv.push_refused_24h + delay: '0' + value_type: UNSIGNED + description: Refused pushes in the last 24h (receivers rate-limit reports, so a lower bound) + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + triggers: + - uuid: ce2584cdbefe455983a4d3f18d8738ff + expression: last(/DeviceVault by HTTP/dv.push_refused_24h)-last(/DeviceVault by HTTP/dv.push_refused_24h,#1:now-1h)>{$DV.PUSH.REFUSED.MAX} + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.push_refused_24h)-last(/DeviceVault by HTTP/dv.push_refused_24h,#1:now-1h)<={$DV.PUSH.REFUSED.MAX} + name: 'DeviceVault: push refusals spiking' + priority: WARNING + description: Receivers refused more than {$DV.PUSH.REFUSED.MAX} pushes in the last hour (unknown sources, failed authentication, wrong protocol...). See the Push refusals page. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: push + - tag: scope + value: security + - tag: group + value: all + - uuid: 03f1a943066b4130ba14334fc5a61c78 + name: 'DeviceVault: Pushes refused: unknown_source (24h)' + type: DEPENDENT + key: dv.push_refused_unknown_source_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "unknown_source" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_unknown_source_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: e153c62f08d949b297bf746562184c33 + name: 'DeviceVault: Pushes refused: auth_failed (24h)' + type: DEPENDENT + key: dv.push_refused_auth_failed_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "auth_failed" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_auth_failed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 767cd5dbc43b490381bf0b4602af9b05 + name: 'DeviceVault: Pushes refused: wrong_protocol (24h)' + type: DEPENDENT + key: dv.push_refused_wrong_protocol_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "wrong_protocol" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_wrong_protocol_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 2254e228de7748a3903285b63b9c2d88 + name: 'DeviceVault: Pushes refused: filename (24h)' + type: DEPENDENT + key: dv.push_refused_filename_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "filename" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_filename_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: cf848a92b86d4b78bd47698a0e3a6e6f + name: 'DeviceVault: Pushes refused: too_large (24h)' + type: DEPENDENT + key: dv.push_refused_too_large_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "too_large" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_too_large_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: b9442c3e887e491ca2115b86c5bf33dd + name: 'DeviceVault: Pushes refused: device_changed (24h)' + type: DEPENDENT + key: dv.push_refused_device_changed_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "device_changed" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_device_changed_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: ddb3dce602414bb3befb08c2be70a172 + name: 'DeviceVault: Pushes refused: error (24h)' + type: DEPENDENT + key: dv.push_refused_error_24h + delay: '0' + value_type: UNSIGNED + description: Pushes refused in the last 24h for reason "error" + preprocessing: + - type: JSONPATH + parameters: + - $.push_refused_error_24h + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 18b15115c27a42408b5a6735c4aed57b + name: 'DeviceVault: Last push' + type: DEPENDENT + key: dv.push_last_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When a device last pushed a file that was accepted (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.push_last_at + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: a0e54c7c2eec4f8988b9fac3a078ed51 + name: 'DeviceVault: Push devices silent (7d)' + type: DEPENDENT + key: dv.push_silent_7d + delay: '0' + value_type: UNSIGNED + description: Active, enabled push devices with no accepted push in the last 7 days + preprocessing: + - type: JSONPATH + parameters: + - $.push_silent_7d + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: push + - uuid: 67f49bedceed4741a898ba9ff7de7ac4 + name: 'DeviceVault: DeviceVault version' + type: DEPENDENT + key: dv.version + delay: '0' + value_type: CHAR + trends: '0' + description: The orchestrator version + preprocessing: + - type: JSONPATH + parameters: + - $.version + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + - uuid: cad58f9a198c4a5baf52fc9836edbc0e + name: 'DeviceVault: Payload encryption' + type: DEPENDENT + key: dv.payload_encryption + delay: '0' + value_type: UNSIGNED + description: Whether backups and secrets sent to workers are encrypted (DV_PAYLOAD_ENCRYPTION) + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.payload_encryption + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + - uuid: 735c6ab5d700422db048c7b21d03ffaa + name: 'DeviceVault: Server time' + type: DEPENDENT + key: dv.server_time + delay: '0' + value_type: UNSIGNED + units: unixtime + description: The orchestrator clock, to compare with the monitoring server (clock skew) + preprocessing: + - type: JSONPATH + parameters: + - $.server_time + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + triggers: + - uuid: c69620d17b864ce4a17647de6be22acd + expression: fuzzytime(/DeviceVault by HTTP/dv.server_time,{$DV.SKEW.MAX})=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: fuzzytime(/DeviceVault by HTTP/dv.server_time,{$DV.SKEW.MAX})=1 + name: 'DeviceVault: clock differs from Zabbix by more than {$DV.SKEW.MAX}s' + priority: WARNING + description: The orchestrator's clock and the Zabbix server's differ; schedules and ages will be off. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: system + - tag: scope + value: availability + - tag: group + value: all + - uuid: d96a16c28a2441ec936b0c670effa8af + name: 'DeviceVault: Figures computed at' + type: DEPENDENT + key: dv.computed_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When these figures were computed (they are cached briefly) + preprocessing: + - type: JSONPATH + parameters: + - $.computed_at + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + - uuid: 459e6977d31e4402b01ec5804459c1cf + name: 'DeviceVault: Figures age' + type: DEPENDENT + key: dv.computed_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since these figures were computed + preprocessing: + - type: JSONPATH + parameters: + - $.computed_age + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: system + - uuid: e0907f452d0a46f7ada01137b04d5b50 + name: 'DeviceVault: Workers online' + type: DEPENDENT + key: dv.workers_online + delay: '0' + value_type: UNSIGNED + description: Approved workers that sent a heartbeat recently + preprocessing: + - type: JSONPATH + parameters: + - $.workers_online + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 70ed2e5d33764a56ac15724f464a2cc9 + name: 'DeviceVault: Workers offline' + type: DEPENDENT + key: dv.workers_offline + delay: '0' + value_type: UNSIGNED + description: Approved workers with no recent heartbeat + preprocessing: + - type: JSONPATH + parameters: + - $.workers_offline + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 1a8a7f9863fb4fc0a76877cf561d46ee + name: 'DeviceVault: Workers pending approval' + type: DEPENDENT + key: dv.workers_pending + delay: '0' + value_type: UNSIGNED + description: Workers waiting for approval + preprocessing: + - type: JSONPATH + parameters: + - $.workers_pending + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 88c0d3bb8ee44636972ed108a892ae49 + name: 'DeviceVault: Workers revoked' + type: DEPENDENT + key: dv.workers_revoked + delay: '0' + value_type: UNSIGNED + description: Revoked workers + preprocessing: + - type: JSONPATH + parameters: + - $.workers_revoked + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 145842d376a74619b442f1776cdf1742 + name: 'DeviceVault: Workers behind on their assignment' + type: DEPENDENT + key: dv.workers_behind + delay: '0' + value_type: UNSIGNED + description: Approved workers whose applied assignment version is older than the current one + preprocessing: + - type: JSONPATH + parameters: + - $.workers_behind + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: f98ee35e560a4f11bc87ce34a4801b6b + name: 'DeviceVault: Scheduler has a leader' + type: DEPENDENT + key: dv.scheduler_leader + delay: '0' + value_type: UNSIGNED + description: Whether a scheduler node holds the lease + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.scheduler_leader + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + triggers: + - uuid: 08ce456e0dbf42dfab0272a8e615dffc + expression: last(/DeviceVault by HTTP/dv.scheduler_leader)=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.scheduler_leader)=1 + name: 'DeviceVault: scheduler has no leader' + priority: HIGH + description: No scheduler node holds the lease, so no scheduled backup starts. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: scheduler + - tag: scope + value: availability + - tag: group + value: all + - uuid: b56f370402974734b2c398f97e7b4372 + name: 'DeviceVault: Scheduler lease remaining' + type: DEPENDENT + key: dv.scheduler_lease_ttl + delay: '0' + value_type: FLOAT + units: s + description: Seconds left on the leader's lease (0 without a leader) + preprocessing: + - type: JSONPATH + parameters: + - $.scheduler_lease_ttl + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + - uuid: 4ab08602bc5e4ab28b73ed9449946919 + name: 'DeviceVault: Scheduler last tick, age' + type: DEPENDENT + key: dv.scheduler_tick_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since the leader last ran its tick (-1 if it never has) + preprocessing: + - type: JSONPATH + parameters: + - $.scheduler_tick_age + error_handler: DISCARD_VALUE + master_item: + key: dv.summary.get + tags: + - tag: component + value: platform + triggers: + - uuid: 0ba886097e6a4a2baac33c52bbde0969 + expression: last(/DeviceVault by HTTP/dv.scheduler_tick_age)>2m or last(/DeviceVault by HTTP/dv.scheduler_tick_age)<0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.scheduler_tick_age)>=0 and last(/DeviceVault by HTTP/dv.scheduler_tick_age)<=2m + name: 'DeviceVault: scheduler is not ticking' + priority: AVERAGE + description: The scheduler leader has not run its tick for over 2 minutes. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + - name: 'DeviceVault: scheduler has no leader' + expression: last(/DeviceVault by HTTP/dv.scheduler_leader)=0 + recovery_expression: last(/DeviceVault by HTTP/dv.scheduler_leader)=1 + tags: + - tag: component + value: scheduler + - tag: scope + value: availability + - tag: group + value: all + discovery_rules: + - uuid: 8ee5ed976cd24231b5aa9df6a263a488 + name: 'DeviceVault: Device groups discovery' + type: DEPENDENT + key: dv.group.discovery + delay: '0' + lifetime: 7d + description: Device groups from device-groups/; resources gone for 7 days are deleted. + item_prototypes: + - uuid: 4a1743a9f04b4189b51f9b7c67bcb040 + name: 'Group {#NAME}: Backups stored (1h)' + type: DEPENDENT + key: dv.group.stored_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].stored_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 2ed654c82eac4450a1b2c4c02a44799d + name: 'Group {#NAME}: Backups failed (1h)' + type: DEPENDENT + key: dv.group.failed_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished failed in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failed_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: b869fe46ea79420980eadfe98d853b5c + name: 'Group {#NAME}: Backups timed out (1h)' + type: DEPENDENT + key: dv.group.timed_out_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished timed out in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].timed_out_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 51ee81aa30304f88a44dfdd6757f1464 + name: 'Group {#NAME}: Backups finished (1h)' + type: DEPENDENT + key: dv.group.finished_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished in the last 1h, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].finished_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 48c25c537c584907856b846208fe0c1a + name: 'Group {#NAME}: Success ratio (1h)' + type: DEPENDENT + key: dv.group.success_ratio_1h[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 1h; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].success_ratio_1h.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 351b05bebd3c4163983df610d31e0497 + name: 'Group {#NAME}: Failure ratio (1h)' + type: DEPENDENT + key: dv.group.failure_ratio_1h[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failure_ratio_1h.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: d87c52054bdc4ddfbf4d0d2ec9c3d35a + name: 'Group {#NAME}: Backups retried (1h)' + type: DEPENDENT + key: dv.group.retried_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 1h that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].retried_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 6f2a8663787e4ac797c156ea763c5f26 + name: 'Group {#NAME}: Backups with changes (1h)' + type: DEPENDENT + key: dv.group.changed_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 1h that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].changed_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 117017daab604ad7ab378d85f5bd0ad1 + name: 'Group {#NAME}: Bytes stored (1h)' + type: DEPENDENT + key: dv.group.bytes_stored_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_stored_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + - uuid: ba3f06adaaa046aa971e9ad56dda3fbc + name: 'Group {#NAME}: Lines added (1h)' + type: DEPENDENT + key: dv.group.lines_added_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_added_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 88a3c5b2665541fe972d246e773d0869 + name: 'Group {#NAME}: Lines removed (1h)' + type: DEPENDENT + key: dv.group.lines_removed_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_removed_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 756ca5ce409a41f78acdad1ba7250cfb + name: 'Group {#NAME}: Lines modified (1h)' + type: DEPENDENT + key: dv.group.lines_modified_1h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 1h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_modified_1h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: abd8cd753fc646499ff15120b9a422e6 + name: 'Group {#NAME}: Backups stored (24h)' + type: DEPENDENT + key: dv.group.stored_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].stored_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 535fbd84bdd646548321735f8ac4c666 + name: 'Group {#NAME}: Backups failed (24h)' + type: DEPENDENT + key: dv.group.failed_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished failed in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failed_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 372673b6afeb4fd9b5287097bf0b27d3 + name: 'Group {#NAME}: Backups timed out (24h)' + type: DEPENDENT + key: dv.group.timed_out_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished timed out in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].timed_out_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 49fada23260244ee881849417c6e0d48 + name: 'Group {#NAME}: Backups finished (24h)' + type: DEPENDENT + key: dv.group.finished_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished in the last 24h, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].finished_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: a8943a7fb4b0431e8fae7f7da3f8fedf + name: 'Group {#NAME}: Success ratio (24h)' + type: DEPENDENT + key: dv.group.success_ratio_24h[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 24h; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].success_ratio_24h.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: e21cb82adad74ff197e492786fcc139f + name: 'Group {#NAME}: Failure ratio (24h)' + type: DEPENDENT + key: dv.group.failure_ratio_24h[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failure_ratio_24h.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: fcdc21a78c8f42fdb5ec1e086bfd223d + name: 'Group {#NAME}: Backups retried (24h)' + type: DEPENDENT + key: dv.group.retried_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 24h that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].retried_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 5b71e8727ce644e9b75f93b471ee1bce + name: 'Group {#NAME}: Backups with changes (24h)' + type: DEPENDENT + key: dv.group.changed_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 24h that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].changed_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: dbd8934338114dd592dc9324962fc3c5 + name: 'Group {#NAME}: Bytes stored (24h)' + type: DEPENDENT + key: dv.group.bytes_stored_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_stored_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + - uuid: 5af8b515f70f451a9d1a919bfbe4d9ba + name: 'Group {#NAME}: Lines added (24h)' + type: DEPENDENT + key: dv.group.lines_added_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_added_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 7262dfbec4de4246bdc2442ace00ea11 + name: 'Group {#NAME}: Lines removed (24h)' + type: DEPENDENT + key: dv.group.lines_removed_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_removed_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: b6456fc4cb2841bb88b93c6318c34ed9 + name: 'Group {#NAME}: Lines modified (24h)' + type: DEPENDENT + key: dv.group.lines_modified_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_modified_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: e006e153a9c24b58980bc73dae785d58 + name: 'Group {#NAME}: Backups stored (7d)' + type: DEPENDENT + key: dv.group.stored_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].stored_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 280948d4a30f4c499ad8a3be9c5fca45 + name: 'Group {#NAME}: Backups failed (7d)' + type: DEPENDENT + key: dv.group.failed_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished failed in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failed_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 7d35f9af7edc402baa1e8dfa431d6edd + name: 'Group {#NAME}: Backups timed out (7d)' + type: DEPENDENT + key: dv.group.timed_out_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished timed out in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].timed_out_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 51a45579fb5d4ea6819857e7c25c63b1 + name: 'Group {#NAME}: Backups finished (7d)' + type: DEPENDENT + key: dv.group.finished_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs of the device group that finished in the last 7d, whatever the outcome + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].finished_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: db511c28c41c4f3bbd22fc746dfc4090 + name: 'Group {#NAME}: Success ratio (7d)' + type: DEPENDENT + key: dv.group.success_ratio_7d[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 7d; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].success_ratio_7d.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 6f8ed993f11b4fdd9beecec26509428d + name: 'Group {#NAME}: Failure ratio (7d)' + type: DEPENDENT + key: dv.group.failure_ratio_7d[{#ID}] + delay: '0' + value_type: FLOAT + units: '%' + description: Failed or timed out as a share of finished jobs in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failure_ratio_7d.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: a8d524bfa90b453fbf30c35f9936cb9b + name: 'Group {#NAME}: Backups retried (7d)' + type: DEPENDENT + key: dv.group.retried_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs created in the last 7d that needed more than one attempt + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].retried_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: df916b4a2a0146888d9fd7d8e6727ddb + name: 'Group {#NAME}: Backups with changes (7d)' + type: DEPENDENT + key: dv.group.changed_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups in the last 7d that differ from the one before + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].changed_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 9d36a29f9177405cbce8d326f78620d7 + name: 'Group {#NAME}: Bytes stored (7d)' + type: DEPENDENT + key: dv.group.bytes_stored_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_stored_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + - uuid: 095bc292c2f84aef84c3efcd4f5ebf05 + name: 'Group {#NAME}: Lines added (7d)' + type: DEPENDENT + key: dv.group.lines_added_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines added by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_added_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: 0cf3fe47de744be3b9b14841c06bd54a + name: 'Group {#NAME}: Lines removed (7d)' + type: DEPENDENT + key: dv.group.lines_removed_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines removed by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_removed_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: e8f9024e2a9d4c52b46ebf4fde49b578 + name: 'Group {#NAME}: Lines modified (7d)' + type: DEPENDENT + key: dv.group.lines_modified_7d[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Configuration lines modified by the backups stored in the last 7d + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].lines_modified_7d.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: changes + - tag: group + value: '{#NAME}' + - uuid: a1c4874145ff48a29e92606df214b84c + name: 'Group {#NAME}: Backups stored (all time)' + type: DEPENDENT + key: dv.group.stored_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every job of the device group that finished stored, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].stored_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 50f3a982eaab4bc487588fc87d0fdbb3 + name: 'Group {#NAME}: Backups failed (all time)' + type: DEPENDENT + key: dv.group.failed_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every job of the device group that finished failed, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failed_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 7f3ec882d62e4e2f9416cf950553ea88 + name: 'Group {#NAME}: Backups timed out (all time)' + type: DEPENDENT + key: dv.group.timed_out_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every job of the device group that finished timed out, while its row exists + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].timed_out_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 4f08a3e8e0d74b96a7dd24a39e332b49 + name: 'Group {#NAME}: Failures: connection (24h)' + type: DEPENDENT + key: dv.group.failures_connection_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "connection" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_connection_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 879984579275444592ca94183297c357 + name: 'Group {#NAME}: Failures: connection (all time)' + type: DEPENDENT + key: dv.group.failures_connection_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "connection" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_connection_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 2306998cc35a40c89c84feb86fc122b5 + name: 'Group {#NAME}: Failures: auth (24h)' + type: DEPENDENT + key: dv.group.failures_auth_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "auth" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_auth_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 5f12e0ec836946aba11b3434062d4ebd + name: 'Group {#NAME}: Failures: auth (all time)' + type: DEPENDENT + key: dv.group.failures_auth_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "auth" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_auth_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 8384eb8ba2714385958276aa9dbf90a9 + name: 'Group {#NAME}: Failures: timeout (24h)' + type: DEPENDENT + key: dv.group.failures_timeout_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "timeout" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_timeout_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 8c7bd16d1c994c41b8429b39ac571694 + name: 'Group {#NAME}: Failures: timeout (all time)' + type: DEPENDENT + key: dv.group.failures_timeout_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "timeout" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_timeout_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: f557e4ac67a24a41b8e59c352dc517fb + name: 'Group {#NAME}: Failures: command (24h)' + type: DEPENDENT + key: dv.group.failures_command_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "command" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_command_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 43653f2190f64d0eb8d91d52c8a98e94 + name: 'Group {#NAME}: Failures: command (all time)' + type: DEPENDENT + key: dv.group.failures_command_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "command" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_command_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: aaf8e8d28cf9461298c54b2ff0c62b44 + name: 'Group {#NAME}: Failures: verification (24h)' + type: DEPENDENT + key: dv.group.failures_verification_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "verification" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_verification_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: d00f86ac2d154282be9a601b842ae2d2 + name: 'Group {#NAME}: Failures: verification (all time)' + type: DEPENDENT + key: dv.group.failures_verification_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "verification" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_verification_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: e33934d57a3d48f6a54594f61e83ef58 + name: 'Group {#NAME}: Failures: storage (24h)' + type: DEPENDENT + key: dv.group.failures_storage_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "storage" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_storage_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 172085b4d3ae4280b03a2b88371f9191 + name: 'Group {#NAME}: Failures: storage (all time)' + type: DEPENDENT + key: dv.group.failures_storage_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "storage" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_storage_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 167d76c2d55b49138f0cba38f6d39e08 + name: 'Group {#NAME}: Failures: integrity (24h)' + type: DEPENDENT + key: dv.group.failures_integrity_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "integrity" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_integrity_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: e35e9bb3cd304b7bb7826563e92e429f + name: 'Group {#NAME}: Failures: integrity (all time)' + type: DEPENDENT + key: dv.group.failures_integrity_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "integrity" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_integrity_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 060485436b3744069abb69ba8c0775f0 + name: 'Group {#NAME}: Failures: push (24h)' + type: DEPENDENT + key: dv.group.failures_push_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "push" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_push_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 91421731449b4be4a10ead71b101808c + name: 'Group {#NAME}: Failures: push (all time)' + type: DEPENDENT + key: dv.group.failures_push_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "push" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_push_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 1cc1c70a147747ad9e51e2bf067b2618 + name: 'Group {#NAME}: Failures: orphaned (24h)' + type: DEPENDENT + key: dv.group.failures_orphaned_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "orphaned" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_orphaned_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 50f3e45fe3bf4fd4b57cd58f535570c3 + name: 'Group {#NAME}: Failures: orphaned (all time)' + type: DEPENDENT + key: dv.group.failures_orphaned_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "orphaned" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_orphaned_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 3f88976874c641f8b98b85e0431b1926 + name: 'Group {#NAME}: Failures: unknown (24h)' + type: DEPENDENT + key: dv.group.failures_unknown_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs that failed in the last 24h for reason "unknown" + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_unknown_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 691354533cfc4c9ba347b2a30ee6c8e3 + name: 'Group {#NAME}: Failures: unknown (all time)' + type: DEPENDENT + key: dv.group.failures_unknown_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Every failed job with reason "unknown" (older failures without a reason count as unknown) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].failures_unknown_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: failures + - tag: group + value: '{#NAME}' + - uuid: 92332234b7104d6d8c406d012fd168d5 + name: 'Group {#NAME}: Backups queued' + type: DEPENDENT + key: dv.group.queued[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs waiting for a collector now + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].queued.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: 1233d8e96cf4479383800b9e69f16dbe + name: 'Group {#NAME}: Backups running' + type: DEPENDENT + key: dv.group.running[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs being collected or stored now + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].running.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: backups + - tag: group + value: '{#NAME}' + - uuid: ef3f8ffa77884bceaebdc996c819a6bc + name: 'Group {#NAME}: Duration of collection, average (24h)' + type: DEPENDENT + key: dv.group.duration_collect_avg[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The average collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_collect_avg.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: d03c4fc0cb284c109c7fe1a3ee1118b0 + name: 'Group {#NAME}: Duration of collection, median (24h)' + type: DEPENDENT + key: dv.group.duration_collect_p50[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The median collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_collect_p50.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 4553c05d602a4cb1bb1ba145b34ddd43 + name: 'Group {#NAME}: Duration of collection, 95th percentile (24h)' + type: DEPENDENT + key: dv.group.duration_collect_p95[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_collect_p95.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: e752315be6954eab88d2194825179283 + name: 'Group {#NAME}: Duration of collection, maximum (24h)' + type: DEPENDENT + key: dv.group.duration_collect_max[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The maximum collection time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_collect_max.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 58311ef021a5495286a58fbda935053e + name: 'Group {#NAME}: Duration of storage, average (24h)' + type: DEPENDENT + key: dv.group.duration_store_avg[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The average storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_store_avg.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 23070bed8d104c8b9ab6b224bd8ec6af + name: 'Group {#NAME}: Duration of storage, median (24h)' + type: DEPENDENT + key: dv.group.duration_store_p50[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The median storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_store_p50.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 37b51cbf9cb7424a8a5d65807bb5b18c + name: 'Group {#NAME}: Duration of storage, 95th percentile (24h)' + type: DEPENDENT + key: dv.group.duration_store_p95[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_store_p95.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: aab4ce82a85e41d6afbfe5d8d3780a23 + name: 'Group {#NAME}: Duration of storage, maximum (24h)' + type: DEPENDENT + key: dv.group.duration_store_max[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The maximum storage time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_store_max.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 8343582210d34439a2a3be8790f513fd + name: 'Group {#NAME}: Duration of queue wait, average (24h)' + type: DEPENDENT + key: dv.group.duration_wait_avg[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The average queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_wait_avg.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: de72ebd870aa42458dd10f8da8912531 + name: 'Group {#NAME}: Duration of queue wait, median (24h)' + type: DEPENDENT + key: dv.group.duration_wait_p50[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The median queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_wait_p50.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: b7a180d088e3444ba2ddb2ee1421e718 + name: 'Group {#NAME}: Duration of queue wait, 95th percentile (24h)' + type: DEPENDENT + key: dv.group.duration_wait_p95[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_wait_p95.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 74983e6b6f0041a890dfdbf3eed770d9 + name: 'Group {#NAME}: Duration of queue wait, maximum (24h)' + type: DEPENDENT + key: dv.group.duration_wait_max[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The maximum queue wait time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_wait_max.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 06d0154fa5b946b1add2b628392bce2b + name: 'Group {#NAME}: Duration of request to stored, average (24h)' + type: DEPENDENT + key: dv.group.duration_total_avg[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The average request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_total_avg.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 09633c4fc9dd4c169bb5688bc0a3c24c + name: 'Group {#NAME}: Duration of request to stored, median (24h)' + type: DEPENDENT + key: dv.group.duration_total_p50[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The median request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_total_p50.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 8f9b2f8294e445b28783167c2e7d232f + name: 'Group {#NAME}: Duration of request to stored, 95th percentile (24h)' + type: DEPENDENT + key: dv.group.duration_total_p95[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The 95th percentile request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_total_p95.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 57eb899d55924ff2870fef96f6630361 + name: 'Group {#NAME}: Duration of request to stored, maximum (24h)' + type: DEPENDENT + key: dv.group.duration_total_max[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: The maximum request to stored time of the jobs that finished in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].duration_total_max.first() + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: timing + - tag: group + value: '{#NAME}' + - uuid: 90e8afac4b0d4096bf681ca76594280d + name: 'Group {#NAME}: Devices' + type: DEPENDENT + key: dv.group.devices_total[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active devices + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_total.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: b9c0c4d705df4853af9442714eb0547d + name: 'Group {#NAME}: Devices enabled' + type: DEPENDENT + key: dv.group.devices_enabled[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active devices that are not paused + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_enabled.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 7bcb7ede9459410a8bc51589a6354c19 + name: 'Group {#NAME}: Devices disabled' + type: DEPENDENT + key: dv.group.devices_disabled[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active devices that are paused + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_disabled.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: b70e01993cf94ced98a94172e3e4e326 + name: 'Group {#NAME}: Devices inactive' + type: DEPENDENT + key: dv.group.devices_inactive[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Devices marked inactive (retired, held) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_inactive.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: a4ad4efe2ff64bf285acea75e8b8f4d1 + name: 'Group {#NAME}: Devices never backed up' + type: DEPENDENT + key: dv.group.devices_never[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices with no stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_never.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 83889d57f3e843fbb88dabda15f8a869 + name: 'Group {#NAME}: Devices overdue' + type: DEPENDENT + key: dv.group.devices_overdue[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last success is older than their schedule allows + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_overdue.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + trigger_prototypes: + - uuid: ef999962267a453c93c7d59f33aa149a + expression: last(/DeviceVault by HTTP/dv.group.devices_overdue[{#ID}])>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.group.devices_overdue[{#ID}])=0 + name: 'DeviceVault: group {#NAME}: devices overdue' + opdata: 'Devices: {ITEM.LASTVALUE1}' + priority: WARNING + description: Devices of the group have no successful backup as recent as their schedule allows. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: freshness + - tag: scope + value: notice + - tag: group + value: '{#NAME}' + - uuid: a5b69d65d2864a0494dad950e0ad8a41 + name: 'Group {#NAME}: Devices failing 1+ times in a row' + type: DEPENDENT + key: dv.group.devices_failing_1[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 1 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_failing_1.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 7bcf29776f354fff99ff8962538f70c3 + name: 'Group {#NAME}: Devices failing 3+ times in a row' + type: DEPENDENT + key: dv.group.devices_failing_3[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 3 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_failing_3.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: b882b69d7599467faafc9638b0557742 + name: 'Group {#NAME}: Devices failing 5+ times in a row' + type: DEPENDENT + key: dv.group.devices_failing_5[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Active, enabled devices whose last 5 or more finished jobs all failed + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].devices_failing_5.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 3be22af8cf43425c9ebce3757bc9fd6f + name: 'Group {#NAME}: Oldest last success, age' + type: DEPENDENT + key: dv.group.oldest_success_age[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: Seconds since the stalest active, enabled device last backed up (0 with none backed up) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].oldest_success_age.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + trigger_prototypes: + - uuid: b5ed63d32cbb4908aa87101d1bf517b6 + expression: last(/DeviceVault by HTTP/dv.group.oldest_success_age[{#ID}])>{$DV.STALE.MAX} + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.group.oldest_success_age[{#ID}])<={$DV.STALE.MAX} + name: 'DeviceVault: group {#NAME}: a device has not backed up for over {$DV.STALE.MAX}' + opdata: 'Age: {ITEM.LASTVALUE1}' + priority: AVERAGE + description: The stalest active, enabled device of the group last backed up longer ago than {$DV.STALE.MAX}. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: freshness + - tag: scope + value: notice + - tag: group + value: '{#NAME}' + - uuid: a43b52b8c5574bb6abb61ac006f6bfa7 + name: 'Group {#NAME}: Oldest last success, device' + type: DEPENDENT + key: dv.group.oldest_success_device[{#ID}] + delay: '0' + value_type: CHAR + trends: '0' + description: Name of that device + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].oldest_success_device.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.group.get + tags: + - tag: component + value: freshness + - tag: group + value: '{#NAME}' + - uuid: 59084e7b2dd444fa830cbfd5f324e0e2 + name: 'Group {#NAME}: Backups held' + type: DEPENDENT + key: dv.group.backups_held[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups not pruned by retention + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].backups_held.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + - uuid: 326d0c0421094876b09aafe5e8a8f686 + name: 'Group {#NAME}: Bytes held' + type: DEPENDENT + key: dv.group.bytes_held[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of those backups (as the collectors reported it) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_held.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.group.get + tags: + - tag: component + value: volume + - tag: group + value: '{#NAME}' + trigger_prototypes: + - uuid: 5f59b736655e4699aeef3e108e377c3b + expression: last(/DeviceVault by HTTP/dv.group.success_ratio_24h[{#ID}])<{$DV.SUCCESS.MIN} and last(/DeviceVault by HTTP/dv.group.finished_24h[{#ID}])>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.group.success_ratio_24h[{#ID}])>={$DV.SUCCESS.MIN} or last(/DeviceVault by HTTP/dv.group.finished_24h[{#ID}])=0 + name: 'DeviceVault: group {#NAME}: success ratio below {$DV.SUCCESS.MIN}% (24h)' + opdata: 'Success ratio: {ITEM.LASTVALUE1}' + priority: WARNING + description: Fewer backups than {$DV.SUCCESS.MIN}% of those that finished in the last 24h were stored. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: backups + - tag: scope + value: notice + - tag: group + value: '{#NAME}' + - uuid: 772a2579e246490ba7fcbebd9d5d2e96 + expression: ({$DV.FAIL.CONSEC}<3 and last(/DeviceVault by HTTP/dv.group.devices_failing_1[{#ID}])>0) or ({$DV.FAIL.CONSEC}>=3 and {$DV.FAIL.CONSEC}<5 and last(/DeviceVault by HTTP/dv.group.devices_failing_3[{#ID}])>0) or ({$DV.FAIL.CONSEC}>=5 and last(/DeviceVault by HTTP/dv.group.devices_failing_5[{#ID}])>0) + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: ({$DV.FAIL.CONSEC}<3 and last(/DeviceVault by HTTP/dv.group.devices_failing_1[{#ID}])=0) or ({$DV.FAIL.CONSEC}>=3 and {$DV.FAIL.CONSEC}<5 and last(/DeviceVault by HTTP/dv.group.devices_failing_3[{#ID}])=0) or ({$DV.FAIL.CONSEC}>=5 and last(/DeviceVault by HTTP/dv.group.devices_failing_5[{#ID}])=0) + name: 'DeviceVault: group {#NAME}: devices failing {$DV.FAIL.CONSEC}+ times in a row' + priority: WARNING + description: Devices of the group failed their last {$DV.FAIL.CONSEC} or more backups (counted at 1, 3 or 5). + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: failures + - tag: scope + value: notice + - tag: group + value: '{#NAME}' + graph_prototypes: + - uuid: b15469bb49da47abb0105cf87e3d4008 + name: 'Group {#NAME}: Device freshness' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.group.devices_overdue[{#ID}] + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.group.devices_never[{#ID}] + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.group.devices_failing_1[{#ID}] + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.group.devices_failing_3[{#ID}] + - sortorder: '4' + color: FC6EA3 + item: + host: DeviceVault by HTTP + key: dv.group.devices_failing_5[{#ID}] + - uuid: 07225627607e4585bf2edd94dd42be74 + name: 'Group {#NAME}: Success ratio' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.group.success_ratio_1h[{#ID}] + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.group.success_ratio_24h[{#ID}] + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.group.success_ratio_7d[{#ID}] + master_item: + key: dv.group.get + lld_macro_paths: + - lld_macro: '{#ID}' + path: $.id + - lld_macro: '{#NAME}' + path: $.name + preprocessing: + - type: JSONPATH + parameters: + - $.data + - uuid: bae2bee53cb14a42beb2570a01fa4bc2 + name: 'DeviceVault: Workers discovery' + type: DEPENDENT + key: dv.worker.discovery + delay: '0' + lifetime: 7d + description: Workers from workers/; resources gone for 7 days are deleted. + item_prototypes: + - uuid: e955765ba1224f88ad43a32a28fe202d + name: 'Worker {#NAME}: Role' + type: DEPENDENT + key: dv.worker.role[{#ID}] + delay: '0' + value_type: CHAR + trends: '0' + description: collector, storage or receiver + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].role.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: 8aadecc7e3804a13b5983b03df760205 + name: 'Worker {#NAME}: Status' + type: DEPENDENT + key: dv.worker.status_code[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Approval status + valuemap: + name: DeviceVault worker status + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].status_code.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: 11ffb095bd704547a65927072382e35b + name: 'Worker {#NAME}: Online' + type: DEPENDENT + key: dv.worker.online[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Sent a heartbeat recently + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].online.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: f163f1fc30d54e02887660b7703741ce + name: 'Worker {#NAME}: Heartbeat age' + type: DEPENDENT + key: dv.worker.heartbeat_age[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: Seconds since the last heartbeat (-1 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].heartbeat_age.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: ff011da8cf8345709d03ce890a8145ca + name: 'Worker {#NAME}: Pending approval for' + type: DEPENDENT + key: dv.worker.pending_age[{#ID}] + delay: '0' + value_type: FLOAT + units: s + description: Seconds since a pending worker registered (0 once decided) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].pending_age.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + trigger_prototypes: + - uuid: 24937a04ea184c9eb2b509b68a29beb8 + expression: last(/DeviceVault by HTTP/dv.worker.pending_age[{#ID}])>{$DV.PENDING.MAX} + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.worker.pending_age[{#ID}])<={$DV.PENDING.MAX} + name: 'DeviceVault: worker {#NAME}: pending approval for over {$DV.PENDING.MAX}' + priority: WARNING + description: A worker registered and is waiting for an administrator to approve or revoke it (Workers page). + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: worker + - tag: scope + value: notice + - tag: group + value: all + - uuid: 74195fd317f94c4f9dd10e9330c3e9f8 + name: 'Worker {#NAME}: Version' + type: DEPENDENT + key: dv.worker.version[{#ID}] + delay: '0' + value_type: CHAR + trends: '0' + description: The DeviceVault version it runs + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].version.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + - uuid: b519b1cd45fa4b9291b46e13aafe7f19 + name: 'Worker {#NAME}: Assignment behind' + type: DEPENDENT + key: dv.worker.assignment_behind[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Applied assignment version is older than the current one + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].assignment_behind.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.worker.get + tags: + - tag: component + value: worker + - tag: worker + value: '{#NAME}' + trigger_prototypes: + - uuid: 755a31b876ef400fbb3d39602ab3c91d + expression: min(/DeviceVault by HTTP/dv.worker.assignment_behind[{#ID}],10m)=1 and last(/DeviceVault by HTTP/dv.worker.assignment_behind[{#ID}],#1:now-10m)=1 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.worker.assignment_behind[{#ID}])=0 + name: 'DeviceVault: worker {#NAME}: assignment behind for 10 minutes' + priority: WARNING + description: The worker has not applied its current assignment for 10 minutes. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: worker + - tag: scope + value: notice + - tag: group + value: all + trigger_prototypes: + - uuid: e88297272d484bb3aba01988e234fd68 + expression: last(/DeviceVault by HTTP/dv.worker.status_code[{#ID}])=1 and (last(/DeviceVault by HTTP/dv.worker.heartbeat_age[{#ID}])>{$DV.WORKER.HEARTBEAT.MAX} or last(/DeviceVault by HTTP/dv.worker.heartbeat_age[{#ID}])<0) + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.worker.status_code[{#ID}])<>1 or (last(/DeviceVault by HTTP/dv.worker.heartbeat_age[{#ID}])>=0 and last(/DeviceVault by HTTP/dv.worker.heartbeat_age[{#ID}])<={$DV.WORKER.HEARTBEAT.MAX}) + name: 'DeviceVault: worker {#NAME}: no heartbeat for {$DV.WORKER.HEARTBEAT.MAX}' + opdata: 'Heartbeat age: {ITEM.LASTVALUE2}' + priority: AVERAGE + description: 'An approved worker has not sent a heartbeat recently: it is down or cannot reach the broker.' + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: worker + - tag: scope + value: availability + - tag: group + value: all + master_item: + key: dv.worker.get + lld_macro_paths: + - lld_macro: '{#ID}' + path: $.id + - lld_macro: '{#NAME}' + path: $.name + preprocessing: + - type: JSONPATH + parameters: + - $.data + - uuid: d40a955469964db28f69ea210a9f9705 + name: 'DeviceVault: Storage targets discovery' + type: DEPENDENT + key: dv.target.discovery + delay: '0' + lifetime: 7d + description: Storage targets from storage-targets/; resources gone for 7 days are deleted. + item_prototypes: + - uuid: e979fd4876f24de9a666e3c1d6caa68a + name: 'Storage target {#NAME}: Driver' + type: DEPENDENT + key: dv.target.driver[{#ID}] + delay: '0' + value_type: CHAR + trends: '0' + description: Storage driver + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].driver.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: 8db9426b364f4cfcac71b9f467124c2c + name: 'Storage target {#NAME}: Workers online' + type: DEPENDENT + key: dv.target.workers_online[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Approved storage workers serving it that are online + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].workers_online.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + trigger_prototypes: + - uuid: 4a6d3f0ca9f44be1917c4106d0ad5e62 + expression: last(/DeviceVault by HTTP/dv.target.workers_online[{#ID}])=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.target.workers_online[{#ID}])>0 + name: 'DeviceVault: storage target {#NAME}: no storage worker online' + priority: HIGH + description: No approved, online storage worker serves the target, so backups for it cannot be stored. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: storage + - tag: scope + value: availability + - tag: group + value: all + - uuid: d6586e7c647b46d194ecbbafc528e0a1 + name: 'Storage target {#NAME}: Last stored' + type: DEPENDENT + key: dv.target.last_stored_at[{#ID}] + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When a backup was last stored there (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].last_stored_at.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: d34db5bbcfbf4715a9785b40d3fa4834 + name: 'Storage target {#NAME}: Storage failures (24h)' + type: DEPENDENT + key: dv.target.storage_failures_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Jobs for it that failed at the storage step in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].storage_failures_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: 83d105687a954b309af7051bd23c1d9c + name: 'Storage target {#NAME}: Backups held' + type: DEPENDENT + key: dv.target.backups_held[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Stored backups not pruned by retention + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].backups_held.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: c93b2689278346aab31af5b6e192520f + name: 'Storage target {#NAME}: Bytes held' + type: DEPENDENT + key: dv.target.bytes_held[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of those backups + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_held.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + - uuid: cee76a86f53a4e629282cc68203d92c3 + name: 'Storage target {#NAME}: Bytes stored (24h)' + type: DEPENDENT + key: dv.target.bytes_stored_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the backups stored there in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].bytes_stored_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.target.get + tags: + - tag: component + value: target + - tag: target + value: '{#NAME}' + master_item: + key: dv.target.get + lld_macro_paths: + - lld_macro: '{#ID}' + path: $.id + - lld_macro: '{#NAME}' + path: $.name + preprocessing: + - type: JSONPATH + parameters: + - $.data + - uuid: f9a54bc1337d4596b693c1249c91b184 + name: 'DeviceVault: Receivers discovery' + type: DEPENDENT + key: dv.receiver.discovery + delay: '0' + lifetime: 7d + description: Receivers from receivers/; resources gone for 7 days are deleted. + item_prototypes: + - uuid: 71c074e5d2bb463dacf4b2f4d0df527c + name: 'Receiver {#NAME}: Online' + type: DEPENDENT + key: dv.receiver.online[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Sent a heartbeat recently + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].online.first() + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.receiver.get + tags: + - tag: component + value: receiver + - tag: receiver + value: '{#NAME}' + - uuid: 224a5839f0964e0890e32f4ebaec2596 + name: 'Receiver {#NAME}: Pushes accepted (24h)' + type: DEPENDENT + key: dv.receiver.accepted_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Files it accepted in the last 24h + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].accepted_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.receiver.get + tags: + - tag: component + value: receiver + - tag: receiver + value: '{#NAME}' + - uuid: 03384f6d675b4e35bfc9b245767a6e73 + name: 'Receiver {#NAME}: Pushes refused (24h)' + type: DEPENDENT + key: dv.receiver.refused_24h[{#ID}] + delay: '0' + value_type: UNSIGNED + description: Pushes it refused in the last 24h (a lower bound) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].refused_24h.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.receiver.get + tags: + - tag: component + value: receiver + - tag: receiver + value: '{#NAME}' + - uuid: d95067a88f354eceb09b5b7ecbe9cb25 + name: 'Receiver {#NAME}: Last push' + type: DEPENDENT + key: dv.receiver.last_push_at[{#ID}] + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When it last accepted a file (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.data[?(@.id == '{#ID}')].last_push_at.first() + error_handler: DISCARD_VALUE + master_item: + key: dv.receiver.get + tags: + - tag: component + value: receiver + - tag: receiver + value: '{#NAME}' + master_item: + key: dv.receiver.get + lld_macro_paths: + - lld_macro: '{#ID}' + path: $.id + - lld_macro: '{#NAME}' + path: $.name + preprocessing: + - type: JSONPATH + parameters: + - $.data + tags: + - tag: class + value: software + - tag: target + value: devicevault + macros: + - macro: '{$DV.INTERVAL}' + value: 1m + description: How often the monitoring endpoints are polled. + - macro: '{$DV.NODATA}' + value: 5m + description: No answer from the monitoring API for this long raises the unreachable trigger. + - macro: '{$DV.SUCCESS.MIN}' + value: '95' + description: Lowest acceptable share of backups stored over 24h, in percent. + - macro: '{$DV.STALE.MAX}' + value: 2d + description: Longest acceptable age of the oldest last successful backup in a device group. + - macro: '{$DV.FAIL.CONSEC}' + value: '3' + description: Consecutive failures that count a device as failing (rounded down to 1, 3 or 5). + - macro: '{$DV.DURATION.MAX}' + value: 10m + description: Longest acceptable 95th percentile request-to-stored time over 24h. + - macro: '{$DV.WORKER.HEARTBEAT.MAX}' + value: 3m + description: Longest acceptable gap between an approved worker's heartbeats. + - macro: '{$DV.PENDING.MAX}' + value: 1h + description: Longest a worker may wait for approval before it is reported. + - macro: '{$DV.SKEW.MAX}' + value: '60' + description: Largest acceptable difference between the DeviceVault and Zabbix clocks, in seconds. + - macro: '{$DV.PUSH.REFUSED.MAX}' + value: '10' + description: More refused pushes than this within an hour raise a trigger. + dashboards: + - uuid: 1110baff42454613aea6d7bb5abb58f6 + name: DeviceVault overview + pages: + - name: Backups and platform + widgets: + - type: graph + width: '36' + height: '5' + fields: + - type: GRAPH + name: graphid.0 + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Success ratio' + - type: STRING + name: reference + value: AAAAA + - type: graph + x: '36' + width: '36' + height: '5' + fields: + - type: GRAPH + name: graphid.0 + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Request to stored time (24h)' + - type: STRING + name: reference + value: AAAAB + - type: graph + y: '5' + width: '36' + height: '5' + fields: + - type: GRAPH + name: graphid.0 + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Failures by reason (24h)' + - type: STRING + name: reference + value: AAAAC + - type: graph + x: '36' + y: '5' + width: '36' + height: '5' + fields: + - type: GRAPH + name: graphid.0 + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Device freshness' + - type: STRING + name: reference + value: AAAAD + - type: graph + y: '10' + width: '36' + height: '5' + fields: + - type: GRAPH + name: graphid.0 + value: + host: DeviceVault by HTTP + name: 'DeviceVault: Platform health' + - type: STRING + name: reference + value: AAAAE + - type: graphprototype + x: '36' + y: '10' + width: '36' + height: '5' + fields: + - type: INTEGER + name: columns + value: '1' + - type: GRAPH_PROTOTYPE + name: graphid.0 + value: + host: DeviceVault by HTTP + name: 'Group {#NAME}: Device freshness' + - type: STRING + name: reference + value: AAAAF + - type: graphprototype + y: '15' + width: '36' + height: '5' + fields: + - type: INTEGER + name: columns + value: '1' + - type: GRAPH_PROTOTYPE + name: graphid.0 + value: + host: DeviceVault by HTTP + name: 'Group {#NAME}: Success ratio' + - type: STRING + name: reference + value: AAAAG + valuemaps: + - uuid: ad56712d048b4e83a64ea74917e50cc8 + name: DeviceVault worker status + mappings: + - value: '0' + newvalue: pending + - value: '1' + newvalue: approved + - value: '2' + newvalue: revoked + - uuid: f3da0efc8f6148f2bba882f825995c2d + name: DeviceVault yes/no + mappings: + - value: '0' + newvalue: 'no' + - value: '1' + newvalue: 'yes' + - uuid: e5718ee14df14607bf11909779322638 + template: DeviceVault Device by HTTP + name: DeviceVault Device by HTTP + description: |- + DeviceVault backup status of this device, matched on its address ({HOST.IP}) or {$DV.DEVICE.ID}. + + Link it to the device hosts Zabbix already monitors. {$DV.RELAY.URL} is the monitoring relay on a DeviceVault receiver + in the zone of the proxy that monitors the host (set it once per zone on a site template that links this one), or + {$DV.URL}/api/monitoring where the proxy can reach DeviceVault directly. {$DV.TOKEN} (a service account's token with + the monitoring scope) is meant to be a global macro. + + Generated by DeviceVault from its metric catalogue; see DEPLOYMENT.md, section Monitoring. + groups: + - name: Templates/Applications + items: + - uuid: ca96cacc609e469094c7c3c06ff3ac51 + name: 'DeviceVault: Get device status' + type: HTTP_AGENT + key: dv.device.get + delay: '{$DV.DEVICE.INTERVAL}' + history: 1h + trends: '0' + value_type: TEXT + description: The device's answer from DeviceVault's published snapshot (503 when the snapshot is stale). + timeout: 15s + url: '{$DV.RELAY.URL}/device/?ip={HOST.IP}&id={$DV.DEVICE.ID}' + headers: + - name: Authorization + value: Bearer {$DV.TOKEN} + tags: + - tag: component + value: raw + status_codes: 200,503 + verify_peer: 'NO' + verify_host: 'NO' + - uuid: 0b705a2ff03d4b6abf9c05b3b87b0fce + name: 'DeviceVault: Answer error' + type: DEPENDENT + key: dv.device.error + delay: '0' + value_type: CHAR + trends: '0' + description: '"stale" when the relay has no current snapshot (503), otherwise "none".' + preprocessing: + - type: JSONPATH + parameters: + - $.error + error_handler: CUSTOM_VALUE + error_handler_params: none + master_item: + key: dv.device.get + tags: + - tag: component + value: api + - uuid: 2f38def1d45148ea81892889eb3fcd7c + name: 'DeviceVault: Found in DeviceVault' + type: DEPENDENT + key: dv.device.found + delay: '0' + value_type: UNSIGNED + description: 1 when the address or id matches a device the token may see + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.found + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + triggers: + - uuid: f6d1caf6791f45e0846e4df9b722fe64 + expression: last(/DeviceVault Device by HTTP/dv.device.found)=0 and {$DV.NOTFOUND.ALERT}=1 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.found)=1 or {$DV.NOTFOUND.ALERT}=0 + name: 'DeviceVault: not backed up by DeviceVault' + priority: INFO + description: No device DeviceVault backs up (and the token may see) has this address or id. Set {$DV.NOTFOUND.ALERT} to 0 on hosts that are not meant to be backed up. + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: notice + - tag: group + value: all + - uuid: 40a1efbd13964034b8c293dea346e521 + name: 'DeviceVault: Id' + type: DEPENDENT + key: dv.device.id + delay: '0' + value_type: CHAR + trends: '0' + description: Identifier, used by low-level discovery + preprocessing: + - type: JSONPATH + parameters: + - $.id + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 8467a2e893ce42ddb52235e697202258 + name: 'DeviceVault: Name' + type: DEPENDENT + key: dv.device.name + delay: '0' + value_type: CHAR + trends: '0' + description: Name + preprocessing: + - type: JSONPATH + parameters: + - $.name + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: bb53b5154f4e4bdf870bf672f5397b33 + name: 'DeviceVault: Device group' + type: DEPENDENT + key: dv.device.group + delay: '0' + value_type: CHAR + trends: '0' + description: Device group name (empty if none) + preprocessing: + - type: JSONPATH + parameters: + - $.group + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: db859423317a497e9676d26870d419fb + name: 'DeviceVault: Address' + type: DEPENDENT + key: dv.device.ip + delay: '0' + value_type: CHAR + trends: '0' + description: The address DeviceVault backs up + preprocessing: + - type: JSONPATH + parameters: + - $.ip + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 9db0f69fc35e4232b6f2383904ea2f2c + name: 'DeviceVault: Enabled' + type: DEPENDENT + key: dv.device.enabled + delay: '0' + value_type: UNSIGNED + description: Not paused + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.enabled + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: e0fa056945c349aa90d5c30c4e3164c3 + name: 'DeviceVault: Lifecycle' + type: DEPENDENT + key: dv.device.lifecycle_code + delay: '0' + value_type: UNSIGNED + description: Active, inactive (retired, held) or deletable + valuemap: + name: DeviceVault lifecycle + preprocessing: + - type: JSONPATH + parameters: + - $.lifecycle_code + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: fa7d2d9077b74297bc7b4342337122b6 + name: 'DeviceVault: Backup status' + type: DEPENDENT + key: dv.device.status_code + delay: '0' + value_type: UNSIGNED + description: 'The latest job: stored, failed, timed out, running, or never backed up' + valuemap: + name: DeviceVault job status + preprocessing: + - type: JSONPATH + parameters: + - $.status_code + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: cad221c62dd24a39b90c4b664c1f1d47 + name: 'DeviceVault: Last attempt' + type: DEPENDENT + key: dv.device.last_attempt_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When the latest job finished, or started if it is still running (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_attempt_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: f44841ebbfc044dcb6747ebe4dc38930 + name: 'DeviceVault: Last successful backup' + type: DEPENDENT + key: dv.device.last_success_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When the newest stored backup finished (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_success_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 1c65dfa53ed64c65959211fa327f0bb9 + name: 'DeviceVault: Last successful backup, age' + type: DEPENDENT + key: dv.device.last_success_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since then, at answer time (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_success_age + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 768084e7b85f4edea770786703b6a0b7 + name: 'DeviceVault: Last failure' + type: DEPENDENT + key: dv.device.last_failure_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When a job last failed or timed out (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_failure_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: b0e0321d217c46aea54c9d3f708d5c84 + name: 'DeviceVault: Failure reason' + type: DEPENDENT + key: dv.device.failure_reason_code + delay: '0' + value_type: UNSIGNED + description: Why the latest job failed (none when it did not) + valuemap: + name: DeviceVault failure reason + preprocessing: + - type: JSONPATH + parameters: + - $.failure_reason_code + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 33095c096c624dd09f226d2a5770c0ec + name: 'DeviceVault: Consecutive failures' + type: DEPENDENT + key: dv.device.consecutive_failures + delay: '0' + value_type: UNSIGNED + description: Finished jobs that failed since the last stored one + preprocessing: + - type: JSONPATH + parameters: + - $.consecutive_failures + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: f0440c89b95944408aa36391dd0bbf49 + name: 'DeviceVault: Expected interval' + type: DEPENDENT + key: dv.device.expected_interval + delay: '0' + value_type: FLOAT + units: s + description: The longest gap between the schedule's runs (0 without a schedule) + preprocessing: + - type: JSONPATH + parameters: + - $.expected_interval + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: b98a330701cd4dd19f1105464588fafa + name: 'DeviceVault: Next backup due by' + type: DEPENDENT + key: dv.device.next_due_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When a successful backup is due before the device counts as overdue (0 without a schedule) + preprocessing: + - type: JSONPATH + parameters: + - $.next_due_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 5d1320ecf6984edaaf73ee521c4f059e + name: 'DeviceVault: Overdue' + type: DEPENDENT + key: dv.device.overdue + delay: '0' + value_type: UNSIGNED + description: Its last success is older than its schedule allows + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.overdue + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: e2e60ebeabfe486ba4ef8a7cfa33f821 + name: 'DeviceVault: Last collection time' + type: DEPENDENT + key: dv.device.last_collect_duration + delay: '0' + value_type: FLOAT + units: s + description: Collection time of the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_collect_duration + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: afd4c90928d948f48f348ee27df39382 + name: 'DeviceVault: Last storage time' + type: DEPENDENT + key: dv.device.last_store_duration + delay: '0' + value_type: FLOAT + units: s + description: Storage time of the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_store_duration + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: dc616356820b4b7da3ff5704f9736c8b + name: 'DeviceVault: Last backup duration' + type: DEPENDENT + key: dv.device.last_duration + delay: '0' + value_type: FLOAT + units: s + description: Request-to-stored time of the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_duration + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: e6100b260b074f869daea2da5b032a60 + name: 'DeviceVault: Last backup size' + type: DEPENDENT + key: dv.device.last_size + delay: '0' + value_type: UNSIGNED + units: B + description: Size of the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_size + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 7901f7bda8bd4bbfbd1ec02e56ce7d47 + name: 'DeviceVault: Last backup changed' + type: DEPENDENT + key: dv.device.last_changed + delay: '0' + value_type: UNSIGNED + description: The newest stored backup differs from the one before + valuemap: + name: DeviceVault yes/no + preprocessing: + - type: JSONPATH + parameters: + - $.last_changed + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: db66c12e466443a29fa4b20bfb032636 + name: 'DeviceVault: Lines added' + type: DEPENDENT + key: dv.device.last_lines_added + delay: '0' + value_type: UNSIGNED + description: By the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_lines_added + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: ef580c780a1c4398a4dab9ec72a9088c + name: 'DeviceVault: Lines removed' + type: DEPENDENT + key: dv.device.last_lines_removed + delay: '0' + value_type: UNSIGNED + description: By the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_lines_removed + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 326ddb83a0554307bc3f6f3bfce1978d + name: 'DeviceVault: Lines modified' + type: DEPENDENT + key: dv.device.last_lines_modified + delay: '0' + value_type: UNSIGNED + description: By the newest stored backup + preprocessing: + - type: JSONPATH + parameters: + - $.last_lines_modified + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 3d17344ee7df40b9ab4dc23a89c056e8 + name: 'DeviceVault: Backups held' + type: DEPENDENT + key: dv.device.backups_held + delay: '0' + value_type: UNSIGNED + description: Stored backups not pruned + preprocessing: + - type: JSONPATH + parameters: + - $.backups_held + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: c0cbd718251245bbb16143c842262d05 + name: 'DeviceVault: Pinned backups' + type: DEPENDENT + key: dv.device.pinned + delay: '0' + value_type: UNSIGNED + description: Backups pinned against retention + preprocessing: + - type: JSONPATH + parameters: + - $.pinned + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 23a76f004983476dae7f52b3c449dd7c + name: 'DeviceVault: Push mode' + type: DEPENDENT + key: dv.device.push_mode + delay: '0' + value_type: CHAR + trends: '0' + description: off, ssh or tftp + preprocessing: + - type: JSONPATH + parameters: + - $.push_mode + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: f2bd3455a4fc47e99d290344d55124a5 + name: 'DeviceVault: Last push' + type: DEPENDENT + key: dv.device.last_push_at + delay: '0' + value_type: UNSIGNED + units: unixtime + description: When the device last pushed a file that was accepted (0 if never) + preprocessing: + - type: JSONPATH + parameters: + - $.last_push_at + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 78a87c73abb04e1e9caff7d82e2aca00 + name: 'DeviceVault: Success ratio (7d)' + type: DEPENDENT + key: dv.device.success_ratio_7d + delay: '0' + value_type: FLOAT + units: '%' + description: Stored as a share of finished jobs in the last 7 days; null with none finished + preprocessing: + - type: JSONPATH + parameters: + - $.success_ratio_7d + error_handler: DISCARD_VALUE + - type: NOT_MATCHES_REGEX + parameters: + - ^(null)?$ + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 6a1a3f1be3414722b8ff0fe93e85a0d3 + name: 'DeviceVault: Data published at' + type: DEPENDENT + key: dv.device.published_at + delay: '0' + value_type: CHAR + trends: '0' + description: When the orchestrator published the snapshot this answer comes from + preprocessing: + - type: JSONPATH + parameters: + - $.published_at + error_handler: DISCARD_VALUE + - type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1h + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + - uuid: 186b51512b1749d48a1e95c7c6610c39 + name: 'DeviceVault: Data age' + type: DEPENDENT + key: dv.device.published_age + delay: '0' + value_type: FLOAT + units: s + description: Seconds since that snapshot was published + preprocessing: + - type: JSONPATH + parameters: + - $.published_age + error_handler: DISCARD_VALUE + master_item: + key: dv.device.get + tags: + - tag: component + value: backup + triggers: + - uuid: 0f0ee0d3ba0843139e2632e377aea3c4 + expression: last(/DeviceVault Device by HTTP/dv.device.published_age)>{$DV.DEVICE.DATA.MAX} + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.published_age)<={$DV.DEVICE.DATA.MAX} + name: 'DeviceVault: DeviceVault data is old' + priority: WARNING + description: 'The snapshot answering was published longer ago than {$DV.DEVICE.DATA.MAX}: the orchestrator has stopped publishing while the relay still serves.' + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: api + - tag: scope + value: availability + - tag: group + value: all + tags: + - tag: class + value: software + - tag: target + value: devicevault + macros: + - macro: '{$DV.RELAY.URL}' + value: https://receiver-site-a:8443 + description: The monitoring relay of a receiver in this zone (or {$DV.URL}/api/monitoring for the orchestrator). Set it once per zone on a site template. + - macro: '{$DV.DEVICE.ID}' + description: 'DeviceVault''s id for this device, for a host whose interface address is not the one DeviceVault backs up. Empty: match on {HOST.IP}.' + - macro: '{$DV.DEVICE.INTERVAL}' + value: 15m + description: How often the device status is polled (snapshots are published every 15m). + - macro: '{$DV.DEVICE.NODATA}' + value: 45m + description: No answer for this long (three poll intervals) raises the no-data trigger. + - macro: '{$DV.NOTFOUND.ALERT}' + value: '1' + description: 1 reports hosts DeviceVault does not back up; 0 switches that trigger off. + - macro: '{$DV.DEVICE.FAIL.CONSEC}' + value: '2' + description: Consecutive failed backups that raise the failing trigger. + - macro: '{$DV.DEVICE.STALE.MAX}' + value: '0' + description: Raise a trigger when the last successful backup is older than this (for example 2d); 0 switches it off in favour of the schedule-based overdue trigger. + - macro: '{$DV.DEVICE.DATA.MAX}' + value: 45m + description: Raise a trigger when the published snapshot answering is older than this. + valuemaps: + - uuid: e9df205963684b62aa3d2152dcef8d53 + name: DeviceVault job status + mappings: + - value: '0' + newvalue: never + - value: '1' + newvalue: stored + - value: '2' + newvalue: failed + - value: '3' + newvalue: timed_out + - value: '4' + newvalue: running + - uuid: fd31acf62a2043608afd3dfff4a5cf74 + name: DeviceVault failure reason + mappings: + - value: '0' + newvalue: none + - value: '1' + newvalue: connection + - value: '2' + newvalue: auth + - value: '3' + newvalue: timeout + - value: '4' + newvalue: command + - value: '5' + newvalue: verification + - value: '6' + newvalue: storage + - value: '7' + newvalue: integrity + - value: '8' + newvalue: push + - value: '9' + newvalue: orphaned + - value: '10' + newvalue: unknown + - uuid: 25b4248acc1947348e0c2a1f26787204 + name: DeviceVault lifecycle + mappings: + - value: '0' + newvalue: active + - value: '1' + newvalue: inactive + - value: '2' + newvalue: deletable + - uuid: 428caf8ffb724ce9bea7019a52add6e2 + name: DeviceVault yes/no + mappings: + - value: '0' + newvalue: 'no' + - value: '1' + newvalue: 'yes' + triggers: + - uuid: 5f7ec9745d7e4ec6805dac9604d8e610 + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + name: 'DeviceVault: monitoring figures are stale' + priority: WARNING + description: The figures are older than three poll intervals, or none arrived for {$DV.NODATA}. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + tags: + - tag: component + value: api + - tag: scope + value: availability + - tag: group + value: all + - uuid: 262d6d188d60400ba208aca504a25c50 + expression: last(/DeviceVault by HTTP/dv.success_ratio_24h)<{$DV.SUCCESS.MIN} and last(/DeviceVault by HTTP/dv.finished_24h)>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.success_ratio_24h)>={$DV.SUCCESS.MIN} or last(/DeviceVault by HTTP/dv.finished_24h)=0 + name: 'DeviceVault: success ratio below {$DV.SUCCESS.MIN}% (24h)' + opdata: 'Success ratio: {ITEM.LASTVALUE1}' + priority: WARNING + description: Fewer backups than {$DV.SUCCESS.MIN}% of those that finished in the last 24h were stored. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: backups + - tag: scope + value: notice + - tag: group + value: all + - uuid: d0f52a03ad3142c3810fd1ce8e0f0dba + expression: last(/DeviceVault by HTTP/dv.duration_total_p95)>{$DV.DURATION.MAX} and last(/DeviceVault by HTTP/dv.finished_24h)>0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault by HTTP/dv.duration_total_p95)<={$DV.DURATION.MAX} or last(/DeviceVault by HTTP/dv.finished_24h)=0 + name: 'DeviceVault: backups are slow (p95 over {$DV.DURATION.MAX})' + opdata: 'p95: {ITEM.LASTVALUE1}' + priority: WARNING + description: The 95th percentile request-to-stored time of the last 24h is above {$DV.DURATION.MAX}. + dependencies: + - name: 'DeviceVault: monitoring API unreachable or refusing' + expression: last(/DeviceVault by HTTP/dv.http.status)<>200 or nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.http.status)=200 and nodata(/DeviceVault by HTTP/dv.http.status,{$DV.NODATA})=0 + - name: 'DeviceVault: monitoring figures are stale' + expression: last(/DeviceVault by HTTP/dv.computed_age)>3*{$DV.INTERVAL} or nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=1 + recovery_expression: last(/DeviceVault by HTTP/dv.computed_age)<=3*{$DV.INTERVAL} and nodata(/DeviceVault by HTTP/dv.server_time,{$DV.NODATA})=0 + tags: + - tag: component + value: timing + - tag: scope + value: performance + - tag: group + value: all + - uuid: fa2f7d0d147341349b66ca380fcd7468 + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + name: 'DeviceVault: no data from DeviceVault' + priority: WARNING + description: The relay at {$DV.RELAY.URL} did not answer for {$DV.DEVICE.NODATA}, refused the token, or has no current snapshot (503 stale). The other DeviceVault triggers of this host depend on this one. + tags: + - tag: component + value: api + - tag: scope + value: availability + - tag: group + value: all + - uuid: 5ccbaa48f4f34b5999d0c20e5d6e67d5 + expression: last(/DeviceVault Device by HTTP/dv.device.overdue)=1 and last(/DeviceVault Device by HTTP/dv.device.found)=1 and last(/DeviceVault Device by HTTP/dv.device.enabled)=1 and last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.overdue)=0 or last(/DeviceVault Device by HTTP/dv.device.found)<>1 or last(/DeviceVault Device by HTTP/dv.device.enabled)<>1 or last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)<>0 + name: 'DeviceVault: backup overdue' + priority: WARNING + description: The last successful backup is older than the schedule allows. + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: notice + - tag: group + value: all + - uuid: a9bfb97f075c4a22aa3121faeba05a68 + expression: last(/DeviceVault Device by HTTP/dv.device.consecutive_failures)>={$DV.DEVICE.FAIL.CONSEC} and last(/DeviceVault Device by HTTP/dv.device.found)=1 and last(/DeviceVault Device by HTTP/dv.device.enabled)=1 and last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.consecutive_failures)<{$DV.DEVICE.FAIL.CONSEC} or last(/DeviceVault Device by HTTP/dv.device.found)<>1 or last(/DeviceVault Device by HTTP/dv.device.enabled)<>1 or last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)<>0 + name: 'DeviceVault: backup failing' + opdata: 'Consecutive failures: {ITEM.LASTVALUE1}' + priority: WARNING + description: The last {$DV.DEVICE.FAIL.CONSEC} or more backups failed. + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: notice + - tag: group + value: all + - uuid: fb065cb052094f1c933cd4429ff84d0e + expression: last(/DeviceVault Device by HTTP/dv.device.failure_reason_code)=2 and last(/DeviceVault Device by HTTP/dv.device.found)=1 and last(/DeviceVault Device by HTTP/dv.device.enabled)=1 and last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)=0 + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: last(/DeviceVault Device by HTTP/dv.device.failure_reason_code)<>2 or last(/DeviceVault Device by HTTP/dv.device.found)<>1 or last(/DeviceVault Device by HTTP/dv.device.enabled)<>1 or last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)<>0 + name: 'DeviceVault: backup authentication failing' + priority: AVERAGE + description: The latest backup failed because the device refused the login or enable credentials. + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: security + - tag: group + value: all + - uuid: fd6e4d6fb2734e51a26db1a01ef65abd + expression: '{$DV.DEVICE.STALE.MAX}>0 and last(/DeviceVault Device by HTTP/dv.device.last_success_at)>0 and last(/DeviceVault Device by HTTP/dv.device.last_success_age)>{$DV.DEVICE.STALE.MAX} and last(/DeviceVault Device by HTTP/dv.device.found)=1 and last(/DeviceVault Device by HTTP/dv.device.enabled)=1 and last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)=0' + recovery_mode: RECOVERY_EXPRESSION + recovery_expression: '{$DV.DEVICE.STALE.MAX}=0 or last(/DeviceVault Device by HTTP/dv.device.last_success_age)<={$DV.DEVICE.STALE.MAX} or last(/DeviceVault Device by HTTP/dv.device.found)<>1 or last(/DeviceVault Device by HTTP/dv.device.enabled)<>1 or last(/DeviceVault Device by HTTP/dv.device.lifecycle_code)<>0' + name: 'DeviceVault: no successful backup for {$DV.DEVICE.STALE.MAX}' + priority: WARNING + description: A flat threshold on the age of the last successful backup ({$DV.DEVICE.STALE.MAX}, 0 = off). + dependencies: + - name: 'DeviceVault: no data from DeviceVault' + expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=1 or last(/DeviceVault Device by HTTP/dv.device.error)="stale" + recovery_expression: nodata(/DeviceVault Device by HTTP/dv.device.get,{$DV.DEVICE.NODATA})=0 and last(/DeviceVault Device by HTTP/dv.device.error)<>"stale" + tags: + - tag: component + value: backup + - tag: scope + value: notice + - tag: group + value: all + graphs: + - uuid: 1cec28292b5d42f68444bfb98be412d8 + name: 'DeviceVault: Success ratio' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.success_ratio_1h + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.success_ratio_24h + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.success_ratio_7d + - uuid: 7ed6c1ea807343d1b74434c108fa0a7a + name: 'DeviceVault: Request to stored time (24h)' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.duration_total_avg + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.duration_total_p50 + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.duration_total_p95 + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.duration_total_max + - uuid: f573a586b441450d9bf961cfca6506c8 + name: 'DeviceVault: Failures by reason (24h)' + type: STACKED + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.failures_connection_24h + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.failures_auth_24h + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.failures_timeout_24h + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.failures_command_24h + - sortorder: '4' + color: FC6EA3 + item: + host: DeviceVault by HTTP + key: dv.failures_verification_24h + - sortorder: '5' + color: 6C59DC + item: + host: DeviceVault by HTTP + key: dv.failures_storage_24h + - sortorder: '6' + color: C7A72D + item: + host: DeviceVault by HTTP + key: dv.failures_integrity_24h + - sortorder: '7' + color: BA2A5D + item: + host: DeviceVault by HTTP + key: dv.failures_push_24h + - sortorder: '8' + color: 0080FF + item: + host: DeviceVault by HTTP + key: dv.failures_orphaned_24h + - sortorder: '9' + color: '808080' + item: + host: DeviceVault by HTTP + key: dv.failures_unknown_24h + - uuid: c93c5351861d412da181216f86e3836f + name: 'DeviceVault: Device freshness' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.devices_overdue + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.devices_never + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.devices_failing_1 + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.devices_failing_3 + - sortorder: '4' + color: FC6EA3 + item: + host: DeviceVault by HTTP + key: dv.devices_failing_5 + - uuid: 166e916cdc594bdb97a1ed86b6da5652 + name: 'DeviceVault: Platform health' + graph_items: + - color: 199C0D + item: + host: DeviceVault by HTTP + key: dv.workers_online + - sortorder: '1' + color: F63100 + item: + host: DeviceVault by HTTP + key: dv.workers_offline + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault by HTTP + key: dv.workers_pending + - sortorder: '3' + color: F7941D + item: + host: DeviceVault by HTTP + key: dv.workers_behind + - sortorder: '4' + color: FC6EA3 + item: + host: DeviceVault by HTTP + key: dv.scheduler_leader + - uuid: 5ea75b63f03e45e19a4555fdc8527e85 + name: 'DeviceVault: Last backup duration' + graph_items: + - color: 199C0D + item: + host: DeviceVault Device by HTTP + key: dv.device.last_duration + - sortorder: '1' + color: F63100 + item: + host: DeviceVault Device by HTTP + key: dv.device.last_collect_duration + - sortorder: '2' + color: 2774A4 + item: + host: DeviceVault Device by HTTP + key: dv.device.last_store_duration diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e2f8bc4..98d95b0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -667,8 +667,8 @@ Only live tokens of active service accounts are listed, and only to receivers wi **Generator.** `monitoring/zabbix.py` builds the templates from the catalogue. `manage.py zabbix_template --version 6.0|7.0` writes one file, and `make zabbix-template` regenerates both into `deploy/zabbix/devicevault_zabbix_6.0.yaml` and `devicevault_zabbix_7.0.yaml`. Both are committed, so changes show in review, and a test fails when either is out of step with the generator. The export format differs between the LTS versions (UUIDs, dashboard widget fields, some preprocessing names), so the generator has one small emitter per version over shared content. Template UUIDs are derived from stable keys (uuid5), so regenerating changes nothing that did not change. Templates do not depend on the installation (drivers and groups are discovered), so the endpoint serves the same output. **`DeviceVault by HTTP`** is linked to one host that stands for DeviceVault. -- **Macros:** `{$DV.URL}`, `{$DV.TOKEN}` (secret text), `{$DV.INTERVAL}` (1m), `{$DV.SUCCESS.MIN}` (95), `{$DV.STALE.MAX}` (2d), `{$DV.FAIL.CONSEC}` (3), `{$DV.DURATION.MAX}` (10m), `{$DV.WORKER.HEARTBEAT.MAX}` (3m), `{$DV.PENDING.MAX}` (1h), `{$DV.SKEW.MAX}` (60). -- **Master items:** one HTTP agent item each for `summary/`, `device-groups/`, `workers/`, `storage-targets/` and `receivers/`, sending `Authorization: Bearer {$DV.TOKEN}`. Every value is a dependent item with JSONPath, and those that rarely change use *Discard unchanged with heartbeat*. +- **Macros:** `{$DV.INTERVAL}` (1m), `{$DV.NODATA}` (5m), `{$DV.SUCCESS.MIN}` (95), `{$DV.STALE.MAX}` (2d), `{$DV.FAIL.CONSEC}` (3, rounded down to the 1, 3 and 5 the catalogue counts), `{$DV.DURATION.MAX}` (10m), `{$DV.WORKER.HEARTBEAT.MAX}` (3m), `{$DV.PENDING.MAX}` (1h), `{$DV.SKEW.MAX}` (60), `{$DV.PUSH.REFUSED.MAX}` (10 an hour). `{$DV.URL}` and `{$DV.TOKEN}` (secret text) are global macros, and the templates deliberately leave them undefined: a template macro takes precedence over a global one, so it would shadow it. +- **Master items:** one HTTP agent item each for `summary/`, `device-groups/`, `workers/`, `storage-targets/` and `receivers/`, sending `Authorization: Bearer {$DV.TOKEN}`, plus one that keeps only the HTTP status of `summary/` (for the endpoint trigger). Every value is a dependent item with JSONPath. A missing field or a JSON null is discarded, so it never turns an item unsupported. Text and value-mapped items use *Discard unchanged with heartbeat* (1h), except those a trigger reads over a time window. - **Discovery:** dependent LLD rules (supported since Zabbix 4.2) for device groups, workers, storage targets and receivers. Each has item and trigger prototypes and a *Keep lost resources* period of 7 days. - **Triggers:** - Endpoint unreachable, or answering 401/403/5xx. @@ -695,11 +695,16 @@ Only live tokens of active service accounts are listed, and only to receivers wi - *Backup failing*: consecutive failures >= `{$DV.DEVICE.FAIL.CONSEC}` (2). - *Authentication failing*: the latest reason is `auth`, severity Average, because credentials do not fix themselves. - *No successful backup for `{$DV.DEVICE.STALE.MAX}`*: 0 by default, which switches it off. This is for estates that prefer a flat threshold to the schedule-based `overdue`. - - *No data from DeviceVault*: `nodata` on the master item for 3 intervals, or the relay answering 503 `stale`. The other device triggers depend on it. + - *No data from DeviceVault*: `nodata` on the master item for `{$DV.DEVICE.NODATA}` (45m, three intervals), or the relay answering 503 `stale`. The other device triggers depend on it. - *DeviceVault data is old*: `published_at` older than `{$DV.DEVICE.DATA.MAX}` (45m), which catches a publisher that has stopped while a relay still serves. - - Device triggers include `enabled=1 and lifecycle=active`, so a paused or retired device stays quiet. Zabbix cannot make a template trigger depend on a trigger of another host (the DeviceVault host), hence the per-host *No data* trigger. + - The backup triggers (overdue, failing, authentication, flat stale) include `found=1 and enabled=1 and lifecycle=active`, so a paused or retired device stays quiet. Zabbix cannot make a template trigger depend on a trigger of another host (the DeviceVault host), hence the per-host *No data* trigger. -**Validation.** `make zabbix-validate` starts Zabbix 6.0 and then 7.0 (server, web and PostgreSQL containers from `deploy/zabbix-test/`), and imports both templates through the Zabbix API (`configuration.import`). Then it creates one host per template, pointing the DeviceVault host at a stub server that serves recorded orchestrator responses, and the device host at a real receiver relay loaded with a recorded, signed snapshot. It checks that every item becomes supported and that the expected triggers fire. It is Docker only, like everything else. It is not part of `make check` (two Zabbix servers make CI slow). A separate `zabbix.yml` workflow runs it on pull requests that touch `monitoring/` or `deploy/zabbix/`, and it is not required to merge. +**Validation.** `make zabbix-validate` starts Zabbix 6.0 and then 7.0 (server, web and PostgreSQL containers from `deploy/zabbix-test/`), and imports both templates through the Zabbix API (`configuration.import`). Then it creates hosts pointing at a stub server (`deploy/zabbix-test/stub.py`) that serves recorded orchestrator and relay responses (`responses/`, kept to the catalogue's keys by a test): +- the DeviceVault host; +- a DeviceVault host whose token is refused; +- device hosts that are found, unknown (`found: 0`), and answered 503 `stale`. + +It checks three things: every item is supported, every item holds its recorded value (nulls and absent fields hold none), and exactly the expected triggers fire. The relay itself is tested by phase 23's own suite against the same `device_response` contract. It is Docker only, like everything else. It is not part of `make check` (two Zabbix servers make CI slow). A separate `zabbix.yml` workflow runs it on pull requests that touch `monitoring/` or `deploy/zabbix/`, and it is not required to merge. **Documentation.** A "Monitoring" section in `DEPLOYMENT.md` covers: - creating the service account and token; diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index f0976f5..f5e482a 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -499,6 +499,7 @@ shows the current figures. | `device-groups/` | `{"data": [...]}`, the same figures per visible device group (Zabbix low-level discovery) | | `device/?ip=
` (or `&id=`) | One device: status, last success and its age, failure reason, failures in a row, schedule interval, next due and overdue, the last backup's duration, size and changes, backups held. Answered from the service account's published snapshot; `found` is 0 for an unknown device or one the account cannot see; 503 `stale` when no current snapshot exists | | `workers/`, `storage-targets/`, `receivers/` | `{"data": [...]}` platform lists (empty without the platform scope) | +| `zabbix/template/?version=6.0\|7.0` | The generated Zabbix templates, as a YAML download (see Zabbix below) | | `metrics` | Prometheus text format: the estate, group, worker, target and receiver figures (no per-device series) | Figures are computed from job rows and cached for `DV_MONITORING_CACHE_SECONDS`. @@ -517,6 +518,127 @@ timeout) or `unknown`; only the reason is exported, never the error text. (`devicevault_group_*`, labelled `device_group`), so sums never double-count the estate. +### Zabbix + +DeviceVault ships two Zabbix templates in one file per Zabbix LTS version: +`deploy/zabbix/devicevault_zabbix_6.0.yaml` and `devicevault_zabbix_7.0.yaml`. You can +also download them from **Admin Settings > Monitoring**, or from +`/api/monitoring/zabbix/template/?version=6.0|7.0` with the token. They are generated +from the metric catalogue and are the same for every installation, because groups, +workers, storage targets and receivers are discovered. They need no Zabbix agent. + +1. **Import** the file for your version (Data collection > Templates > Import, or + `configuration.import` through the API). It creates `DeviceVault by HTTP` and + `DeviceVault Device by HTTP` in `Templates/Applications`. +2. **Global macros** (Administration > Macros): + - `{$DV.URL}`: the web address of DeviceVault, without `/api` (for example + `https://devicevault.example.com`). + - `{$DV.TOKEN}`: the service account's token, as a **Secret text** macro. + + The templates deliberately do not define these two, so a template value cannot + shadow the global one. You can also set them on a host or a site template. +3. **The DeviceVault host.** Create one host (for example `DeviceVault`; it needs no + interface) and link `DeviceVault by HTTP`. Monitor it from the Zabbix server, or from + a proxy that can reach the web tier. Five HTTP agent items poll `summary/`, + `device-groups/`, `workers/`, `storage-targets/` and `receivers/` every + `{$DV.INTERVAL}`. A sixth checks the HTTP status. Every figure is a dependent item. + Dependent discovery rules create items, triggers and graphs per device group, worker, + storage target and receiver, and delete them 7 days after they disappear. The + template's dashboard, *DeviceVault overview*, shows the success ratio, duration + percentiles, failures by reason, freshness overall and per group, and platform health. + Give the token the **Platform health** scope to fill in the worker, scheduler and + storage-target parts. Without it those items just stay empty. +4. **Device hosts.** Link `DeviceVault Device by HTTP` to the hosts Zabbix already + monitors, for example across a host group (Data collection > Hosts, select, Mass + update > Templates). Each host asks + `{$DV.RELAY.URL}/device/?ip={HOST.IP}&id={$DV.DEVICE.ID}` every + `{$DV.DEVICE.INTERVAL}`. `{HOST.IP}` is the host's default interface address, so + linking is usually the whole setup. Set `{$DV.DEVICE.ID}` (the DeviceVault device id) + on a host whose interface address is not the one DeviceVault backs up. + - The proxies that monitor devices usually cannot reach the orchestrator. Point them + at **the monitoring relay on a receiver** in their own zone instead (see the + receiver documentation for enabling it; it listens on 8443, so allow the proxy to + reach the receiver on that port). + - Zabbix has no host-group or proxy macros. So create one small **site template** + per zone (for example `DeviceVault site A`) that links `DeviceVault Device by HTTP` + and sets `{$DV.RELAY.URL}` to `https://receiver-site-a:8443`. Then link the site + template, not the device template, to that zone's hosts. + - Where the proxies can reach the web tier, set `{$DV.RELAY.URL}` to + `{$DV.URL}/api/monitoring` instead and skip the relay. + - The item does not verify the relay's certificate, because relays use a self-signed + one. + - The device template can use the same `monitoring` token as the DeviceVault host. + +**Macros and thresholds.** Override any of these on a host or site template. + +| Macro | Default | Template | Meaning | +|---|---|---|---| +| `{$DV.URL}` | (global) | both | DeviceVault's web address | +| `{$DV.TOKEN}` | (global, secret) | both | Service account token (`monitoring`, plus `monitoring_platform` for platform health) | +| `{$DV.INTERVAL}` | `1m` | DeviceVault | Poll interval | +| `{$DV.NODATA}` | `5m` | DeviceVault | No answer for this long: *monitoring API unreachable* | +| `{$DV.SUCCESS.MIN}` | `95` | DeviceVault | Lowest acceptable success ratio over 24h (%), estate and per group | +| `{$DV.STALE.MAX}` | `2d` | DeviceVault | Oldest acceptable last success, per group | +| `{$DV.FAIL.CONSEC}` | `3` | DeviceVault | Consecutive failures that count a device as failing, per group (rounded down to 1, 3 or 5) | +| `{$DV.DURATION.MAX}` | `10m` | DeviceVault | Longest acceptable p95 request-to-stored time over 24h | +| `{$DV.WORKER.HEARTBEAT.MAX}` | `3m` | DeviceVault | Longest gap between an approved worker's heartbeats | +| `{$DV.PENDING.MAX}` | `1h` | DeviceVault | Longest a worker may wait for approval | +| `{$DV.SKEW.MAX}` | `60` | DeviceVault | Largest clock difference to the Zabbix server, in seconds | +| `{$DV.PUSH.REFUSED.MAX}` | `10` | DeviceVault | Refused pushes per hour that count as a spike | +| `{$DV.RELAY.URL}` | `https://receiver-site-a:8443` | Device | The zone's relay, or `{$DV.URL}/api/monitoring` | +| `{$DV.DEVICE.ID}` | empty | Device | Match on this device id instead of `{HOST.IP}` | +| `{$DV.DEVICE.INTERVAL}` | `15m` | Device | Poll interval (snapshots are published every 15 minutes, so faster gains nothing) | +| `{$DV.DEVICE.NODATA}` | `45m` | Device | No answer for this long (three polls): *no data from DeviceVault* | +| `{$DV.NOTFOUND.ALERT}` | `1` | Device | `0` silences *not backed up by DeviceVault* (an Information problem) | +| `{$DV.DEVICE.FAIL.CONSEC}` | `2` | Device | Consecutive failures for *backup failing* | +| `{$DV.DEVICE.STALE.MAX}` | `0` (off) | Device | Flat age limit for the last success, for example `2d` | +| `{$DV.DEVICE.DATA.MAX}` | `45m` | Device | Oldest acceptable published snapshot | + +**Triggers.** +- **DeviceVault host:** + - the API is unreachable or answers 401, 403 or 5xx; + - the figures are stale; + - the success ratio is low (estate and per group); + - per group: the oldest last success is too old, devices are overdue, or devices + are failing; + - devices have never been backed up; + - the p95 duration is too high; + - clock skew; + - a spike in push refusals; + - the scheduler has no leader, or is not ticking; + - a worker has no heartbeat, has been pending approval too long, or has been + behind its assignment for 10 minutes; + - a storage target has no online worker. + + They all depend on the API trigger, so an outage raises one problem. +- **Device hosts:** + - not backed up by DeviceVault (Information); + - backup overdue; + - backup failing; + - backup authentication failing (Average); + - no successful backup for `{$DV.DEVICE.STALE.MAX}`; + - DeviceVault data is old; + - no data from DeviceVault (no answer, a refused token, or the relay's 503 `stale`). + + The backup triggers only fire for a device that is found, enabled and active, so a + paused or retired device stays quiet. All of them depend on that host's *no data* + trigger. + +**Request rate and staleness.** The DeviceVault host makes six requests per +`{$DV.INTERVAL}` (six a minute by default), and the figures are cached for +`DV_MONITORING_CACHE_SECONDS`. The device template makes one request per host per +`{$DV.DEVICE.INTERVAL}`: hosts รท 900 s, so 10,000 hosts make about 11 requests a +second, spread over the zone's receivers. Each answer is a dictionary lookup. Device +data can be up to one publish interval plus one poll interval old (about 30 minutes +at the defaults). That is why `{$DV.DEVICE.DATA.MAX}` defaults to 45 minutes. + +**Checking the templates.** `make zabbix-validate` imports both files into throwaway +Zabbix 6.0 and 7.0 servers (`deploy/zabbix-test/`, official `zabbix/*` images). It +links them to hosts that poll a stub serving recorded answers, and checks that every +item is supported with the recorded value and that exactly the expected triggers fire. +After changing the catalogue or `monitoring/zabbix.py`, run `make zabbix-template` and +commit the regenerated files. A test fails while they are out of date. + ## 7. High availability Run two orchestrator nodes behind a load balancer or a virtual IP. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 9901cec..28df33d 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -55,12 +55,12 @@ Small, self-contained, and each one removes a reason for an evaluator to say "no | Q1 | [Retention enforcement](#q1-retention-enforcement) | S-M | Complete: PR #58 (`5502de9`), phase 14 | Storage stops growing without bound. | | Q2 | [Notifications: webhooks and email](#q2-notifications-webhooks-and-email) | S-M | | Failures and changes reach people instead of waiting to be noticed. | | Q3 | [Change summary on every backup](#q3-change-summary-on-every-backup) | S | Complete: PR #59 (`0930972`), phase 15 | "What changed?" answered at a glance, with added/removed line counts. | -| Q4 | [Zabbix and Prometheus monitoring](#q4-monitoring-endpoints-zabbix-first-prometheus-alongside) | S-M | Scheduled: phases 22-24 (after Q9) | Backup times, success/failure counts and ratios in the monitoring stack you already run, with ready-made Zabbix templates. | -| Q5 | [Stale and missing backup detection](#q5-stale-and-missing-backup-detection) | S | | The classic silent failure: a device that stopped backing up weeks ago. | +| Q4 | [Zabbix and Prometheus monitoring](#q4-monitoring-endpoints-zabbix-first-prometheus-alongside) | S-M | Built: PRs #72-#74, phases 22-24 | Backup times, success/failure counts and ratios in the monitoring stack you already run, with ready-made Zabbix templates. | +| Q5 | [Stale and missing backup detection](#q5-stale-and-missing-backup-detection) | S | Partly built: overdue detection (phase 22, PR #72) | The classic silent failure: a device that stopped backing up weeks ago. | | Q6 | [Config search](#q6-config-search) | S-M | Complete: PRs #61, #62 (`2b4435c`, `d6b4250`), phases 16-17 | "Which devices still have SNMP community `public`?" in seconds. | | Q7 | [Bulk device import and export](#q7-bulk-device-import-and-export) | S | | Onboarding hundreds of devices from a CSV instead of a form. | | Q8 | [Pin, label and annotate backups](#q8-pin-label-and-annotate-backups) | S | Complete: PR #65 (`159e163`), phase 19 | Mark known-good versions, exempt them from retention, record why. | -| Q9 | [Personal API tokens](#q9-personal-api-tokens) | S | Scheduled: phase 21 | Automation and scripting without borrowing a browser session. | +| Q9 | [Personal API tokens](#q9-personal-api-tokens) | S | Built: PR #71, phase 21 | Automation and scripting without borrowing a browser session. | | Q10 | [Audit log export and syslog forwarding](#q10-audit-log-export-and-syslog-forwarding) | S | | Satisfies "send it to the SIEM" without a project. | | Q11 | [Add more built-in vendor drivers](#q11-more-built-in-vendor-drivers) | S each | | Each Netmiko-based driver is a few dozen lines; covers a new customer base. | | Q12 | [Backup verification probes](#q12-backup-verification-probes) | S | | Reject empty, truncated or error-page "backups" before they replace a good one. | diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index f8cb4a5..7c7887f 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -70,6 +70,7 @@ import type { ProviderTestResult, Worker, WorkerAssignment, + ZabbixVersion, } from './types' type Params = Record @@ -265,6 +266,9 @@ export const serviceAccountsApi = { export const monitoringApi = { summary: async () => (await http.get('/monitoring/summary/')).data, + /** The generated Zabbix templates for one Zabbix version, as the YAML file to import. */ + zabbixTemplate: async (version: ZabbixVersion) => + (await http.get('/monitoring/zabbix/template/', { params: { version }, responseType: 'blob' })).data, } const authProviders = resource('/auth-providers/') diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 1535d76..75d0445 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -524,6 +524,9 @@ export type ServiceAccountWrite = Pick +/** The Zabbix LTS versions the generated templates (`/api/monitoring/zabbix/template/`) are exported for. */ +export type ZabbixVersion = '6.0' | '7.0' + export type AuditOutcome = 'success' | 'failure' | 'denied' export interface AuditEntry { diff --git a/frontend/src/pages/admin/MonitoringPage.vue b/frontend/src/pages/admin/MonitoringPage.vue index b2b3417..5f9a51b 100644 --- a/frontend/src/pages/admin/MonitoringPage.vue +++ b/frontend/src/pages/admin/MonitoringPage.vue @@ -37,6 +37,25 @@ +
Zabbix templates
+
+ DeviceVault by HTTP for one host standing for DeviceVault, and + DeviceVault Device by HTTP for the device hosts Zabbix already has. Import the file for your Zabbix + version, then set {$DV.URL} and {$DV.TOKEN} as global macros. +
+
+ +
+
Current figures
@@ -72,9 +91,10 @@ import type { QTableColumn } from 'quasar' import { computed, onMounted, ref } from 'vue' import { monitoringApi } from '@/api/endpoints' -import type { MonitoringFigures } from '@/api/types' +import type { MonitoringFigures, ZabbixVersion } from '@/api/types' import PageCard from '@/components/PageCard.vue' import { useNotify } from '@/composables/useNotify' +import { saveFile } from '@/utils/download' import { formatAge, formatPercent } from '@/utils/monitoring' const notify = useNotify() @@ -82,6 +102,19 @@ const origin = window.location.origin const figures = ref(null) const loading = ref(false) const filter = ref('') +const zabbixVersions: ZabbixVersion[] = ['6.0', '7.0'] +const downloading = ref(null) + +async function downloadTemplate(version: ZabbixVersion): Promise { + downloading.value = version + try { + saveFile(await monitoringApi.zabbixTemplate(version), `devicevault_zabbix_${version}.yaml`) + } catch (error) { + notify.error(error, 'Failed to download the Zabbix templates') + } finally { + downloading.value = null + } +} const endpoints = [ { path: 'summary/', returns: 'Estate outcomes, timing, freshness, change, volume and push (platform health with that scope)' }, @@ -90,6 +123,7 @@ const endpoints = [ { path: 'workers/', returns: 'Workers: status, heartbeat age, assignment (platform health scope)' }, { path: 'storage-targets/', returns: 'Storage targets: workers online, last stored, failures, size (platform scope)' }, { path: 'receivers/', returns: 'Receivers: pushes accepted and refused, last push (platform scope)' }, + { path: 'zabbix/template/?version=7.0', returns: 'The generated Zabbix templates (6.0 or 7.0), as YAML to import' }, { path: 'metrics', returns: 'Prometheus text format: the figures above, with no per-device series' }, ] diff --git a/orchestrator/devicevault/urls.py b/orchestrator/devicevault/urls.py index 324451a..a66cb14 100644 --- a/orchestrator/devicevault/urls.py +++ b/orchestrator/devicevault/urls.py @@ -68,6 +68,7 @@ path('api/monitoring/workers/', metrics_api.workers), path('api/monitoring/storage-targets/', metrics_api.storage_targets), path('api/monitoring/receivers/', metrics_api.receivers), + path('api/monitoring/zabbix/template/', metrics_api.zabbix_template), path('api/monitoring/metrics', metrics_api.prometheus_metrics), path('api/onboarding/', views.onboarding), path('api/dashboard/summary/', dashboard.summary), diff --git a/orchestrator/monitoring/management/__init__.py b/orchestrator/monitoring/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/orchestrator/monitoring/management/commands/__init__.py b/orchestrator/monitoring/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/orchestrator/monitoring/management/commands/zabbix_template.py b/orchestrator/monitoring/management/commands/zabbix_template.py new file mode 100644 index 0000000..8de0002 --- /dev/null +++ b/orchestrator/monitoring/management/commands/zabbix_template.py @@ -0,0 +1,27 @@ +"""Write the generated Zabbix template file (monitoring.zabbix) for one Zabbix version.""" + +from pathlib import Path + +from django.core.management.base import BaseCommand + +from monitoring import zabbix + + +class Command(BaseCommand): + help = 'Write the DeviceVault Zabbix templates for Zabbix 6.0 or 7.0 (to stdout, or --output)' + + def create_parser(self, prog_name, subcommand, **kwargs): + # --version names the Zabbix version here, replacing Django's own --version. + return super().create_parser(prog_name, subcommand, conflict_handler='resolve', **kwargs) + + def add_arguments(self, parser): + parser.add_argument('--version', required=True, choices=zabbix.VERSIONS, help='Zabbix version') + parser.add_argument('--output', help='Write to this file instead of stdout') + + def handle(self, *args, version, output=None, **options): + content = zabbix.render(version) + if output: + Path(output).write_text(content) + self.stderr.write(f'Wrote the Zabbix {version} templates to {output}') + else: + self.stdout.write(content, ending='') diff --git a/orchestrator/monitoring/metrics_api.py b/orchestrator/monitoring/metrics_api.py index 5bb9c6e..41e2c04 100644 --- a/orchestrator/monitoring/metrics_api.py +++ b/orchestrator/monitoring/metrics_api.py @@ -20,7 +20,7 @@ from accounts.models import ApiToken from accounts.tokens import token_scopes from devicevault_protocol import monitoring as protocol -from monitoring import metrics, prometheus, snapshots +from monitoring import metrics, prometheus, snapshots, zabbix from monitoring.catalogue import PLATFORM, RECEIVER, TARGET, WORKER, keys SCOPES = (ApiToken.MONITORING, ApiToken.API) @@ -37,6 +37,13 @@ def render(self, data, accepted_media_type=None, renderer_context=None): return data if isinstance(data, bytes | str) else str(data) +class YamlRenderer(TextRenderer): + """Lets a download asking for YAML (``Accept: application/yaml``) through content negotiation.""" + + media_type = 'application/yaml' + format = 'yaml' + + def _scope_key(request) -> str: token = getattr(request, 'auth', None) if isinstance(token, ApiToken): @@ -146,6 +153,25 @@ def device(request): return response.Response(answer) +@token_scopes(*SCOPES) +@decorators.api_view(['GET']) +@decorators.renderer_classes([renderers.JSONRenderer, YamlRenderer]) +@decorators.permission_classes([CanReadMonitoring]) +def zabbix_template(request): + """The generated Zabbix templates (``?version=6.0|7.0``), as the YAML file Zabbix imports. + + The same for every installation and caller, since groups, workers and targets are discovered. + """ + version = request.query_params.get('version', '') + if version not in zabbix.VERSIONS: + return response.Response( + {'version': [f'Choose one of {", ".join(zabbix.VERSIONS)}.']}, status=status.HTTP_400_BAD_REQUEST + ) + answer = HttpResponse(zabbix.render(version), content_type='application/yaml; charset=utf-8') + answer['Content-Disposition'] = f'attachment; filename="devicevault_zabbix_{version}.yaml"' + return answer + + @token_scopes(*SCOPES) @decorators.api_view(['GET']) @decorators.renderer_classes([renderers.JSONRenderer, TextRenderer]) diff --git a/orchestrator/monitoring/tests/test_zabbix.py b/orchestrator/monitoring/tests/test_zabbix.py new file mode 100644 index 0000000..86f870e --- /dev/null +++ b/orchestrator/monitoring/tests/test_zabbix.py @@ -0,0 +1,174 @@ +import json +import re +import uuid +from pathlib import Path + +import pytest +import yaml +from django.core.management import call_command +from rest_framework.test import APIClient + +from monitoring import catalogue, zabbix + +DEPLOY = Path(__file__).resolve().parents[3] / 'deploy' +HOST_SCOPES = (catalogue.ESTATE, catalogue.PLATFORM, catalogue.GROUP, catalogue.WORKER, catalogue.TARGET, + catalogue.RECEIVER) # fmt: skip + + +def templates(version): + document = yaml.safe_load(zabbix.render(version))['zabbix_export'] + return document, {template['template']: template for template in document['templates']} + + +def item_keys(template): + keys = {item['key'] for item in template.get('items', [])} + for rule in template.get('discovery_rules', []): + keys |= {item['key'] for item in rule['item_prototypes']} + return keys + + +def all_triggers(document, template): + found = list(document.get('triggers', [])) + for item in template.get('items', []): + found += item.get('triggers', []) + for rule in template.get('discovery_rules', []): + found += rule.get('trigger_prototypes', []) + for item in rule['item_prototypes']: + found += item.get('trigger_prototypes', []) + return [trigger for trigger in found if f'/{template["template"]}/' in trigger['expression']] + + +@pytest.mark.parametrize('version', zabbix.VERSIONS) +def test_the_committed_templates_are_the_generator_output(version): + committed = zabbix.default_path(version) + + assert committed.read_text() == zabbix.render(version), ( + f'{committed.name} is out of date: run `make zabbix-template` and commit the result' + ) + + +@pytest.mark.parametrize('version', zabbix.VERSIONS) +def test_the_export_has_the_versions_shape(version): + document, by_name = templates(version) + + assert document['version'] == version + assert ('groups' if version == '6.0' else 'template_groups') in document + assert set(by_name) == {zabbix.HOST_TEMPLATE, zabbix.DEVICE_TEMPLATE} + widgets = by_name[zabbix.HOST_TEMPLATE]['dashboards'][0]['pages'][0]['widgets'] + assert {widget['type'] for widget in widgets} == ( + {'GRAPH_CLASSIC', 'GRAPH_PROTOTYPE'} if version == '6.0' else {'graph', 'graphprototype'} + ) + + +@pytest.mark.parametrize('version', zabbix.VERSIONS) +def test_every_catalogue_metric_is_an_item_or_prototype(version): + _, by_name = templates(version) + host, device = item_keys(by_name[zabbix.HOST_TEMPLATE]), item_keys(by_name[zabbix.DEVICE_TEMPLATE]) + discoveries = {d.scope: d for d in zabbix.DISCOVERIES} + + for scope in HOST_SCOPES: + for metric in catalogue.metrics(scope): + if scope in discoveries: + if metric.key in zabbix.LLD_MACROS: + continue # {#ID} and {#NAME} + assert zabbix.prototype_key(discoveries[scope], metric.key) in host, metric.key + else: + assert zabbix.estate_key(metric.key) in host, metric.key + for metric in catalogue.metrics(catalogue.DEVICE): + assert zabbix.device_key(metric.key) in device, metric.key + + +def test_items_follow_the_catalogue_units_value_maps_and_nulls(): + _, by_name = templates('7.0') + items = {item['key']: item for item in by_name[zabbix.HOST_TEMPLATE]['items']} + + ratio = items['dv.success_ratio_24h'] + assert ratio['units'] == '%' and ratio['value_type'] == 'FLOAT' + assert [step['type'] for step in ratio['preprocessing']] == ['JSONPATH', 'NOT_MATCHES_REGEX'] + assert all(step['error_handler'] == 'DISCARD_VALUE' for step in ratio['preprocessing']) + assert items['dv.push_last_at']['units'] == 'unixtime' and items['dv.bytes_held']['units'] == 'B' + assert items['dv.payload_encryption']['valuemap'] == {'name': 'DeviceVault yes/no'} + assert items['dv.version']['preprocessing'][-1]['type'] == 'DISCARD_UNCHANGED_HEARTBEAT' + maps = {vm['name']: vm for vm in by_name[zabbix.DEVICE_TEMPLATE]['valuemaps']} + reasons = {m['value']: m['newvalue'] for m in maps['DeviceVault failure reason']['mappings']} + assert reasons == {str(code): label for code, label in catalogue.VALUE_MAPS['failure_reason'].items()} + + +@pytest.mark.parametrize('version', zabbix.VERSIONS) +def test_references_resolve(version): + document, by_name = templates(version) + + for template in by_name.values(): + keys = item_keys(template) + rules = template.get('discovery_rules', []) + masters = [*template.get('items', []), *[i for rule in rules for i in rule['item_prototypes']], *rules] + assert {item['master_item']['key'] for item in masters if 'master_item' in item} <= keys + triggers = all_triggers(document, template) + names = {trigger['name'] for trigger in triggers} + for trigger in triggers: + for expression in (trigger['expression'], trigger['recovery_expression']): + used = set(re.findall(rf'/{re.escape(template["template"])}/([^,)]+)', expression)) + assert used and used <= keys, (trigger['name'], used - keys) + assert {dependency['name'] for dependency in trigger.get('dependencies', [])} <= names + assert {tag['tag'] for tag in trigger['tags']} == {'component', 'scope', 'group'} + maps = {vm['name'] for vm in template['valuemaps']} + assert {item['valuemap']['name'] for item in masters if 'valuemap' in item} <= maps + for rule in by_name[zabbix.HOST_TEMPLATE]['discovery_rules']: + assert rule['lifetime'] == '7d' and rule['type'] == 'DEPENDENT' + + +@pytest.mark.parametrize('version', zabbix.VERSIONS) +def test_uuids_are_stable_unique_version_4(version): + text = zabbix.render(version) + found = re.findall(r'uuid: ([0-9a-f]+)', text) + + assert len(found) == len(set(found)) > 100 + assert all(uuid.UUID(value).version == 4 for value in found) + assert zabbix.render(version) == text + + +def test_the_device_template_polls_the_relay_with_the_token(): + _, by_name = templates('6.0') + master = next(item for item in by_name[zabbix.DEVICE_TEMPLATE]['items'] if item['type'] == 'HTTP_AGENT') + macros = {macro['macro'] for template in by_name.values() for macro in template['macros']} + + assert master['url'] == '{$DV.RELAY.URL}/device/?ip={HOST.IP}&id={$DV.DEVICE.ID}' + assert master['headers'] == [{'name': 'Authorization', 'value': 'Bearer {$DV.TOKEN}'}] + assert master['verify_peer'] == 'NO' and master['status_codes'] == '200,503' + assert '{$DV.TOKEN}' not in macros and '{$DV.URL}' not in macros # global macros, not shadowed + + +def test_the_recorded_validation_responses_have_the_catalogue_keys(): + responses = DEPLOY / 'zabbix-test' / 'responses' + + def load(name): + return json.loads((responses / f'{name}.json').read_text()) + + summary = set(catalogue.keys(catalogue.ESTATE)) | set(catalogue.keys(catalogue.PLATFORM)) + assert set(load('summary')) == summary + for name, scope in [('device-groups', catalogue.GROUP), ('workers', catalogue.WORKER), + ('storage-targets', catalogue.TARGET), ('receivers', catalogue.RECEIVER)]: # fmt: skip + assert all(set(entry) == set(catalogue.keys(scope)) for entry in load(name)['data']), name + assert set(load('device')) == set(catalogue.keys(catalogue.DEVICE)) + + +def test_the_command_writes_a_version(tmp_path): + out = tmp_path / 'template.yaml' + + call_command('zabbix_template', '--version', '7.0', '--output', str(out)) + + assert out.read_text() == zabbix.render('7.0') + + +def test_the_endpoint_serves_the_template_as_a_download(setup, service_account, make_token, token_client): + _, raw = make_token(service_account(), scopes=['monitoring']) + client = token_client(raw) + + answer = client.get('/api/monitoring/zabbix/template/?version=6.0') + + assert answer.status_code == 200 and answer['Content-Type'].startswith('application/yaml') + assert answer['Content-Disposition'] == 'attachment; filename="devicevault_zabbix_6.0.yaml"' + assert answer.content.decode() == zabbix.render('6.0') + assert client.get('/api/monitoring/zabbix/template/?version=6.4').status_code == 400 + assert client.get('/api/monitoring/zabbix/template/', HTTP_ACCEPT='application/yaml').status_code == 400 + assert APIClient().get('/api/monitoring/zabbix/template/?version=7.0').status_code == 401 diff --git a/orchestrator/monitoring/zabbix.py b/orchestrator/monitoring/zabbix.py new file mode 100644 index 0000000..0cfddcd --- /dev/null +++ b/orchestrator/monitoring/zabbix.py @@ -0,0 +1,960 @@ +"""The Zabbix templates, generated from the metric catalogue (``monitoring.catalogue``). + +Two templates, exported for Zabbix 6.0 and 7.0 LTS (``render(version)``): + +- ``DeviceVault by HTTP``, for one host standing for DeviceVault. HTTP agent items poll + ``summary/``, ``device-groups/``, ``workers/``, ``storage-targets/`` and ``receivers/``; + every value is a dependent item, and dependent low-level discovery rules turn the lists + into per-group, per-worker, per-target and per-receiver item and trigger prototypes. +- ``DeviceVault Device by HTTP``, linked to the device hosts Zabbix already has. One HTTP + agent item asks a receiver relay (or the orchestrator) for ``device/?ip={HOST.IP}``. + +The content is built once (``_Template``) and written by a small emitter per version, which +differ in the export ``version``, template groups and dashboard widgets. UUIDs are derived +from stable names, so regenerating changes nothing that did not change. ``{$DV.URL}`` and +``{$DV.TOKEN}`` are deliberately not template macros: a template macro would shadow the +global macro the deployment guide sets them as. + +``deploy/zabbix/devicevault_zabbix_.yaml`` hold the output (``make zabbix-template``), +and a test fails when they differ from it. +""" + +import re +import uuid +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +from devicevault_protocol import jobs +from monitoring import catalogue as cat +from monitoring.catalogue import VALUE_MAPS, Metric, code + +VERSIONS = ('6.0', '7.0') +HOST_TEMPLATE = 'DeviceVault by HTTP' +DEVICE_TEMPLATE = 'DeviceVault Device by HTTP' +TEMPLATE_GROUP = 'Templates/Applications' +API_PATH = '/api/monitoring/' + +_NAMESPACE = uuid.UUID('9b6d3c1e-58a4-4f0e-b1d7-2f4c8e6a0d35') + +VALUEMAP_NAMES = { + 'job_status': 'DeviceVault job status', + 'failure_reason': 'DeviceVault failure reason', + 'lifecycle': 'DeviceVault lifecycle', + 'worker_status': 'DeviceVault worker status', + 'boolean': 'DeviceVault yes/no', +} + +# A value that is not there (a platform key without the platform scope, a device that is not +# found) is discarded rather than turning the item unsupported; so is a JSON null (a ratio or +# duration with nothing to measure), which JSONPath returns as ``null`` or an empty string. +NULL = '^(null)?$' +# Text and value-mapped items rarely change, so unchanged values are kept only hourly, except +# those a trigger reads over a time window (which needs a value in every poll). +HEARTBEAT = '1h' +WINDOWED = {'assignment_behind'} +TIMEOUT = '15s' +COLOURS = ('199C0D', 'F63100', '2774A4', 'F7941D', 'FC6EA3', '6C59DC', 'C7A72D', 'BA2A5D', '0080FF', '808080') + +ESTATE = 'all' # the ``group`` tag of triggers that are not about one device group + + +def _uuid(*parts: str) -> str: + """A UUID that only changes when ``parts`` do (Zabbix accepts only version 4 UUIDs).""" + return uuid.UUID(bytes=uuid.uuid5(_NAMESPACE, '\x1f'.join(parts)).bytes, version=4).hex + + +def default_path(version: str) -> Path: + return Path(__file__).resolve().parents[2] / 'deploy' / 'zabbix' / f'devicevault_zabbix_{version}.yaml' + + +# --- discovery ----------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Discovery: + scope: str + slug: str + title: str + label: str + path: str + + +DISCOVERIES = ( + Discovery(cat.GROUP, 'group', 'Device groups', 'Group', 'device-groups/'), + Discovery(cat.WORKER, 'worker', 'Workers', 'Worker', 'workers/'), + Discovery(cat.TARGET, 'target', 'Storage targets', 'Storage target', 'storage-targets/'), + Discovery(cat.RECEIVER, 'receiver', 'Receivers', 'Receiver', 'receivers/'), +) +# Identity fields of discovered entries travel as LLD macros, not as items. +LLD_MACROS = {'id': '{#ID}', 'name': '{#NAME}'} + + +def master_key(discovery: Discovery) -> str: + return f'dv.{discovery.slug}.get' + + +def estate_key(key: str) -> str: + return f'dv.{key}' + + +def prototype_key(discovery: Discovery, key: str) -> str: + return f'dv.{discovery.slug}.{key}[{{#ID}}]' + + +def device_key(key: str) -> str: + return f'dv.device.{key}' + + +SUMMARY_KEY = 'dv.summary.get' +STATUS_KEY = 'dv.http.status' +DEVICE_MASTER_KEY = 'dv.device.get' +DEVICE_ERROR_KEY = 'dv.device.error' + + +# --- content ------------------------------------------------------------------------------- + + +@dataclass +class Trigger: + name: str + expression: str + recovery: str + priority: str + description: str + component: str + scope: str + group: str = ESTATE + depends: list[Trigger] = field(default_factory=list) + opdata: str = '' + + def reference(self) -> dict: + return {'name': self.name, 'expression': self.expression, 'recovery_expression': self.recovery} + + +@dataclass +class Graph: + name: str + keys: list[str] + stacked: bool = False + + +@dataclass +class Rule: + """A dependent low-level discovery rule and its prototypes.""" + + discovery: Discovery + items: list[dict] = field(default_factory=list) + triggers: list[Trigger] = field(default_factory=list) + graphs: list[Graph] = field(default_factory=list) + + +@dataclass +class _Template: + name: str + description: str + items: list[dict] = field(default_factory=list) + rules: list[Rule] = field(default_factory=list) + triggers: list[Trigger] = field(default_factory=list) + graphs: list[Graph] = field(default_factory=list) + macros: list[dict] = field(default_factory=list) + dashboard: list[tuple[str, str]] = field(default_factory=list) # (graph or prototype, name) + + def ref(self, key: str) -> str: + return f'/{self.name}/{key}' + + def valuemaps(self) -> list[str]: + used = {item['valuemap']['name'] for item in self.items if 'valuemap' in item} + used |= {item['valuemap']['name'] for rule in self.rules for item in rule.items if 'valuemap' in item} + return [name for value_map, name in VALUEMAP_NAMES.items() if name in used] + + +def _value_type(metric: Metric) -> str: + if metric.value_type == 'text': + return 'CHAR' + if metric.value_type == 'float' or metric.unit == 's': # ages may be -1 (never) + return 'FLOAT' + return 'UNSIGNED' + + +def _component(metric: Metric) -> str: + key = metric.key + for prefix, component in [ + ('duration_', 'timing'), + ('failures_', 'failures'), + ('devices_', 'freshness'), + ('oldest_', 'freshness'), + ('push_', 'push'), + ('lines_', 'changes'), + ('changed_', 'changes'), + ('bytes_', 'volume'), + ('backups_held', 'volume'), + ('workers_', 'workers'), + ('scheduler_', 'scheduler'), + ]: + if key.startswith(prefix): + return component + if metric.scope == cat.ESTATE and key in ('version', 'payload_encryption', 'server_time', 'computed_at', + 'computed_age'): # fmt: skip + return 'system' + return 'backups' + + +def _tags(**tags: str) -> list[dict]: + return [{'tag': tag, 'value': value} for tag, value in tags.items()] + + +def _dependent(template: str, metric: Metric, key: str, name: str, master: str, path: str, tags: list[dict]) -> dict: + item = { + 'uuid': _uuid(template, 'item', key), + 'name': name, + 'type': 'DEPENDENT', + 'key': key, + 'delay': '0', + 'value_type': _value_type(metric), + } + if metric.value_type == 'text': + item['trends'] = '0' + if metric.unit: + item['units'] = metric.unit + if metric.description: + item['description'] = metric.description + if metric.value_map: + item['valuemap'] = {'name': VALUEMAP_NAMES[metric.value_map]} + steps = [{'type': 'JSONPATH', 'parameters': [path], 'error_handler': 'DISCARD_VALUE'}] + if metric.value_type == 'float': + steps.append({'type': 'NOT_MATCHES_REGEX', 'parameters': [NULL], 'error_handler': 'DISCARD_VALUE'}) + if metric.value_type == 'text' or (metric.value_map and metric.key not in WINDOWED): + steps.append({'type': 'DISCARD_UNCHANGED_HEARTBEAT', 'parameters': [HEARTBEAT]}) + item['preprocessing'] = steps + item['master_item'] = {'key': master} + item['tags'] = tags + return item + + +def _http(template: str, key: str, name: str, url: str, description: str, delay: str, **extra) -> dict: + item = { + 'uuid': _uuid(template, 'item', key), + 'name': name, + 'type': 'HTTP_AGENT', + 'key': key, + 'delay': delay, + 'history': '0', + 'trends': '0', + 'value_type': 'TEXT', + 'description': description, + 'timeout': TIMEOUT, + 'url': url, + 'headers': [{'name': 'Authorization', 'value': 'Bearer {$DV.TOKEN}'}], + 'tags': _tags(component='raw'), + } + item.update(extra) + return item + + +def _macro(macro: str, value: str, description: str) -> dict: + out = {'macro': macro} + if value: + out['value'] = value + out['description'] = description + return out + + +# --- DeviceVault by HTTP ------------------------------------------------------------------- + + +HOST_DESCRIPTION = """\ +Backup health of a DeviceVault installation, polled from its read-only monitoring API. + +Link it to one host standing for DeviceVault, monitored by the Zabbix server or a proxy that can reach the web tier. +Set {$DV.URL} (the web address, without /api) and {$DV.TOKEN} (a service account's token with the monitoring scope, +plus monitoring_platform for workers, the scheduler, storage targets and receivers) as global macros, or on the host. + +Generated by DeviceVault from its metric catalogue; see DEPLOYMENT.md, section Monitoring.""" + +HOST_MACROS = [ + _macro('{$DV.INTERVAL}', '1m', 'How often the monitoring endpoints are polled.'), + _macro('{$DV.NODATA}', '5m', 'No answer from the monitoring API for this long raises the unreachable trigger.'), + _macro('{$DV.SUCCESS.MIN}', '95', 'Lowest acceptable share of backups stored over 24h, in percent.'), + _macro('{$DV.STALE.MAX}', '2d', 'Longest acceptable age of the oldest last successful backup in a device group.'), + _macro( + '{$DV.FAIL.CONSEC}', '3', 'Consecutive failures that count a device as failing (rounded down to 1, 3 or 5).' + ), + _macro('{$DV.DURATION.MAX}', '10m', 'Longest acceptable 95th percentile request-to-stored time over 24h.'), + _macro('{$DV.WORKER.HEARTBEAT.MAX}', '3m', "Longest acceptable gap between an approved worker's heartbeats."), + _macro('{$DV.PENDING.MAX}', '1h', 'Longest a worker may wait for approval before it is reported.'), + _macro( + '{$DV.SKEW.MAX}', '60', 'Largest acceptable difference between the DeviceVault and Zabbix clocks, in seconds.' + ), + _macro('{$DV.PUSH.REFUSED.MAX}', '10', 'More refused pushes than this within an hour raise a trigger.'), +] + + +def _failing(template: _Template, key) -> tuple[str, str]: + """``{$DV.FAIL.CONSEC}``+ failing devices, from the counts the catalogue has (1+, 3+, 5+).""" + bounds = list(cat.FAILING) + [None] + fire, clear = [], [] + for low, high in zip(bounds, bounds[1:], strict=False): + cond = f'{{$DV.FAIL.CONSEC}}>={low}' if low != bounds[0] else '' + if high is not None: + cond = f'{cond} and {{$DV.FAIL.CONSEC}}<{high}' if cond else f'{{$DV.FAIL.CONSEC}}<{high}' + last = f'last({template.ref(key(f"devices_failing_{low}"))})' + fire.append(f'({cond} and {last}>0)') + clear.append(f'({cond} and {last}=0)') + return ' or '.join(fire), ' or '.join(clear) + + +def _ratio_trigger(t: _Template, key, where: str, group: str, depends: list[Trigger]) -> Trigger: + ratio, finished = t.ref(key('success_ratio_24h')), t.ref(key('finished_24h')) + return Trigger( + f'DeviceVault: {where}success ratio below {{$DV.SUCCESS.MIN}}% (24h)', + f'last({ratio})<{{$DV.SUCCESS.MIN}} and last({finished})>0', + f'last({ratio})>={{$DV.SUCCESS.MIN}} or last({finished})=0', + 'WARNING', + 'Fewer backups than {$DV.SUCCESS.MIN}% of those that finished in the last 24h were stored.', + 'backups', + 'notice', + group, + depends, + opdata='Success ratio: {ITEM.LASTVALUE1}', + ) + + +def _host_template() -> _Template: + t = _Template(HOST_TEMPLATE, HOST_DESCRIPTION, macros=HOST_MACROS) + name = t.name + url = f'{{$DV.URL}}{API_PATH}' + + t.items.append( + _http( + name, SUMMARY_KEY, 'DeviceVault: Get summary', f'{url}summary/', + 'Estate outcomes, timing, freshness, change, volume, push and self, and platform health with that scope.', + '{$DV.INTERVAL}', + ) + ) # fmt: skip + t.items.append( + { + 'uuid': _uuid(name, 'item', STATUS_KEY), + 'name': 'DeviceVault: API status', + 'type': 'HTTP_AGENT', + 'key': STATUS_KEY, + 'delay': '{$DV.INTERVAL}', + 'description': 'The HTTP status the summary endpoint answers with (200 when the token is accepted).', + 'timeout': TIMEOUT, + 'url': f'{url}summary/', + 'status_codes': '', + 'headers': [{'name': 'Authorization', 'value': 'Bearer {$DV.TOKEN}'}], + 'retrieve_mode': 'HEADERS', + 'preprocessing': [{'type': 'REGEX', 'parameters': ['^HTTP\\/[\\d.]+\\s+(\\d+)', '\\1']}], + 'tags': _tags(component='api'), + } + ) + for discovery in DISCOVERIES: + t.items.append( + _http( + name, master_key(discovery), f'DeviceVault: Get {discovery.title.lower()}', f'{url}{discovery.path}', + f'The {discovery.title.lower()} list ({discovery.path}), read by discovery.', '{$DV.INTERVAL}', + ) + ) # fmt: skip + + for metric in cat.metrics(cat.ESTATE) + cat.metrics(cat.PLATFORM): + component = 'platform' if metric.scope == cat.PLATFORM else _component(metric) + t.items.append( + _dependent( + name, metric, estate_key(metric.key), f'DeviceVault: {metric.name}', SUMMARY_KEY, + f'$.{metric.key}', _tags(component=component), + ) + ) # fmt: skip + + ref = t.ref + endpoint = Trigger( + 'DeviceVault: monitoring API unreachable or refusing', + f'last({ref(STATUS_KEY)})<>200 or nodata({ref(STATUS_KEY)},{{$DV.NODATA}})=1', + f'last({ref(STATUS_KEY)})=200 and nodata({ref(STATUS_KEY)},{{$DV.NODATA}})=0', + 'HIGH', + 'The summary endpoint did not answer for {$DV.NODATA}, or answered with an error (401 or 403: the token is ' + 'unknown, revoked, expired, out of its networks or lacks the monitoring scope; 5xx: DeviceVault is failing).', + 'api', + 'availability', + opdata='HTTP status: {ITEM.LASTVALUE1}', + ) + stale = Trigger( + 'DeviceVault: monitoring figures are stale', + f'last({ref(estate_key("computed_age"))})>3*{{$DV.INTERVAL}} or nodata({ref(estate_key("server_time"))},{{$DV.NODATA}})=1', + f'last({ref(estate_key("computed_age"))})<=3*{{$DV.INTERVAL}} and nodata({ref(estate_key("server_time"))},{{$DV.NODATA}})=0', + 'WARNING', + 'The figures are older than three poll intervals, or none arrived for {$DV.NODATA}.', + 'api', + 'availability', + depends=[endpoint], + ) # fmt: skip + base = [endpoint, stale] + estate = lambda key: estate_key(key) # noqa: E731 + leader = Trigger( + 'DeviceVault: scheduler has no leader', + f'last({ref(estate("scheduler_leader"))})=0', + f'last({ref(estate("scheduler_leader"))})=1', + 'HIGH', + 'No scheduler node holds the lease, so no scheduled backup starts.', + 'scheduler', + 'availability', + depends=base, + ) + t.triggers += [ + endpoint, + stale, + _ratio_trigger(t, estate, '', ESTATE, base), + Trigger( + 'DeviceVault: devices never backed up', + f'last({ref(estate("devices_never"))})>0', + f'last({ref(estate("devices_never"))})=0', + 'WARNING', + 'Active, enabled devices have no stored backup yet.', + 'freshness', + 'notice', + depends=base, + opdata='Devices: {ITEM.LASTVALUE1}', + ), + Trigger( + 'DeviceVault: backups are slow (p95 over {$DV.DURATION.MAX})', + f'last({ref(estate("duration_total_p95"))})>{{$DV.DURATION.MAX}} and last({ref(estate("finished_24h"))})>0', + f'last({ref(estate("duration_total_p95"))})<={{$DV.DURATION.MAX}} or last({ref(estate("finished_24h"))})=0', + 'WARNING', + 'The 95th percentile request-to-stored time of the last 24h is above {$DV.DURATION.MAX}.', + 'timing', + 'performance', + depends=base, + opdata='p95: {ITEM.LASTVALUE1}', + ), + Trigger( + 'DeviceVault: clock differs from Zabbix by more than {$DV.SKEW.MAX}s', + f'fuzzytime({ref(estate("server_time"))},{{$DV.SKEW.MAX}})=0', + f'fuzzytime({ref(estate("server_time"))},{{$DV.SKEW.MAX}})=1', + 'WARNING', + "The orchestrator's clock and the Zabbix server's differ; schedules and ages will be off.", + 'system', + 'availability', + depends=base, + ), + Trigger( + 'DeviceVault: push refusals spiking', + f'last({ref(estate("push_refused_24h"))})-last({ref(estate("push_refused_24h"))},#1:now-1h)>{{$DV.PUSH.REFUSED.MAX}}', + f'last({ref(estate("push_refused_24h"))})-last({ref(estate("push_refused_24h"))},#1:now-1h)<={{$DV.PUSH.REFUSED.MAX}}', + 'WARNING', + 'Receivers refused more than {$DV.PUSH.REFUSED.MAX} pushes in the last hour (unknown sources, failed ' + 'authentication, wrong protocol...). See the Push refusals page.', + 'push', + 'security', + depends=base, + ), # fmt: skip + leader, + Trigger( + 'DeviceVault: scheduler is not ticking', + f'last({ref(estate("scheduler_tick_age"))})>2m or last({ref(estate("scheduler_tick_age"))})<0', + f'last({ref(estate("scheduler_tick_age"))})>=0 and last({ref(estate("scheduler_tick_age"))})<=2m', + 'AVERAGE', + 'The scheduler leader has not run its tick for over 2 minutes.', + 'scheduler', + 'availability', + depends=[*base, leader], + ), + ] + t.graphs = [ + Graph('DeviceVault: Success ratio', [estate(f'success_ratio_{w}') for w in cat.WINDOWS]), + Graph('DeviceVault: Request to stored time (24h)', [estate(f'duration_total_{s}') for s in cat.STATS]), + Graph('DeviceVault: Failures by reason (24h)', [estate(f'failures_{r}_24h') for r in jobs.FAILURE_REASONS], + stacked=True), + Graph('DeviceVault: Device freshness', [estate(k) for k in ('devices_overdue', 'devices_never')] + + [estate(f'devices_failing_{n}') for n in cat.FAILING]), + Graph('DeviceVault: Platform health', [estate(f'workers_{s}') for s in ('online', 'offline', 'pending')] + + [estate('workers_behind'), estate('scheduler_leader')]), + ] # fmt: skip + + for discovery in DISCOVERIES: + rule = Rule(discovery) + key = lambda k, d=discovery: prototype_key(d, k) # noqa: E731 + for metric in cat.metrics(discovery.scope): + if metric.key in LLD_MACROS: + continue + component = discovery.slug if discovery.scope != cat.GROUP else _component(metric) + tags = _tags(component=component, **{discovery.slug: '{#NAME}'}) + rule.items.append( + _dependent( + name, metric, key(metric.key), f'{discovery.label} {{#NAME}}: {metric.name}', + master_key(discovery), f"$.data[?(@.id == '{{#ID}}')].{metric.key}.first()", tags, + ) + ) # fmt: skip + rule.triggers = _prototype_triggers(t, discovery, key, base) + if discovery.scope == cat.GROUP: + rule.graphs = [ + Graph('Group {#NAME}: Device freshness', [key('devices_overdue'), key('devices_never')] + + [key(f'devices_failing_{n}') for n in cat.FAILING]), + Graph('Group {#NAME}: Success ratio', [key(f'success_ratio_{w}') for w in cat.WINDOWS]), + ] # fmt: skip + t.rules.append(rule) + + t.dashboard = [('graph', graph.name) for graph in t.graphs] + [ + ('prototype', 'Group {#NAME}: Device freshness'), + ('prototype', 'Group {#NAME}: Success ratio'), + ] + return t + + +def _prototype_triggers(t: _Template, discovery: Discovery, key, base: list[Trigger]) -> list[Trigger]: + ref = t.ref + label = f'{discovery.label.lower()} {{#NAME}}' + if discovery.scope == cat.GROUP: + failing, recovered = _failing(t, key) + return [ + _ratio_trigger(t, key, f'{label}: ', '{#NAME}', base), + Trigger( + f'DeviceVault: {label}: a device has not backed up for over {{$DV.STALE.MAX}}', + f'last({ref(key("oldest_success_age"))})>{{$DV.STALE.MAX}}', + f'last({ref(key("oldest_success_age"))})<={{$DV.STALE.MAX}}', + 'AVERAGE', + 'The stalest active, enabled device of the group last backed up longer ago than {$DV.STALE.MAX}.', + 'freshness', + 'notice', + '{#NAME}', + base, + opdata='Age: {ITEM.LASTVALUE1}', + ), + Trigger( + f'DeviceVault: {label}: devices overdue', + f'last({ref(key("devices_overdue"))})>0', + f'last({ref(key("devices_overdue"))})=0', + 'WARNING', + 'Devices of the group have no successful backup as recent as their schedule allows.', + 'freshness', + 'notice', + '{#NAME}', + base, + opdata='Devices: {ITEM.LASTVALUE1}', + ), + Trigger( + f'DeviceVault: {label}: devices failing {{$DV.FAIL.CONSEC}}+ times in a row', + failing, + recovered, + 'WARNING', + 'Devices of the group failed their last {$DV.FAIL.CONSEC} or more backups (counted at 1, 3 or 5).', + 'failures', + 'notice', + '{#NAME}', + base, + ), + ] + if discovery.scope == cat.WORKER: + approved = code('worker_status', 'approved') + status, heartbeat = ref(key('status_code')), ref(key('heartbeat_age')) + return [ + Trigger( + f'DeviceVault: {label}: no heartbeat for {{$DV.WORKER.HEARTBEAT.MAX}}', + f'last({status})={approved} and (last({heartbeat})>{{$DV.WORKER.HEARTBEAT.MAX}} or last({heartbeat})<0)', + f'last({status})<>{approved} or (last({heartbeat})>=0 and last({heartbeat})<={{$DV.WORKER.HEARTBEAT.MAX}})', + 'AVERAGE', + 'An approved worker has not sent a heartbeat recently: it is down or cannot reach the broker.', + 'worker', + 'availability', + depends=base, + opdata='Heartbeat age: {ITEM.LASTVALUE2}', + ), + Trigger( + f'DeviceVault: {label}: pending approval for over {{$DV.PENDING.MAX}}', + f'last({ref(key("pending_age"))})>{{$DV.PENDING.MAX}}', + f'last({ref(key("pending_age"))})<={{$DV.PENDING.MAX}}', + 'WARNING', + 'A worker registered and is waiting for an administrator to approve or revoke it (Workers page).', + 'worker', + 'notice', + depends=base, + ), + Trigger( + f'DeviceVault: {label}: assignment behind for 10 minutes', + f'min({ref(key("assignment_behind"))},10m)=1 and last({ref(key("assignment_behind"))},#1:now-10m)=1', + f'last({ref(key("assignment_behind"))})=0', + 'WARNING', + 'The worker has not applied its current assignment for 10 minutes.', + 'worker', + 'notice', + depends=base, + ), + ] # fmt: skip + if discovery.scope == cat.TARGET: + return [ + Trigger( + f'DeviceVault: {label}: no storage worker online', + f'last({ref(key("workers_online"))})=0', + f'last({ref(key("workers_online"))})>0', + 'HIGH', + 'No approved, online storage worker serves the target, so backups for it cannot be stored.', + 'storage', + 'availability', + depends=base, + ) + ] + return [] + + +# --- DeviceVault Device by HTTP ------------------------------------------------------------ + + +DEVICE_DESCRIPTION = """\ +DeviceVault backup status of this device, matched on its address ({HOST.IP}) or {$DV.DEVICE.ID}. + +Link it to the device hosts Zabbix already monitors. {$DV.RELAY.URL} is the monitoring relay on a DeviceVault receiver +in the zone of the proxy that monitors the host (set it once per zone on a site template that links this one), or +{$DV.URL}/api/monitoring where the proxy can reach DeviceVault directly. {$DV.TOKEN} (a service account's token with +the monitoring scope) is meant to be a global macro. + +Generated by DeviceVault from its metric catalogue; see DEPLOYMENT.md, section Monitoring.""" + +DEVICE_MACROS = [ + _macro('{$DV.RELAY.URL}', 'https://receiver-site-a:8443', 'The monitoring relay of a receiver in this zone (or ' + '{$DV.URL}/api/monitoring for the orchestrator). Set it once per zone on a site template.'), + _macro('{$DV.DEVICE.ID}', '', "DeviceVault's id for this device, for a host whose interface address is not the " + 'one DeviceVault backs up. Empty: match on {HOST.IP}.'), + _macro('{$DV.DEVICE.INTERVAL}', '15m', 'How often the device status is polled (snapshots are published every 15m).'), + _macro('{$DV.DEVICE.NODATA}', '45m', 'No answer for this long (three poll intervals) raises the no-data trigger.'), + _macro('{$DV.NOTFOUND.ALERT}', '1', '1 reports hosts DeviceVault does not back up; 0 switches that trigger off.'), + _macro('{$DV.DEVICE.FAIL.CONSEC}', '2', 'Consecutive failed backups that raise the failing trigger.'), + _macro('{$DV.DEVICE.STALE.MAX}', '0', 'Raise a trigger when the last successful backup is older than this (for ' + 'example 2d); 0 switches it off in favour of the schedule-based overdue trigger.'), + _macro('{$DV.DEVICE.DATA.MAX}', '45m', 'Raise a trigger when the published snapshot answering is older than this.'), +] # fmt: skip + + +def _device_template() -> _Template: + t = _Template(DEVICE_TEMPLATE, DEVICE_DESCRIPTION, macros=DEVICE_MACROS) + name = t.name + t.items.append( + _http( + name, DEVICE_MASTER_KEY, 'DeviceVault: Get device status', + '{$DV.RELAY.URL}/device/?ip={HOST.IP}&id={$DV.DEVICE.ID}', + "The device's answer from DeviceVault's published snapshot (503 when the snapshot is stale).", + '{$DV.DEVICE.INTERVAL}', history='1h', status_codes='200,503', verify_peer='NO', verify_host='NO', + ) + ) # fmt: skip + t.items.append( + { + 'uuid': _uuid(name, 'item', DEVICE_ERROR_KEY), + 'name': 'DeviceVault: Answer error', + 'type': 'DEPENDENT', + 'key': DEVICE_ERROR_KEY, + 'delay': '0', + 'value_type': 'CHAR', + 'trends': '0', + 'description': '"stale" when the relay has no current snapshot (503), otherwise "none".', + 'preprocessing': [ + { + 'type': 'JSONPATH', + 'parameters': ['$.error'], + 'error_handler': 'CUSTOM_VALUE', + 'error_handler_params': 'none', + }, + ], # fmt: skip + 'master_item': {'key': DEVICE_MASTER_KEY}, + 'tags': _tags(component='api'), + } + ) + for metric in cat.metrics(cat.DEVICE): + t.items.append( + _dependent( + name, metric, device_key(metric.key), f'DeviceVault: {metric.name}', DEVICE_MASTER_KEY, + f'$.{metric.key}', _tags(component='backup'), + ) + ) # fmt: skip + + ref = lambda key: t.ref(device_key(key)) # noqa: E731 + master = t.ref(DEVICE_MASTER_KEY) + error = t.ref(DEVICE_ERROR_KEY) + nodata = Trigger( + 'DeviceVault: no data from DeviceVault', + f'nodata({master},{{$DV.DEVICE.NODATA}})=1 or last({error})="stale"', + f'nodata({master},{{$DV.DEVICE.NODATA}})=0 and last({error})<>"stale"', + 'WARNING', + 'The relay at {$DV.RELAY.URL} did not answer for {$DV.DEVICE.NODATA}, refused the token, or has no current ' + 'snapshot (503 stale). The other DeviceVault triggers of this host depend on this one.', + 'api', + 'availability', + ) + found, enabled, lifecycle = ref('found'), ref('enabled'), ref('lifecycle_code') + active = code('lifecycle', 'active') + gate = f'last({found})=1 and last({enabled})=1 and last({lifecycle})={active}' + ungate = f'last({found})<>1 or last({enabled})<>1 or last({lifecycle})<>{active}' + auth = code('failure_reason', jobs.FAILURE_AUTH) + t.triggers = [ + nodata, + Trigger( + 'DeviceVault: not backed up by DeviceVault', + f'last({found})=0 and {{$DV.NOTFOUND.ALERT}}=1', + f'last({found})=1 or {{$DV.NOTFOUND.ALERT}}=0', + 'INFO', + 'No device DeviceVault backs up (and the token may see) has this address or id. Set {$DV.NOTFOUND.ALERT} ' + 'to 0 on hosts that are not meant to be backed up.', + 'backup', + 'notice', + depends=[nodata], + ), + Trigger( + 'DeviceVault: backup overdue', + f'last({ref("overdue")})=1 and {gate}', + f'last({ref("overdue")})=0 or {ungate}', + 'WARNING', + 'The last successful backup is older than the schedule allows.', + 'backup', + 'notice', + depends=[nodata], + ), + Trigger( + 'DeviceVault: backup failing', + f'last({ref("consecutive_failures")})>={{$DV.DEVICE.FAIL.CONSEC}} and {gate}', + f'last({ref("consecutive_failures")})<{{$DV.DEVICE.FAIL.CONSEC}} or {ungate}', + 'WARNING', + 'The last {$DV.DEVICE.FAIL.CONSEC} or more backups failed.', + 'backup', + 'notice', + depends=[nodata], + opdata='Consecutive failures: {ITEM.LASTVALUE1}', + ), + Trigger( + 'DeviceVault: backup authentication failing', + f'last({ref("failure_reason_code")})={auth} and {gate}', + f'last({ref("failure_reason_code")})<>{auth} or {ungate}', + 'AVERAGE', + 'The latest backup failed because the device refused the login or enable credentials.', + 'backup', + 'security', + depends=[nodata], + ), + Trigger( + 'DeviceVault: no successful backup for {$DV.DEVICE.STALE.MAX}', + f'{{$DV.DEVICE.STALE.MAX}}>0 and last({ref("last_success_at")})>0 and ' + f'last({ref("last_success_age")})>{{$DV.DEVICE.STALE.MAX}} and {gate}', + f'{{$DV.DEVICE.STALE.MAX}}=0 or last({ref("last_success_age")})<={{$DV.DEVICE.STALE.MAX}} or {ungate}', + 'WARNING', + 'A flat threshold on the age of the last successful backup ({$DV.DEVICE.STALE.MAX}, 0 = off).', + 'backup', + 'notice', + depends=[nodata], + ), + Trigger( + 'DeviceVault: DeviceVault data is old', + f'last({ref("published_age")})>{{$DV.DEVICE.DATA.MAX}}', + f'last({ref("published_age")})<={{$DV.DEVICE.DATA.MAX}}', + 'WARNING', + 'The snapshot answering was published longer ago than {$DV.DEVICE.DATA.MAX}: the orchestrator has ' + 'stopped publishing while the relay still serves.', + 'api', + 'availability', + depends=[nodata], + ), + ] + t.graphs = [Graph('DeviceVault: Last backup duration', [device_key(k) for k in + ('last_duration', 'last_collect_duration', 'last_store_duration')])] # fmt: skip + return t + + +def templates() -> list[_Template]: + return [_host_template(), _device_template()] + + +# --- export -------------------------------------------------------------------------------- + + +def _trigger(template: str, trigger: Trigger, prototype: bool = False) -> dict: + out = { + 'uuid': _uuid(template, 'trigger-prototype' if prototype else 'trigger', trigger.name), + 'expression': trigger.expression, + 'recovery_mode': 'RECOVERY_EXPRESSION', + 'recovery_expression': trigger.recovery, + 'name': trigger.name, + } + if trigger.opdata: + out['opdata'] = trigger.opdata + out['priority'] = trigger.priority + out['description'] = trigger.description + if trigger.depends: + out['dependencies'] = [dependency.reference() for dependency in trigger.depends] + out['tags'] = _tags(component=trigger.component, scope=trigger.scope, group=trigger.group) + return out + + +def _referenced(template: str, expression: str) -> set[str]: + return set(re.findall(rf'/{re.escape(template)}/([^,)]+)', expression)) + + +def _place(template: str, items: list[dict], triggers: list[Trigger], prototype: bool) -> list[dict]: + """Attach single-item triggers to their item, as Zabbix exports them; return the rest.""" + by_key = {item['key']: item for item in items} + rest = [] + for trigger in triggers: + keys = _referenced(template, trigger.expression) | _referenced(template, trigger.recovery) + exported = _trigger(template, trigger, prototype) + if len(keys) == 1 and (key := next(iter(keys))) in by_key: + by_key[key].setdefault('triggers' if not prototype else 'trigger_prototypes', []).append(exported) + else: + rest.append(exported) + return rest + + +def _graph(template: str, graph: Graph, prototype: bool = False) -> dict: + out = {'uuid': _uuid(template, 'graph-prototype' if prototype else 'graph', graph.name), 'name': graph.name} + if graph.stacked: + out['type'] = 'STACKED' + out['graph_items'] = [] + for n, key in enumerate(graph.keys): + entry = {'sortorder': str(n)} if n else {} + entry.update(color=COLOURS[n % len(COLOURS)], item={'host': template, 'key': key}) + out['graph_items'].append(entry) + return out + + +def _rule(template: str, rule: Rule) -> dict: + discovery = rule.discovery + items = [dict(item) for item in rule.items] + rest = _place(template, items, rule.triggers, prototype=True) + out = { + 'uuid': _uuid(template, 'discovery', discovery.slug), + 'name': f'DeviceVault: {discovery.title} discovery', + 'type': 'DEPENDENT', + 'key': f'dv.{discovery.slug}.discovery', + 'delay': '0', + 'lifetime': '7d', + 'description': f'{discovery.title} from {discovery.path}; resources gone for 7 days are deleted.', + 'item_prototypes': items, + } + if rest: + out['trigger_prototypes'] = rest + if rule.graphs: + out['graph_prototypes'] = [_graph(template, graph, prototype=True) for graph in rule.graphs] + out['master_item'] = {'key': master_key(discovery)} + out['lld_macro_paths'] = [{'lld_macro': macro, 'path': f'$.{key}'} for key, macro in LLD_MACROS.items()] + out['preprocessing'] = [{'type': 'JSONPATH', 'parameters': ['$.data']}] + return out + + +def _valuemaps(template: _Template) -> list[dict]: + names = {name: value_map for value_map, name in VALUEMAP_NAMES.items()} + return [ + { + 'uuid': _uuid(template.name, 'valuemap', name), + 'name': name, + 'mappings': [{'value': str(n), 'newvalue': label} for n, label in VALUE_MAPS[names[name]].items()], + } + for name in template.valuemaps() + ] + + +def _reference(n: int) -> str: + """The 7.0 widget reference: five capital letters, unique within a dashboard.""" + letters = '' + for _ in range(5): + n, rest = divmod(n, 26) + letters = chr(ord('A') + rest) + letters + return letters + + +def _widgets(template: _Template, version: str) -> list[dict]: + """Two columns of graphs; Zabbix 6.0 dashboards are 24 units wide, 7.0's 72.""" + width = 24 if version == '6.0' else 72 + widgets = [] + for n, (kind, name) in enumerate(template.dashboard): + widget = {} + widget['type'] = { + ('6.0', 'graph'): 'GRAPH_CLASSIC', + ('6.0', 'prototype'): 'GRAPH_PROTOTYPE', + ('7.0', 'graph'): 'graph', + ('7.0', 'prototype'): 'graphprototype', + }[version, kind] + if n % 2: + widget['x'] = str(width // 2) + if n // 2: + widget['y'] = str(5 * (n // 2)) + widget['width'] = str(width // 2) + widget['height'] = '5' + graph_field = 'graphid' if version == '6.0' else 'graphid.0' + fields = [] + if kind == 'prototype': + fields.append({'type': 'INTEGER', 'name': 'columns', 'value': '1'}) + fields.append( + { + 'type': 'GRAPH' if kind == 'graph' else 'GRAPH_PROTOTYPE', + 'name': graph_field, + 'value': {'host': template.name, 'name': name}, + } + ) + if version != '6.0': + fields.append({'type': 'STRING', 'name': 'reference', 'value': _reference(n)}) + widget['fields'] = fields + widgets.append(widget) + return widgets + + +def _template(template: _Template, version: str) -> tuple[dict, list[dict], list[dict]]: + """(the template entry, its multi-item triggers, its graphs), for ``version``.""" + items = [dict(item) for item in template.items] + rest = _place(template.name, items, template.triggers, prototype=False) + out = { + 'uuid': _uuid(template.name, 'template'), + 'template': template.name, + 'name': template.name, + 'description': template.description, + 'groups': [{'name': TEMPLATE_GROUP}], + 'items': items, + } + if template.rules: + out['discovery_rules'] = [_rule(template.name, rule) for rule in template.rules] + out['tags'] = _tags(**{'class': 'software', 'target': 'devicevault'}) + out['macros'] = template.macros + if template.dashboard: + out['dashboards'] = [ + { + 'uuid': _uuid(template.name, 'dashboard', 'overview'), + 'name': 'DeviceVault overview', + 'pages': [{'name': 'Backups and platform', 'widgets': _widgets(template, version)}], + } + ] + out['valuemaps'] = _valuemaps(template) + return out, rest, [_graph(template.name, graph) for graph in template.graphs] + + +def export(version: str) -> dict: + """The Zabbix export document for ``version`` (``VERSIONS``).""" + if version not in VERSIONS: + raise ValueError(f'unsupported Zabbix version {version!r}; expected one of {", ".join(VERSIONS)}') + entries, triggers, graphs = [], [], [] + for template in templates(): + entry, rest, template_graphs = _template(template, version) + entries.append(entry) + triggers += rest + graphs += template_graphs + group = [{'uuid': _uuid('group', TEMPLATE_GROUP), 'name': TEMPLATE_GROUP}] + document = {'version': version} + document['groups' if version == '6.0' else 'template_groups'] = group + document['templates'] = entries + if triggers: + document['triggers'] = triggers + if graphs: + document['graphs'] = graphs + return {'zabbix_export': document} + + +class _Dumper(yaml.SafeDumper): + """Block style as Zabbix writes it: indented lists, literal multi-line strings.""" + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + +def _str(dumper, value): + style = '|' if '\n' in value else None + return dumper.represent_scalar('tag:yaml.org,2002:str', value, style=style) + + +_Dumper.add_representer(str, _str) + + +def render(version: str) -> str: + """The template file for ``version``, as YAML.""" + return yaml.dump(export(version), Dumper=_Dumper, sort_keys=False, allow_unicode=True, width=1_000_000) diff --git a/orchestrator/pyproject.toml b/orchestrator/pyproject.toml index 3bfd587..a260790 100644 --- a/orchestrator/pyproject.toml +++ b/orchestrator/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "psycopg[binary]>=3.3.6,<3.4", "pydantic>=2.14,<3", "python-ldap>=3.4.5", + "pyyaml>=6.0.3", "requests>=2.32", "whitenoise>=6.11", ] diff --git a/uv.lock b/uv.lock index fb0ebd8..99e9862 100644 --- a/uv.lock +++ b/uv.lock @@ -521,6 +521,7 @@ dependencies = [ { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "python-ldap" }, + { name = "pyyaml" }, { name = "requests" }, { name = "whitenoise" }, ] @@ -550,6 +551,7 @@ requires-dist = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.3.6,<3.4" }, { name = "pydantic", specifier = ">=2.14,<3" }, { name = "python-ldap", specifier = ">=3.4.5" }, + { name = "pyyaml", specifier = ">=6.0.3" }, { name = "requests", specifier = ">=2.32" }, { name = "whitenoise", specifier = ">=6.11" }, ]