From f82d620772617784caebbd9a49cc96bb48be5ef0 Mon Sep 17 00:00:00 2001 From: mikim <1441941+mikim@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:15:01 +0900 Subject: [PATCH] =?UTF-8?q?health:=20=EC=82=AC=EB=B3=B8=EC=9D=B4=20?= =?UTF-8?q?=ED=98=B8=EC=8A=A4=ED=8A=B8=EB=A5=BC=20=EB=B2=97=EC=96=B4?= =?UTF-8?q?=EB=82=98=EC=A7=80=20=EB=AA=BB=ED=96=88=EC=9C=BC=EB=A9=B4=20?= =?UTF-8?q?=EB=B0=B1=EC=97=85=20=ED=96=89=EC=9D=80=20ok=20=EA=B0=80=20?= =?UTF-8?q?=EC=95=84=EB=8B=88=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 지난주 들어온 일일 백업 cron 은 스냅샷을 뜨고 PRAGMA integrity_check 로 "이 파일이 실제로 복원되는가"까지 확인한다. 그 두 질문에는 답하지만 세 번째 질문에는 답하지 않는다 — 이 호스트의 디스크가 아닌 곳에 사본이 있는가. 스냅샷은 ~/backups/alpha 에 떨어지고 그건 원본 DB 와 같은 볼륨이다 (프로덕션 실측: 원본 /home/atrn/data/moss_land.sqlite 와 사본 6개 모두 /dev/mapper/ubuntu--vg-ubuntu--lv). BACKUP_REMOTE 는 설정돼 있지 않다. 그런데 /api/health 의 db_backup 행은 ok 였다. 디스크 하나 죽으면 원본과 백업이 같이 사라지는 상태에 대해서. 이건 trackable_calls 가 95일 무산출 동안 ok 를 보고하던 것과 같은 모양이고, 그때와 같은 방식으로 고친다: heartbeat 판정 위에 두 번째 의견을 얹는다. - scripts/backup-db.ts 가 heartbeat note 앞에 offhost=none|ok|fail 을 적는다. - lib/health.ts 의 applyOffHostGap 이 그 토큰만 읽고 none 이면 warn 으로 내린다. - 토큰은 lib/cron-heartbeat.ts 에 둔다. 쓰는 쪽과 읽는 쪽이 각자의 문자열 리터럴로 갈라지면 이 행은 조용히 ok 로 돌아간다. warn 에서 멈추고 fail 로 가지 않는다. cron 자체는 건강하고, ?strict=1 이 설정 공백을 503 으로 바꾸면 안 된다. 다만 ok 는 과장이었다. 토큰이 없는 옛 heartbeat 는 건드리지 않는다 — 토큰의 부재가 사본의 부재를 뜻하지는 않으므로 추측하지 않고, 다음 03:00 실행이 사실을 채운다. 오프호스트 목적지 자체는 이 커밋의 범위가 아니다. Alpha 는 Signal 로 흡수될 예정이라, 무엇을 이관하고 무엇을 폐기할지 목록이 먼저 나와야 백업 대상이 DB 전체인지 테이블 두어 개인지 정해진다. Co-Authored-By: Claude Opus 5 --- lib/cron-heartbeat.ts | 19 +++++++++++++ lib/health.ts | 32 +++++++++++++++++++-- scripts/backup-db.ts | 10 +++++-- tests/offhost.test.ts | 65 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 tests/offhost.test.ts diff --git a/lib/cron-heartbeat.ts b/lib/cron-heartbeat.ts index 6df0fa4..864684e 100644 --- a/lib/cron-heartbeat.ts +++ b/lib/cron-heartbeat.ts @@ -102,3 +102,22 @@ export function getAllHeartbeats(): Heartbeat[] { runCount: r.run_count, })); } + +/** + * Off-host state of the daily DB backup, carried inside its heartbeat note. + * + * The note itself is prose for whoever reads /health. This token is the one + * part `lib/health.ts` parses, and it lives here — beside the contract it + * rides on — so the writer and the reader cannot drift apart into two string + * literals in two files. + */ +export type OffHostState = "none" | "ok" | "fail"; + +export function offHostToken(state: OffHostState): string { + return `offhost=${state}`; +} + +export function readOffHost(note: string | null | undefined): OffHostState | null { + const m = /\boffhost=(none|ok|fail)\b/.exec(note ?? ""); + return m ? (m[1] as OffHostState) : null; +} diff --git a/lib/health.ts b/lib/health.ts index 00ac5dd..a40077f 100644 --- a/lib/health.ts +++ b/lib/health.ts @@ -10,7 +10,7 @@ import fs from "node:fs"; import path from "node:path"; import { getDb } from "./db"; import { rateLimitSnapshot } from "./rate-limit"; -import { getHeartbeat } from "./cron-heartbeat"; +import { getHeartbeat, readOffHost } from "./cron-heartbeat"; import { assetCoverage, isCallableAsset } from "./prices"; import { todayAiSpendUsd } from "./grok"; import { recentAuditRuns, type AuditRun } from "./audit-log"; @@ -132,6 +132,34 @@ function applyContentStaleness( }; } +/** + * Third opinion, for the backup only: did the copy leave the box? + * + * Same failure shape as the one above. The heartbeat answers "did the cron + * run" and the verification answers "would this file restore" — neither + * answers "does a copy exist anywhere the host's disk isn't". Snapshots land + * in ~/backups/alpha, on the same volume as the DB they copy, so with + * BACKUP_REMOTE unset the row reported `ok` for something one disk failure + * away from nothing. + * + * Capped at `warn`, never `fail`, for the same reason as content staleness: + * the cron is healthy and ?strict=1 must not 503 a monitor over a + * configuration gap. But `ok` overstated it, and this is the one row whose + * whole job is to be true before someone needs it. + */ +function applyOffHostGap( + subsystem: SubsystemHealth, + heartbeat: ReturnType +): SubsystemHealth { + if (subsystem.status === "fail") return subsystem; + if (readOffHost(heartbeat?.lastNote) !== "none") return subsystem; + return { + ...subsystem, + status: "warn", + note: `${subsystem.note ? subsystem.note + " " : ""}사본이 원본과 같은 호스트에 있습니다 — BACKUP_REMOTE 미설정. 호스트 손실 시 둘 다 사라집니다.`, + }; +} + /** Event-driven crons may be legitimately quiet for days; a fortnight of * nothing is a warning, six weeks is a stopped subsystem. */ const CONTENT_WARN_SEC = 14 * 24 * 3600; @@ -536,7 +564,7 @@ export function getSystemHealth(): { ? `마지막 실행 ${hb.lastStatus}. ${hb.lastNote ?? ""}`.trim() : "heartbeat 없음 — cron 첫 실행 대기 중.", }); - return applyHeartbeatFailure(sub, hb); + return applyOffHostGap(applyHeartbeatFailure(sub, hb), hb); })(), (() => { // Liveness of the weekly audit cron — deliberately NOT its citation diff --git a/scripts/backup-db.ts b/scripts/backup-db.ts index 945d76e..b618f01 100644 --- a/scripts/backup-db.ts +++ b/scripts/backup-db.ts @@ -41,6 +41,9 @@ import path from "node:path"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { loadScriptEnv } from "../lib/script-env"; +// Type-only: the value side stays behind the dynamic import below, which +// must not run before loadScriptEnv() has put DB_PATH in the environment. +import type { OffHostState } from "../lib/cron-heartbeat"; loadScriptEnv(); @@ -124,7 +127,7 @@ async function main() { return; } - const { recordHeartbeat } = await import("../lib/cron-heartbeat"); + const { recordHeartbeat, offHostToken } = await import("../lib/cron-heartbeat"); const src = process.env.DB_PATH; if (!src || !fs.existsSync(src)) { const note = `DB_PATH 없음 또는 파일 없음: ${src ?? "(unset)"}`; @@ -180,6 +183,7 @@ async function main() { // warning — the local copy still exists, but the box is a single point of // failure again and someone has to know. let offHost = "off-host 미설정 (BACKUP_REMOTE)"; + let offHostState: OffHostState = "none"; let status: "ok" | "error" = "ok"; if (remote) { const rsync = process.env.BACKUP_RSYNC_BIN || "rsync"; @@ -188,9 +192,11 @@ async function main() { timeout: 15 * 60_000, }); offHost = `off-host 복사 완료 → ${remote}`; + offHostState = "ok"; console.log(offHost); } catch (err) { offHost = `off-host 복사 실패 → ${remote}: ${(err as Error).message.slice(0, 200)}`; + offHostState = "fail"; console.error(offHost); status = "error"; } @@ -201,7 +207,7 @@ async function main() { } const removed = prune(dir, keep); - const note = `${path.basename(dest)} ${sizeMb}MB · ${check.note} · ${offHost} · 정리 ${removed}건 (보관 ${keep})`; + const note = `${offHostToken(offHostState)} · ${path.basename(dest)} ${sizeMb}MB · ${check.note} · ${offHost} · 정리 ${removed}건 (보관 ${keep})`; console.log(`Heartbeat: ${status} — ${note}`); recordHeartbeat("alpha-backup-cron", status, note); if (status === "error") process.exitCode = 1; diff --git a/tests/offhost.test.ts b/tests/offhost.test.ts new file mode 100644 index 0000000..1dbbb54 --- /dev/null +++ b/tests/offhost.test.ts @@ -0,0 +1,65 @@ +/** + * The off-host marker in the backup heartbeat note. + * + * scripts/backup-db.ts writes this token and lib/health.ts reads it, and the + * whole point of the row is to be true before someone needs it. A silent + * drift between writer and reader would put the row back where it was — + * reporting `ok` for a copy sitting on the same disk as the original. + * + * Pure string work. The DB_PATH dance below exists only because + * lib/cron-heartbeat.ts imports lib/db, which mkdirs its data directory at + * module load; the temp path keeps that out of the checkout. + */ + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { after, before, describe, it } from "node:test"; + +type Heartbeat = typeof import("../lib/cron-heartbeat"); + +let hb: Heartbeat; +let tmpDir: string; + +before(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "alpha-offhost-test-")); + process.env.DB_PATH = path.join(tmpDir, "test.sqlite"); + hb = await import("../lib/cron-heartbeat"); +}); + +after(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe("off-host marker", () => { + it("round-trips every state", () => { + for (const state of ["none", "ok", "fail"] as const) { + assert.equal(hb.readOffHost(hb.offHostToken(state)), state); + } + }); + + it("reads the note backup-db.ts actually writes", () => { + // Verbatim shape from scripts/backup-db.ts: token first, then prose. + const note = + "offhost=none · alpha-daily-20260825T180000Z.sqlite 17.6MB · " + + "integrity ok, posts=978 · off-host 미설정 (BACKUP_REMOTE) · 정리 0건 (보관 14)"; + assert.equal(hb.readOffHost(note), "none"); + }); + + it("follows the token, not the prose", () => { + // The prose says 미설정 and the token says otherwise. The token wins, + // because prose is for people and gets rewritten; this is the contract. + const note = "offhost=ok · off-host 미설정 이라는 옛 문구가 남아 있어도"; + assert.equal(hb.readOffHost(note), "ok"); + }); + + it("returns null when there is no token", () => { + // Heartbeats written before the token existed carry no verdict, and an + // absent token is not evidence of an absent copy. health.ts leaves the + // row alone rather than guessing; the next 03:00 run supplies the truth. + assert.equal(hb.readOffHost("integrity ok, posts=978"), null); + assert.equal(hb.readOffHost(null), null); + assert.equal(hb.readOffHost(undefined), null); + }); +});