From 42ad48a29061bd5301f8d612e8a5b72489a36195 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 27 Oct 2026 20:41:23 +0100 Subject: [PATCH 1/2] fix(coroot): publish checkpoint correction for deployed releases --- .../publish-coroot-checkpoint-hotfix.yaml | 214 ++++++++++++++++++ .../coroot/CorootCheckpoint.Dockerfile | 21 ++ .../CorootClusterAgentCheckpoint.Dockerfile | 12 + ...ster-agent-v1.11.7-checkpoint-record.patch | 87 +++++++ .../coroot-v1.28.1-checkpoint-record.patch | 159 +++++++++++++ 5 files changed, 493 insertions(+) create mode 100644 .github/workflows/publish-coroot-checkpoint-hotfix.yaml create mode 100644 k8s/bases/infrastructure/coroot/CorootCheckpoint.Dockerfile create mode 100644 k8s/bases/infrastructure/coroot/CorootClusterAgentCheckpoint.Dockerfile create mode 100644 k8s/bases/infrastructure/coroot/cluster-agent-v1.11.7-checkpoint-record.patch create mode 100644 k8s/bases/infrastructure/coroot/coroot-v1.28.1-checkpoint-record.patch diff --git a/.github/workflows/publish-coroot-checkpoint-hotfix.yaml b/.github/workflows/publish-coroot-checkpoint-hotfix.yaml new file mode 100644 index 0000000000..c22d9c83a0 --- /dev/null +++ b/.github/workflows/publish-coroot-checkpoint-hotfix.yaml @@ -0,0 +1,214 @@ +name: Publish Coroot Checkpoint Hotfix + +on: + pull_request: + branches: [main] + paths: + - .github/workflows/publish-coroot-checkpoint-hotfix.yaml + - k8s/bases/infrastructure/coroot/coroot-v1.28.1-checkpoint-record.patch + - k8s/bases/infrastructure/coroot/CorootCheckpoint.Dockerfile + - k8s/bases/infrastructure/coroot/cluster-agent-v1.11.7-checkpoint-record.patch + - k8s/bases/infrastructure/coroot/CorootClusterAgentCheckpoint.Dockerfile + push: + branches: [main] + paths: + - .github/workflows/publish-coroot-checkpoint-hotfix.yaml + - k8s/bases/infrastructure/coroot/coroot-v1.28.1-checkpoint-record.patch + - k8s/bases/infrastructure/coroot/CorootCheckpoint.Dockerfile + - k8s/bases/infrastructure/coroot/cluster-agent-v1.11.7-checkpoint-record.patch + - k8s/bases/infrastructure/coroot/CorootClusterAgentCheckpoint.Dockerfile + workflow_dispatch: + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + verify: + name: Verify ${{ matrix.component }} + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - component: server + repository: coroot/coroot + source: 15137ef2f43e01d0a87245e44b937b25515e0afe + patch: coroot-v1.28.1-checkpoint-record.patch + dockerfile: CorootCheckpoint.Dockerfile + image: platform-coroot + version: v1.28.1-checkpoints.1 + - component: collector + repository: coroot/coroot-cluster-agent + source: 4291458563fa8ea55615e23e0f11412db79742b6 + patch: cluster-agent-v1.11.7-checkpoint-record.patch + dockerfile: CorootClusterAgentCheckpoint.Dockerfile + image: platform-coroot-cluster-agent + version: v1.11.7-checkpoints.1 + env: + IMAGE: ghcr.io/devantler-tech/${{ matrix.image }} + IMAGE_TAG: ${{ matrix.version }}-${{ github.sha }} + steps: + - name: Harden runner + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + with: + egress-policy: audit + + - name: Check out platform contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check out exact upstream release source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ matrix.repository }} + ref: ${{ matrix.source }} + path: upstream + persist-credentials: false + + - name: Apply reviewed release patch + working-directory: upstream + env: + EXPECTED_SOURCE_COMMIT: ${{ matrix.source }} + HOTFIX_PATCH: ${{ github.workspace }}/k8s/bases/infrastructure/coroot/${{ matrix.patch }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "${EXPECTED_SOURCE_COMMIT}" + git apply --check "${HOTFIX_PATCH}" + git apply "${HOTFIX_PATCH}" + git diff --check + + - name: Set up Buildx container driver + uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 + with: + driver: docker-container + + - name: Build without publishing tested release image + id: build + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ./upstream + file: ./k8s/bases/infrastructure/coroot/${{ matrix.dockerfile }} + platforms: linux/amd64 + push: false + tags: ${{ env.IMAGE }}:${{ env.IMAGE_TAG }} + build-args: VERSION=${{ env.IMAGE_TAG }} + provenance: mode=max + sbom: true + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ env.IMAGE_TAG }} + io.devantler.coroot-checkpoint.upstream-revision=${{ matrix.source }} + + publish: + name: Publish and sign ${{ matrix.component }} + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: verify + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + strategy: + fail-fast: false + matrix: + include: + - component: server + repository: coroot/coroot + source: 15137ef2f43e01d0a87245e44b937b25515e0afe + patch: coroot-v1.28.1-checkpoint-record.patch + dockerfile: CorootCheckpoint.Dockerfile + image: platform-coroot + version: v1.28.1-checkpoints.1 + - component: collector + repository: coroot/coroot-cluster-agent + source: 4291458563fa8ea55615e23e0f11412db79742b6 + patch: cluster-agent-v1.11.7-checkpoint-record.patch + dockerfile: CorootClusterAgentCheckpoint.Dockerfile + image: platform-coroot-cluster-agent + version: v1.11.7-checkpoints.1 + env: + IMAGE: ghcr.io/devantler-tech/${{ matrix.image }} + IMAGE_TAG: ${{ matrix.version }}-${{ github.sha }} + steps: + - name: Harden runner + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + with: + egress-policy: audit + + - name: Check out platform contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check out exact upstream release source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ matrix.repository }} + ref: ${{ matrix.source }} + path: upstream + persist-credentials: false + + - name: Apply reviewed release patch + working-directory: upstream + env: + EXPECTED_SOURCE_COMMIT: ${{ matrix.source }} + HOTFIX_PATCH: ${{ github.workspace }}/k8s/bases/infrastructure/coroot/${{ matrix.patch }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "${EXPECTED_SOURCE_COMMIT}" + git apply --check "${HOTFIX_PATCH}" + git apply "${HOTFIX_PATCH}" + git diff --check + + - name: Set up Buildx container driver + uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 + with: + driver: docker-container + + - name: Log in to GHCR + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push tested release image + id: build + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ./upstream + file: ./k8s/bases/infrastructure/coroot/${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: ${{ env.IMAGE }}:${{ env.IMAGE_TAG }} + build-args: VERSION=${{ env.IMAGE_TAG }} + provenance: mode=max + sbom: true + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ env.IMAGE_TAG }} + io.devantler.coroot-checkpoint.upstream-revision=${{ matrix.source }} + + - name: Set up Cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Sign and verify published digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + test -n "${DIGEST}" + cosign sign --yes "${IMAGE}@${DIGEST}" + cosign verify \ + --certificate-identity "https://github.com/devantler-tech/platform/.github/workflows/publish-coroot-checkpoint-hotfix.yaml@refs/heads/main" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + "${IMAGE}@${DIGEST}" + printf 'Published signed image: %s@%s\n' "${IMAGE}" "${DIGEST}" diff --git a/k8s/bases/infrastructure/coroot/CorootCheckpoint.Dockerfile b/k8s/bases/infrastructure/coroot/CorootCheckpoint.Dockerfile new file mode 100644 index 0000000000..3aec5bf44c --- /dev/null +++ b/k8s/bases/infrastructure/coroot/CorootCheckpoint.Dockerfile @@ -0,0 +1,21 @@ +FROM docker.io/library/node:24-bookworm@sha256:f9b3548b20b15b705b66aeecaeac7dd690c73d5e4661357aac02e60e3d38c88a AS frontend +WORKDIR /src/front +COPY front/package.json front/package-lock.json ./ +RUN npm ci +COPY front/ ./ +RUN npm run build-prod + +FROM docker.io/library/golang:1.27.3-bookworm@sha256:fea12ba16096adec0f699ce1c4da2154d320450e56c1f63671f95d2a3551c77b AS builder +RUN apt-get update && apt-get install --yes --no-install-recommends liblz4-dev && rm -rf /var/lib/apt/lists/* +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +COPY --from=frontend /src/static ./static +RUN go test ./... +ARG VERSION=unknown +RUN go build -mod=readonly -ldflags "-X main.version=$VERSION" -o /out/coroot . + +FROM ghcr.io/coroot/coroot:1.28.1@sha256:1b908030e1cce068ce6fb907cd02c41552e27aab382e61f1bbb523ceb50a0913 +COPY --from=builder --chmod=0755 /out/coroot /usr/bin/coroot +RUN ["/usr/bin/coroot", "--help"] diff --git a/k8s/bases/infrastructure/coroot/CorootClusterAgentCheckpoint.Dockerfile b/k8s/bases/infrastructure/coroot/CorootClusterAgentCheckpoint.Dockerfile new file mode 100644 index 0000000000..d7271a312b --- /dev/null +++ b/k8s/bases/infrastructure/coroot/CorootClusterAgentCheckpoint.Dockerfile @@ -0,0 +1,12 @@ +FROM docker.io/library/golang:1.27.3-bookworm@sha256:fea12ba16096adec0f699ce1c4da2154d320450e56c1f63671f95d2a3551c77b AS builder +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN go test ./... +ARG VERSION=unknown +RUN go build -mod=readonly -ldflags "-X main.version=$VERSION" -o /out/coroot-cluster-agent . + +FROM ghcr.io/coroot/coroot-cluster-agent:1.11.7@sha256:851015f822ca6c8b2934ef98c6a20312a1ea9823045737ac3cdd141efc63d7a8 +COPY --from=builder --chmod=0755 /out/coroot-cluster-agent /usr/bin/coroot-cluster-agent +RUN ["/usr/bin/coroot-cluster-agent", "--help"] diff --git a/k8s/bases/infrastructure/coroot/cluster-agent-v1.11.7-checkpoint-record.patch b/k8s/bases/infrastructure/coroot/cluster-agent-v1.11.7-checkpoint-record.patch new file mode 100644 index 0000000000..5d9b7225fb --- /dev/null +++ b/k8s/bases/infrastructure/coroot/cluster-agent-v1.11.7-checkpoint-record.patch @@ -0,0 +1,87 @@ +diff --git a/metrics/postgres/collector.go b/metrics/postgres/collector.go +index db1f1da..d94c33d 100644 +--- a/metrics/postgres/collector.go ++++ b/metrics/postgres/collector.go +@@ -52,21 +52,22 @@ var ( + dWalReceiveLsn = desc("pg_wal_receive_lsn", "WAL sequence number that has been received and synced to disk by streaming replication") + dWalReplyLsn = desc("pg_wal_reply_lsn", "WAL sequence number that has been replayed during recovery") + +- dCheckpointsScheduled = desc("pg_checkpoints_scheduled_total", "Number of scheduled checkpoints, including skipped ones", "type") +- dCheckpoints = desc("pg_checkpoints_total", "Number of checkpoints that have been completed") +- dRestartpoints = desc("pg_restartpoints_total", "Number of restartpoints that have been completed on a standby") +- dBuffersWritten = desc("pg_buffers_written_total", "Total number of dirty buffers flushed to disk", "source") +- dTimeSinceLastCheckpoint = desc("pg_time_since_last_checkpoint_seconds", "Seconds since the last checkpoint observed by the agent") +- dWalSinceLastCheckpoint = desc("pg_wal_since_last_checkpoint_bytes", "Amount of WAL written since the last completed checkpoint (to be replayed in the case of a crash)") +- dWalSize = desc("pg_wal_size_bytes", "Size of the WAL directory") +- dReplicationSlotRetained = desc("pg_replication_slot_retained_wal_bytes", "Amount of WAL retained for the replication slot", "slot", "active", "wal_status") +- dXidAge = desc("pg_xid_age", "Transactions since the oldest unfrozen transaction ID (age of datfrozenxid)", "db") +- dMultixactAge = desc("pg_multixact_age", "Multixacts since the oldest unfrozen multixact ID (age of datminmxid)", "db") +- dOldestXminAge = desc("pg_oldest_xmin_age", "Age, in transactions, of the oldest transaction ID held back from freezing, by holder", "holder") +- dTransactionSeconds = desc("pg_transaction_seconds", "Age of the longest-running transaction, by query", "db", "user", "query") +- dWalArchivedSegments = desc("pg_wal_archived_segments_total", "Number of WAL files successfully archived") +- dWalArchiveFailures = desc("pg_wal_archive_failures_total", "Number of failed attempts to archive WAL files") +- dWalArchivingStatus = desc("pg_wal_archiving_status", "1 if the last WAL archive attempt succeeded, 0 if it failed") ++ dCheckpointsScheduled = desc("pg_checkpoints_scheduled_total", "Number of scheduled checkpoints, including skipped ones", "type") ++ dCheckpoints = desc("pg_checkpoints_total", "Number of checkpoints that have been completed") ++ dRestartpoints = desc("pg_restartpoints_total", "Number of restartpoints that have been completed on a standby") ++ dBuffersWritten = desc("pg_buffers_written_total", "Total number of dirty buffers flushed to disk", "source") ++ dTimeSinceLastCheckpoint = desc("pg_time_since_last_checkpoint_seconds", "Seconds since the last checkpoint observed by the agent") ++ dWalSinceLastCheckpoint = desc("pg_wal_since_last_checkpoint_bytes", "Amount of WAL written since the last completed checkpoint (to be replayed in the case of a crash)") ++ dWalSinceLastCheckpointRecord = desc("pg_wal_since_last_checkpoint_record_bytes", "Amount of WAL inserted or replayed after the last completed checkpoint or restartpoint record") ++ dWalSize = desc("pg_wal_size_bytes", "Size of the WAL directory") ++ dReplicationSlotRetained = desc("pg_replication_slot_retained_wal_bytes", "Amount of WAL retained for the replication slot", "slot", "active", "wal_status") ++ dXidAge = desc("pg_xid_age", "Transactions since the oldest unfrozen transaction ID (age of datfrozenxid)", "db") ++ dMultixactAge = desc("pg_multixact_age", "Multixacts since the oldest unfrozen multixact ID (age of datminmxid)", "db") ++ dOldestXminAge = desc("pg_oldest_xmin_age", "Age, in transactions, of the oldest transaction ID held back from freezing, by holder", "holder") ++ dTransactionSeconds = desc("pg_transaction_seconds", "Age of the longest-running transaction, by query", "db", "user", "query") ++ dWalArchivedSegments = desc("pg_wal_archived_segments_total", "Number of WAL files successfully archived") ++ dWalArchiveFailures = desc("pg_wal_archive_failures_total", "Number of failed attempts to archive WAL files") ++ dWalArchivingStatus = desc("pg_wal_archiving_status", "1 if the last WAL archive attempt succeeded, 0 if it failed") + + dDbSize = desc("pg_database_size_bytes", "Total size of the database in bytes", "db") + dTableSize = desc("pg_table_size_bytes", "Total size of the table in bytes including indexes and TOAST", "db", "schema", "table") +@@ -139,6 +140,7 @@ type Collector struct { + cpRestartsDone float64 + cpBuffers float64 + cpWalBytes sql.Null[float64] ++ cpWalAfterRecord sql.Null[float64] + lastCheckpointAt time.Time + + walSize sql.Null[float64] +@@ -582,6 +584,9 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { + if c.cpWalBytes.Valid { + ch <- gauge(dWalSinceLastCheckpoint, c.cpWalBytes.V) + } ++ if c.cpWalAfterRecord.Valid { ++ ch <- gauge(dWalSinceLastCheckpointRecord, c.cpWalAfterRecord.V) ++ } + if c.walSize.Valid { + ch <- gauge(dWalSize, c.walSize.V) + } +@@ -645,6 +650,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { + ch <- dBuffersWritten + ch <- dTimeSinceLastCheckpoint + ch <- dWalSinceLastCheckpoint ++ ch <- dWalSinceLastCheckpointRecord + ch <- dWalSize + ch <- dReplicationSlotRetained + ch <- dWalArchivedSegments +diff --git a/metrics/postgres/pg_checkpointer.go b/metrics/postgres/pg_checkpointer.go +index aec0470..af1dc0d 100644 +--- a/metrics/postgres/pg_checkpointer.go ++++ b/metrics/postgres/pg_checkpointer.go +@@ -46,9 +46,12 @@ func (c *Collector) getCheckpointStats(ctx context.Context, version semver.Versi + return err + } + if err := c.db.QueryRowContext(ctx, +- `SELECT pg_wal_lsn_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() ELSE pg_current_wal_lsn() END, redo_lsn) FROM pg_control_checkpoint()`, +- ).Scan(&c.cpWalBytes); err != nil { ++ `SELECT pg_wal_lsn_diff(lsn, redo_lsn), pg_wal_lsn_diff(lsn, checkpoint_lsn) ++ FROM pg_control_checkpoint(), ++ (SELECT CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() ELSE pg_current_wal_lsn() END AS lsn) wal`, ++ ).Scan(&c.cpWalBytes, &c.cpWalAfterRecord); err != nil { + c.cpWalBytes = sql.Null[float64]{} ++ c.cpWalAfterRecord = sql.Null[float64]{} + c.logger.Warning(err) + } + if c.cpPrev != nil { diff --git a/k8s/bases/infrastructure/coroot/coroot-v1.28.1-checkpoint-record.patch b/k8s/bases/infrastructure/coroot/coroot-v1.28.1-checkpoint-record.patch new file mode 100644 index 0000000000..64b0601758 --- /dev/null +++ b/k8s/bases/infrastructure/coroot/coroot-v1.28.1-checkpoint-record.patch @@ -0,0 +1,159 @@ +diff --git a/auditor/postgres.go b/auditor/postgres.go +index 673a8e9..c85443f 100644 +--- a/auditor/postgres.go ++++ b/auditor/postgres.go +@@ -1434,7 +1434,7 @@ func pgCheckpoints(report *model.AuditReport, instance *model.Instance, instance + ), + ) + if last := pg.TimeSinceLastCheckpoint.Last(); !timeseries.IsNaN(last) && last > timeout.Last()*mult { +- if wal := pg.WalSinceLastCheckpoint.Last(); wal > pgWalSegmentSizeBytes { ++ if wal := pgCheckpointWalDistance(pg); wal > pgWalSegmentSizeBytes { + checkpointCheck.AddItem("%s", instance.Name) + if cause := pgCheckpointStallCause(instance, instances, primaryLsn); cause != "" { + checkpointCheck.AddDetail("%s: %s", instance.Name, cause) +@@ -1445,6 +1445,20 @@ func pgCheckpoints(report *model.AuditReport, instance *model.Instance, instance + } + } + ++// Archive-timeout switches during a checkpoint can leave its REDO pointer more ++// than one segment behind, even when PostgreSQL correctly skips idle checkpoints. ++// Measure subsequent WAL from the completed checkpoint/restartpoint record. Keep ++// the original check when the collector does not provide a current finite value. ++func pgCheckpointWalDistance(pg *model.Postgres) float32 { ++ observedAt, _ := pg.TimeSinceLastCheckpoint.LastNotNull() ++ recordAt, _ := pg.WalSinceLastCheckpointRecord.LastNotNull() ++ distance := pg.WalSinceLastCheckpointRecord.Last() ++ if recordAt == observedAt && distance >= 0 && !timeseries.IsInf(distance, 0) { ++ return distance ++ } ++ return pg.WalSinceLastCheckpoint.Last() ++} ++ + func pgSettingBytes(s model.PgSetting) *timeseries.TimeSeries { + var mult float32 + switch s.Unit { +diff --git a/auditor/postgres_test.go b/auditor/postgres_test.go +new file mode 100644 +index 0000000..b7e980a +--- /dev/null ++++ b/auditor/postgres_test.go +@@ -0,0 +1,70 @@ ++package auditor ++ ++import ( ++ "math" ++ "testing" ++ ++ "github.com/coroot/coroot/model" ++ "github.com/coroot/coroot/timeseries" ++) ++ ++func TestPgCheckpointsExcludesArchivePadding(t *testing.T) { ++ const from = timeseries.Time(10000) ++ const step = timeseries.Duration(5) ++ const points = 21 ++ constant := func(value float32) *timeseries.TimeSeries { ++ values := make([]float32, points) ++ for i := range values { ++ values[i] = value ++ } ++ return timeseries.NewWithData(from, step, values) ++ } ++ stale := timeseries.NewWithData(from, step, []float32{16777064}) ++ future := timeseries.NewWithData(from.Add(105), step, []float32{16777064}) ++ tests := []struct { ++ name string ++ recordBytes *timeseries.TimeSeries ++ attempts float32 ++ wantFindings int ++ }{ ++ {name: "idle_primary_archive_padding", recordBytes: constant(16777064), attempts: 1, wantFindings: 0}, ++ {name: "idle_standby_with_no_primary_checkpoint_attempts", recordBytes: constant(16777064), wantFindings: 0}, ++ {name: "record_distance_at_existing_segment_boundary", recordBytes: constant(16777216), wantFindings: 0}, ++ {name: "record_distance_above_existing_segment_boundary", recordBytes: constant(16777218), wantFindings: 1}, ++ {name: "stalled_checkpointer", recordBytes: constant(75332296), wantFindings: 1}, ++ {name: "failed_attempts_with_real_wal_debt", recordBytes: constant(75332296), attempts: 1, wantFindings: 1}, ++ {name: "missing_record_metric", attempts: 1, wantFindings: 1}, ++ {name: "missing_record_samples", recordBytes: constant(timeseries.NaN), attempts: 1, wantFindings: 1}, ++ {name: "invalid_infinite_distance", recordBytes: constant(float32(math.Inf(1))), attempts: 1, wantFindings: 1}, ++ {name: "negative_distance", recordBytes: constant(-1), attempts: 1, wantFindings: 1}, ++ {name: "stale_record_distance", recordBytes: stale, attempts: 1, wantFindings: 1}, ++ {name: "future_record_distance", recordBytes: future, attempts: 1, wantFindings: 1}, ++ } ++ for _, test := range tests { ++ t.Run(test.name, func(t *testing.T) { ++ pg := model.NewPostgres() ++ pg.Settings["checkpoint_timeout"] = model.PgSetting{Samples: constant(30)} ++ ages := make([]float32, points) ++ for i := range ages { ++ ages[i] = float32(i * 5) ++ } ++ pg.TimeSinceLastCheckpoint = timeseries.NewWithData(from, step, ages) ++ // Native PG18 archive-timeout reproduction: the REDO pointer is ++ // over one segment behind while distance after the checkpoint ++ // record remains below a segment. Attempts alone can also advance ++ // on unsuccessful checkpoints and must not clear real WAL debt. ++ pg.WalSinceLastCheckpoint = constant(33150320) ++ pg.WalSinceLastCheckpointRecord = test.recordBytes ++ pg.Checkpoints = constant(0) ++ pg.CheckpointsScheduledByType["timed"] = constant(test.attempts) ++ instance := &model.Instance{Name: "fixture", Postgres: pg} ++ report := model.NewAuditReport(&model.Application{}, timeseries.NewContext(from, from.Add(100), step), nil, model.AuditReportPostgres, true) ++ check := report.CreateCheck(model.Checks.PostgresCheckpoint) ++ check.Threshold = 3 ++ pgCheckpoints(report, instance, []*model.Instance{instance}, nil, check) ++ if got := check.Items().Len(); got != test.wantFindings { ++ t.Fatalf("checkpoint findings = %d, want %d", got, test.wantFindings) ++ } ++ }) ++ } ++} +diff --git a/constructor/postgres.go b/constructor/postgres.go +index 741ec38..961b004 100644 +--- a/constructor/postgres.go ++++ b/constructor/postgres.go +@@ -202,6 +202,8 @@ func postgres(instance *model.Instance, queryName string, m *model.MetricValues, + pg.TimeSinceLastCheckpoint = merge(pg.TimeSinceLastCheckpoint, values, timeseries.Any) + case "pg_wal_since_last_checkpoint_bytes": + pg.WalSinceLastCheckpoint = merge(pg.WalSinceLastCheckpoint, values, timeseries.Any) ++ case "pg_wal_since_last_checkpoint_record_bytes": ++ pg.WalSinceLastCheckpointRecord = merge(pg.WalSinceLastCheckpointRecord, values, timeseries.Any) + } + } + +diff --git a/constructor/queries.go b/constructor/queries.go +index f24e98a..f32ddfd 100644 +--- a/constructor/queries.go ++++ b/constructor/queries.go +@@ -471,6 +471,7 @@ var QUERIES = []Query{ + qDB("pg_buffers_written_total", `rate(pg_buffers_written_total[$RANGE])`, "source"), + qDB("pg_time_since_last_checkpoint_seconds", `pg_time_since_last_checkpoint_seconds`), + qDB("pg_wal_since_last_checkpoint_bytes", `pg_wal_since_last_checkpoint_bytes`), ++ qDB("pg_wal_since_last_checkpoint_record_bytes", `pg_wal_since_last_checkpoint_record_bytes`), + + qDB("mysql_up", `mysql_up`), + qDB("mysql_scrape_error", `mysql_scrape_error`, "error", "warning"), +diff --git a/model/postgres.go b/model/postgres.go +index 0a6938f..05ac049 100644 +--- a/model/postgres.go ++++ b/model/postgres.go +@@ -103,12 +103,13 @@ type Postgres struct { + OldestXminAge map[string]*timeseries.TimeSeries + TransactionSeconds map[QueryKey]*timeseries.TimeSeries + +- CheckpointsScheduledByType map[string]*timeseries.TimeSeries +- Checkpoints *timeseries.TimeSeries +- Restartpoints *timeseries.TimeSeries +- TimeSinceLastCheckpoint *timeseries.TimeSeries +- WalSinceLastCheckpoint *timeseries.TimeSeries +- BuffersWrittenBySource map[string]*timeseries.TimeSeries ++ CheckpointsScheduledByType map[string]*timeseries.TimeSeries ++ Checkpoints *timeseries.TimeSeries ++ Restartpoints *timeseries.TimeSeries ++ TimeSinceLastCheckpoint *timeseries.TimeSeries ++ WalSinceLastCheckpoint *timeseries.TimeSeries ++ WalSinceLastCheckpointRecord *timeseries.TimeSeries ++ BuffersWrittenBySource map[string]*timeseries.TimeSeries + + DatabaseSize map[string]*timeseries.TimeSeries + TableSize map[DbTableKey]*timeseries.TimeSeries From 48772234587a261646cdbfa692b1b7e141ce9da7 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 27 Oct 2026 21:48:09 +0100 Subject: [PATCH 2/2] fix(coroot): verify signing tools before installation --- .github/workflows/publish-coroot-checkpoint-hotfix.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-coroot-checkpoint-hotfix.yaml b/.github/workflows/publish-coroot-checkpoint-hotfix.yaml index c22d9c83a0..66f35344c1 100644 --- a/.github/workflows/publish-coroot-checkpoint-hotfix.yaml +++ b/.github/workflows/publish-coroot-checkpoint-hotfix.yaml @@ -197,8 +197,8 @@ jobs: org.opencontainers.image.version=${{ env.IMAGE_TAG }} io.devantler.coroot-checkpoint.upstream-revision=${{ matrix.source }} - - name: Set up Cosign - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - name: Install checksum-verified supply-chain tools + run: bash .github/scripts/setup-supply-chain-tools.sh - name: Sign and verify published digest env: