From 4fe65ee3c38dbfb4de01a46e8fd17a08efb01e62 Mon Sep 17 00:00:00 2001 From: Jon Bringhurst Date: Fri, 11 Sep 2026 12:27:11 -0700 Subject: [PATCH 1/3] tests: reject invalid release hashes, archives and checkouts --- tests/unit/li_bridge_release_inputs_test.py | 145 ++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 tests/unit/li_bridge_release_inputs_test.py diff --git a/tests/unit/li_bridge_release_inputs_test.py b/tests/unit/li_bridge_release_inputs_test.py new file mode 100644 index 0000000000000..070eb514dec35 --- /dev/null +++ b/tests/unit/li_bridge_release_inputs_test.py @@ -0,0 +1,145 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Release-input validation fixtures, not evidence of a qualified Kafka release.""" + +import io +import subprocess +import sys +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).parents[1] / "bin")) +import verify_li_bridge_release as RELEASE + + +class LiBridgeReleaseInputsTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.directory = Path(temporary.name) + self.root, kafka_commit = self.checkout("kafka") + self.wrapper, wrapper_commit = self.checkout("wrapper") + self.env = {"KAFKA_30_COMMIT": "a" * 40, "KAFKA_39_COMMIT": kafka_commit, + "WRAPPER_COMMIT": wrapper_commit, "WRAPPER_ROOT": str(self.wrapper), + "EVIDENCE_DIR": str(self.directory / "evidence")} + for generation in ("30", "39"): + self.set_archive(generation, self.env[f"KAFKA_{generation}_COMMIT"][:16]) + + def checkout(self, name): + root = self.directory / name + subprocess.run(["git", "init", "-q", str(root)], check=True) + (root / "tracked").write_text(name) + subprocess.run(["git", "-C", str(root), "add", "tracked"], check=True) + subprocess.run(["git", "-C", str(root), "-c", "user.name=Fixture", + "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", + "commit", "-qm", name], check=True) + commit = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip() + return root, commit + + def set_archive(self, generation, encoded): + version = "3.0.1.83" if generation == "30" else "3.9.2.17" + jar = io.BytesIO() + with zipfile.ZipFile(jar, "w") as zipped: + zipped.writestr("kafka/kafka-version.properties", f"version={version}\ncommitId={encoded}\n") + path = self.directory / f"kafka-{generation}.tgz" + with tarfile.open(path, "w:gz") as archive: + for name, data in ((f"kafka-clients-{version}.jar", jar.getvalue()), + (f"kafka_2.12-{version}.jar", b"fixture core")): + entry = tarfile.TarInfo(f"kafka/libs/{name}") + entry.size = len(data) + archive.addfile(entry, io.BytesIO(data)) + self.env[f"KAFKA_{generation}_TGZ"] = str(path) + self.env[f"KAFKA_{generation}_SHA256"] = RELEASE.file_sha256(path) + + def test_matching_archives_and_clean_checkouts_pass_identity_validation(self): + RELEASE.check_inputs(self.root, self.env) + + def test_every_required_input_is_enforced(self): + for name in RELEASE.REQUIRED_INPUTS: + with self.subTest(name=name): + missing = dict(self.env) + missing.pop(name) + with self.assertRaisesRegex(ValueError, name): + RELEASE.check_inputs(self.root, missing) + + def test_refs_and_non_hex_checksums_are_not_immutable_identifiers(self): + for name in ("KAFKA_30_COMMIT", "KAFKA_39_COMMIT", "WRAPPER_COMMIT", + "KAFKA_30_SHA256", "KAFKA_39_SHA256"): + for value in ("main", "a" * 12, "G" * 64): + with self.subTest(name=name, value=value), self.assertRaises(ValueError): + RELEASE.check_inputs(self.root, dict(self.env, **{name: value})) + + def test_archive_checksum_and_source_must_both_match(self): + for generation in ("30", "39"): + for suffix, value in (("SHA256", "0" * 64), ("COMMIT", "0" * 40)): + with self.subTest(generation=generation, suffix=suffix), self.assertRaises(ValueError): + RELEASE.check_inputs(self.root, dict(self.env, **{f"KAFKA_{generation}_{suffix}": value})) + + def test_unknown_or_too_short_embedded_source_is_rejected(self): + for generation in ("30", "39"): + for encoded in ("unknown", "", self.env[f"KAFKA_{generation}_COMMIT"][:11]): + with self.subTest(generation=generation, encoded=encoded): + self.set_archive(generation, encoded) + with self.assertRaisesRegex(ValueError, "Archive source metadata"): + RELEASE.check_inputs(self.root, self.env) + self.set_archive(generation, self.env[f"KAFKA_{generation}_COMMIT"][:16]) + + def test_dirty_checkout_and_wrong_wrapper_head_are_rejected(self): + for root in (self.root, self.wrapper): + for name in ("tracked", "untracked"): + with self.subTest(root=root.name, name=name): + path = root / name + previous = path.read_bytes() if path.exists() else None + path.write_text("modified fixture") + with self.assertRaisesRegex(ValueError, "clean selected checkout"): + RELEASE.check_inputs(self.root, self.env) + if previous is None: + path.unlink() + else: + path.write_bytes(previous) + with self.assertRaisesRegex(ValueError, "clean selected checkout"): + RELEASE.check_inputs(self.root, dict(self.env, WRAPPER_COMMIT="0" * 40)) + + def test_entry_point_forces_full_checks_and_audits_clean_archives(self): + supplied = dict(self.env, SKIP_LOCAL_STAGE="0", BRIDGE_VERIFY_FULL="0", ALLOW_PARTIAL="1") + with mock.patch.dict("os.environ", supplied, clear=True), \ + mock.patch.object(RELEASE, "check_inputs") as check, \ + mock.patch.object(RELEASE.subprocess, "run") as run: + self.assertEqual(0, RELEASE.main()) + check.assert_called_once() + self.assertEqual(2, run.call_count) + for call in run.call_args_list: + env = call.kwargs["env"] + self.assertEqual(("1", "1", "0"), + (env["SKIP_LOCAL_STAGE"], env["BRIDGE_VERIFY_FULL"], env["ALLOW_PARTIAL"])) + self.assertTrue(call.kwargs["check"]) + audit = run.call_args_list[1].args[0] + for flag in ("--require-clean", "--require-full", "--require-archives", "--allow-missing-stage"): + self.assertIn(flag, audit) + + def test_invalid_input_never_launches_the_verifier(self): + with mock.patch.dict("os.environ", {}, clear=True), \ + mock.patch.object(RELEASE.subprocess, "run") as run, mock.patch("sys.stderr", new=io.StringIO()): + self.assertEqual(1, RELEASE.main()) + run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From 03028acfe584d66e2e8e0588ad9fe0ba1d1bb2b0 Mon Sep 17 00:00:00 2001 From: Jon Bringhurst Date: Fri, 11 Sep 2026 12:32:07 -0700 Subject: [PATCH 2/3] docs: correct bucket-list disposition and record the native quota audit --- docs/ops/li-bridge-review-comments.md | 4 +-- docs/ops/li-bridge-review.md | 38 ++++++++++++++++++--------- docs/ops/li-bridge-upgrade.md | 8 +++--- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/docs/ops/li-bridge-review-comments.md b/docs/ops/li-bridge-review-comments.md index 83891b6ad7168..9be4d20202276 100644 --- a/docs/ops/li-bridge-review-comments.md +++ b/docs/ops/li-bridge-review-comments.md @@ -27,7 +27,7 @@ All 62 original threads have replies with published source decisions. A fresh Gr | [F19: recreated topic already assigned to returning replica](https://github.com/linkedin/kafka/pull/584#issuecomment-5631079569) | Paired identity recovery in 585/586; missing/zero IDs, errors, retries, current/future copies and mixed-batch tests. [Qualification update](https://github.com/linkedin/kafka/pull/586#issuecomment-5638005151). | All four revision-4 record checks passed, but full final-source and wrapper qualification remain open. | | [F20: rotated protocol logs omitted](https://github.com/linkedin/kafka/pull/584#issuecomment-5638004839) | PR 584 retains and scans hourly rotations; 586 is restacked on it. New positive and negative tests fail before the fix and pass afterward. | The previous failed CI job stays failed. Re-run the corrected collector. | | [F21: churn exits during controller movement](https://github.com/linkedin/kafka/pull/586#issuecomment-5638182322) | PR 584 fixes the workload retry policy without changing the upstream broker response. [Test and code update](https://github.com/linkedin/kafka/pull/584#issuecomment-5638482746). | The complete revision-4 process run and audit pass; final F22/wrapper qualification remains required. | -| F22: bridge-state MBeans register by default | Paired [588](https://github.com/linkedin/kafka/pull/588)/[589](https://github.com/linkedin/kafka/pull/589) add a default-off diagnostics flag. Tests cover disabled registration, enabled readings, restart scope, KRaft and cleanup. | All 72 Python tests pass. The wrapper mapping still needs matching-jar qualification. | +| F22: bridge-state MBeans register by default | Paired [588](https://github.com/linkedin/kafka/pull/588)/[589](https://github.com/linkedin/kafka/pull/589) add a default-off diagnostics flag. Tests cover disabled registration, enabled readings, restart scope, KRaft and cleanup. | All 72 Python tests pass. The matching-jar wrapper suite now passes 133 tests; the complete final-source verifier remains open. | ## PR 541 @@ -101,7 +101,7 @@ All 62 original threads have replies with published source decisions. A fresh Gr | Comment | Decision | Code/test evidence | Reply | |---|---|---|---| | [1](https://github.com/linkedin/kafka/pull/551#discussion_r3925877328) | Capture the post-truncation offset and size while holding the log lock. | UnifiedLog.truncateTo; truncation test and mixed stale-leader process scenario. | [reply](https://github.com/linkedin/kafka/pull/551#discussion_r3984229174) | -| [2](https://github.com/linkedin/kafka/pull/551#discussion_r3925877401) | Convert malformed numbers to ConfigException. Keep empty lists as an intentional way to disable a bucket dimension. | KafkaConfigTest.testInvalidRequestMetricBuckets and testEmptyRequestMetricBuckets; empty is not rejected because it is supported. | [reply](https://github.com/linkedin/kafka/pull/551#discussion_r3984229326) | +| [2](https://github.com/linkedin/kafka/pull/551#discussion_r3925877401) | Reject empty bucket lists with ConfigException, as requested. The previous reply and ledger incorrectly claimed support for empty lists. | KafkaConfigTest.testEmptyRequestMetricBuckets asserts rejection; malformed, negative and unordered cases have separate assertions. | [correction](https://github.com/linkedin/kafka/pull/551#discussion_r3992669600) | | [3](https://github.com/linkedin/kafka/pull/551#discussion_r3925877442) | Close cumulative counters when their associated ingress metric is removed. | BrokerTopicMetricsTest.testCloseMetricClosesCumulativeIngressCounters; moved to the storage-metrics split. | [reply](https://github.com/linkedin/kafka/pull/551#discussion_r3984229507) | | [4](https://github.com/linkedin/kafka/pull/551#discussion_r3926413418) | Reject negative and non-increasing boundaries. | KafkaConfigTest.testRequestMetricBucketsMustBeOrderedAndNonNegative. | [reply](https://github.com/linkedin/kafka/pull/551#discussion_r3984229700) | | [5](https://github.com/linkedin/kafka/pull/551#discussion_r3926413465) | Assert on the specific metric names, not global registry size. | BrokerTopicMetricsTest uses per-metric presence checks; storage split tests pass. | [reply](https://github.com/linkedin/kafka/pull/551#discussion_r3984229872) | diff --git a/docs/ops/li-bridge-review.md b/docs/ops/li-bridge-review.md index b1b45dcbc08f2..006918b2aa47a 100644 --- a/docs/ops/li-bridge-review.md +++ b/docs/ops/li-bridge-review.md @@ -19,17 +19,17 @@ limitations under the License. ## Current verdict -**Do not deploy this candidate. The coverage audit found two offline topic-name-reuse failures after an earlier full verifier passed.** Both paired repairs are published in 583–586, but final qualification remains incomplete. The current inventory has 39 open PRs. The complete revision-4 process run now passes, including all four offline-reuse record checks. The later diagnostic-metrics opt-in and matching wrapper still need final qualification. An earlier green bundle or CI run does not cover a scenario it never exercised. +**Do not deploy this candidate. The coverage audit found two offline topic-name-reuse failures after an earlier full verifier passed.** Both paired repairs are published in 583–586, but final qualification remains incomplete. The current inventory has 41 open PRs. The complete revision-4 process run now passes, including all four offline-reuse record checks. The diagnostic-metrics opt-in and matching wrapper pass their scoped suites. The full verifier is running on that pair; the later quota regression repair still needs final-source qualification. An earlier green bundle or CI run does not cover a scenario it never exercised. The findings below record what the initial review and later tests found. Instructions in an original finding describe the repair that was needed; use the current disposition and evidence sections for status. This is not a line-by-line approval of every Kafka change. ### Reviewed revisions - Workspace plan: `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`. Canonical runbook: `docs/ops/li-bridge-upgrade.md`. -- Current 3.9 behavior change: PR 589, `43b03f35049efa5510a40a6a46bb2bf78bf4750a`; its documentation update follows that commit. +- Current 3.9 behavior change: PR 590, `3.9-li-bridge/native-quota-window`; release-input tests and documentation follow in PR 591. - Current 3.0 source: PR 588, `ec940efaacdebf38b82c786e61ac46e91eecd5da`. - CI: PR 558, `3e799b08ea`; PR 559, `a86214e2da`. -- Wrapper: `1a9ecccf`, including the ACL test fix `6ddf2a87`. +- Wrapper: `1764cc95bfa21e19d3ff89e0164e5896808b5507`, including the diagnostic opt-in and the earlier ACL test fix. GitHub heads, bases, labels and sizes were read back after publication. The upgrade stacks rooted at PRs 543 and 575 retain separate release histories. CI foundations have no upgrade label. PRs 563/564 were automatically closed during the reorder; replacement reviews 579/578 preserve those scopes. No release branch was merged by this work. Original findings began from aggregate `a658c512df`; aggregate PR 542 and old docs PR 555 remain closed. @@ -279,7 +279,17 @@ Both generations constructed `LiProtocolBridgeMetrics` and registered new MBeans PRs 588/589 add `li.protocol.bridge.config.metrics.enable`, default false, ZooKeeper-only and restart-scoped. Enabled diagnostics still report disabled behavior flags without activating them. The existing constructors remain available. A disabled instance does not remove an enabled instance's gauges during cleanup. -The default-off test failed on both previous implementations and passes with the repair. Tests also cover enabled readings, dynamic behavior flags, ignored live updates to the restart-only setting, KRaft exclusion and cleanup. The selected 3.0 and 3.9 suites passed 22 and 39 tests respectively. All 72 Python tests pass, including missing/false opt-in rejection in every migration phase. The process profile enables diagnostics explicitly. The wrapper mapping and its new negative test still need qualification with matching staged jars. +The default-off test failed on both previous implementations and passes with the repair. Tests also cover enabled readings, dynamic behavior flags, ignored live updates to the restart-only setting, KRaft exclusion and cleanup. The selected 3.0 and 3.9 suites passed 22 and 39 tests respectively. All 72 Python tests pass, including missing/false opt-in rejection in every migration phase. The process profile enables diagnostics explicitly. The wrapper mapping and its new negative test pass with matching staged jars: 133 tests, no failures or skips, unchanged main/test-classifier hashes. This scoped pass does not replace the complete verifier. + +### F23 — P2: Static quota fallback changes a flag-off native limit + +`ClientQuotaManager.getMaxValueInQuotaWindow` treated any limit at or above `Long.MaxValue` as absent. That also changed an explicit dynamic quota when the bridge static-default flag was off. A new regression expected the native finite window limit `9.223372036854776E19`, but got `Double.MaxValue`. + +PR 590 restores the original callback calculation and uses the gated static default only when the callback returns no limit. The large explicit-quota and fallback-removal regressions pass with the full `ClientQuotaManagerTest`, `QuotaFactoryTest` and `RequestQuotaTest` suites. No default is enabled by this repair. + +### Review-record correction — empty metric buckets are rejected + +The original reply and ledger for PR 551 comment 2 incorrectly said empty buckets were supported. The published parser and `testEmptyRequestMetricBuckets` reject them with `ConfigException`, as the reviewer requested. The [correction](https://github.com/linkedin/kafka/pull/551#discussion_r3992669600) is explicit; the original reply remains in the history. No code was changed to match the inaccurate reply. ## PR dispositions and dependency audit @@ -326,12 +336,14 @@ Every PR below has a distinct migration or CI purpose. Keep these scopes, but do | [586](https://github.com/linkedin/kafka/pull/586) | 3.9 topic identity and scenario-revision-4 qualification — storage/verification | | [587](https://github.com/linkedin/kafka/pull/587) | current findings, evidence limits and completion checklist — operations/review | | [589](https://github.com/linkedin/kafka/pull/589) | 3.9 diagnostic-metrics opt-in and admission checks — observability/verification | +| [590](https://github.com/linkedin/kafka/pull/590) | native quota-window behavior and gated fallback regression — quotas | +| [591](https://github.com/linkedin/kafka/pull/591) | release-input negative tests and final audit records — verification/release | Merge CI 558/559 into their own release branches first. Then retarget the upgrade stack bottoms as described in the runbook. Never force a Git dependency between the 3.0 and 3.9 CI branches. When reordering again, change PR bases before pushing a head that becomes an ancestor of its former base; GitHub can otherwise auto-close and delete that branch. The oversized original scopes were split. Control wire definitions are in 565 (503 lines), handlers in 545 (845); storage metrics are in 566 (221), broker metrics/watchdog wiring in 551 (910); workload helpers precede runner 553 (754). The new evidence auditor, verifier and release gate are separate layers. All 36 implementation/CI diffs were below 1,000 changed lines at readback. This documentation follow-up stays separate from the 863-line original runbook PR. -The original 62 review threads now have replies with published source decisions and code/test references. See `docs/ops/li-bridge-review-comments.md`. The static `LeaderTransferManager.noOp()` call is valid: javap confirms the forwarder, and the Java builder compiles. Empty metric bucket lists remain intentionally supported; malformed, negative and unordered boundaries are rejected. The latest readback covers 37 PRs and finds all 62 expected replies, no mismatches and no new review threads. The later F18/F19 issue comments are retained and have follow-up code and qualification records. Re-fetch after the final publication; comment status is not proof that the code is correct. +The original 62 review threads now have replies with published source decisions and code/test references. See `docs/ops/li-bridge-review-comments.md`. The static `LeaderTransferManager.noOp()` call is valid: javap confirms the forwarder, and the Java builder compiles. Empty, malformed, negative and unordered metric bucket lists are rejected. The earlier claim that empty lists were supported has been corrected against the actual source and test assertion. The latest readback covers 37 PRs and finds all 62 expected replies, no mismatches and no new review threads. The later F18/F19 issue comments are retained and have follow-up code and qualification records. Re-fetch after the final publication; comment status is not proof that the code is correct. ## Verification and remaining requirements @@ -341,23 +353,23 @@ This prompt-to-artifact checklist separates observed results from open requireme | Requirement | Artifact and verification surface | Evidence / open work | |---|---|---| -| Review the named plan and every open public PR | `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`; canonical runbook; GitHub inventory | 39 open PRs are listed in both tables. Recheck the exact PR set after any further publication. | -| Explain scope and dependencies | PR responsibility table, heads/bases, stack membership | Separate protocol, handlers, storage metrics, runner, auditor, verifier and release-gate layers. Stack 582 has 31 upgrade PRs; stack 581 has six. CI 558/559 remain on independent release histories. No release branch was merged. | -| Apply the requested label | GitHub labels | All 37 upgrade PRs have `kafka-upgrade-august-2026`; CI 558/559 do not. | +| Review the named plan and every open public PR | `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`; canonical runbook; GitHub inventory | 41 open PRs are listed in both tables. Recheck the exact PR set after any further publication. | +| Explain scope and dependencies | PR responsibility table, heads/bases, stack membership | Separate protocol, handlers, storage metrics, runner, auditor, verifier and release-gate layers. Stack 582 has 33 upgrade PRs; stack 581 has six. CI 558/559 remain on independent release histories. No release branch was merged. | +| Apply the requested label | GitHub labels | All 39 upgrade PRs have `kafka-upgrade-august-2026`; CI 558/559 do not. | | Keep PRs below 1,000 changed lines, preferably near 500 | Additions plus deletions, not file length | Established diffs passed the limit; recheck the new metrics follow-ups after publication. The largest established diffs are 981 and 963 lines. | | Use plain, direct English | Plan, review, comment replies and workflow comments | Final wording/link review remains required. Historical findings are not current deployment instructions. | | Gate every Kafka behavior change | `KafkaConfig`, `DynamicBrokerConfig`, runtime call sites, metrics, wrapper mapping | 24 default-off 3.9 gates. F18/F19 use the cleanup gate; ISR retry repair uses bridge mode; F22 separately gates diagnostic registration. Dedicated disabled/activation tests pass. The final per-change call-site audit remains open. | | Select symmetric v2/v5/v1 control | Schemas, controller selectors, wire fixtures and retained logs | Both generations have fixtures and real-process coverage. F20 makes rotated log checks fail closed too. Final-source process qualification remains required. | | Fence activation and preserve callbacks | `RequestSendThreadBridgeTest` | Blocked dequeue, sustained queue and admitted-deletion callback tests pass. Controller restart remains mandatory. | -| Enforce all six phases and unchanged clients | `li_bridge_contract.py`, preflight, persistent client/Streams/Connect and private-API helpers | 72 Python tests pass. Native control stays at IBP 3.0 before the separate IBP roll. One 3.0 archive is not the deployed client/tool floor. | +| Enforce all six phases and unchanged clients | `li_bridge_contract.py`, preflight, persistent client/Streams/Connect and private-API helpers | 80 Python tests pass. Native control stays at IBP 3.0 before the separate IBP roll. One 3.0 archive is not the deployed client/tool floor. | | Prove persisted rollback and recovery | Process runner, record helper, timings and JUnit | Historical full run covers canary/all-3.9 rollback, cancellation, crashes and truncation. Repeat against the final source; registration or ISR alone is not proof. | | Prove deletion and name reuse | `TopicDeletionManager`, `ZkMetadataCache`, `ReplicaManager`, `BridgeTopicIdentity` | Gated F12/F13/F15/F17/F18/F19 repairs have tests. Revision 4 requires four post-promotion record checks. They passed in the current scoped checks, but do not replace complete qualification. | | Handle version-changing ISR retries | `BridgeAlterPartitionRetryTest` | One queued builder crosses versions 3/1 and activation without reusing mutated request data. Earlier failed logs remain failed evidence. | | Collect real runtime/configuration/state | Live inventory, runtime probe and negative-input tests | Disposable-cluster collection passes. Production binaries, settings, state dispositions, owners and client/tool floor remain required inputs. | | Qualify pagination with the loaded runtime | `KafkaZkClient`, five vendor-client tests and release runtime probe | Startup rejects an unsupported client. Vendor tests pass. The actual deployed client/Jute/server pairing remains a release gate. | | Follow Google shell style, including comments | Four wrappers, all extracted workflow Bash blocks, ShellCheck, shfmt, syntax/length checks | The refreshed audit checks exact reviewed source revisions: 54 changed-workflow blocks and four wrappers pass ShellCheck, shfmt, syntax, no-tab and 80-column checks. Unmodified upstream Docker workflows are outside these PRs; the broader diagnostic results are retained separately. | -| Preserve wrapper/API compatibility | Factory mapping tests, ACL tests, complete wrapper suite and jar comparison | Historical 132-test suite passed with matching jars and stable hashes. The required Mint refresh has now produced a fresh dependency spec. Test the new metrics mapping against matching staged jars; no TTL or artifact-identity bypass is allowed. | -| Qualify real archives and reject incomplete evidence | `verify_li_bridge.sh`, `audit_li_bridge_evidence.py`, `verify_li_bridge_release.sh` and negative fixtures | Earlier full verifier passed, but predates F18/F19. Final full-source verification and release-guard inputs/negative behavior remain open. | +| Preserve wrapper/API compatibility | Factory mapping tests, ACL tests, complete wrapper suite and jar comparison | The current 133-test suite passes with matching main jars, stable main/test-classifier hashes and unchanged source. Wrapper commit 1764cc95 is published. The required Mint refresh produced a fresh dependency spec; no TTL or artifact-identity bypass was used. | +| Qualify real archives and reject incomplete evidence | `verify_li_bridge.sh`, `audit_li_bridge_evidence.py`, `verify_li_bridge_release.sh` and negative fixtures | Earlier full verifier passed but predates F18/F19. The metrics-gated full run is active. Eight new release-input tests cover missing inputs, malformed identifiers, mismatched/unknown archive metadata, dirty/wrong checkouts, forced full mode and rejection before launch. A real-archive identity-only check also passes; neither it nor the fixtures grant release approval. | | Address every review comment with evidence | Comment ledger, source/test decisions, GraphQL readback | All original 62 replies verified across 37 PRs; no new review threads. Later issue findings have published fixes and explicit qualification limits. Re-fetch after final publication. | | Keep the three documents consistent | Workspace files and `docs/ops/li-bridge-{upgrade,review,review-comments}.md` | This follow-up synchronizes the records. Verify relative links and actual PR/source/evidence state before calling the review complete. | | Preserve side-task PR 2039 | ADU worktree, commit history and formatting checks | Rebased/pushed on master at `ae007420`; formatting is one separate commit and is idempotent. Functional patches remain unchanged. | @@ -385,7 +397,9 @@ Later evidence supersedes the inventory and coverage limits of those historical The complete revision-4 process run `/tmp/li-scenario-4-churn-fixed` passed on clean `d8f255f8a4` / `1a5d02403b`, with all four name-reuse checks, unchanged source and an issue-free process audit. It includes F20/F21 but predates F22. The real `mint --no-metrics dependency create-dependency-spec --detect-variant --overwrite` command has now refreshed the wrapper metadata successfully. An invocation without `--overwrite` returned success without refreshing the expired file; that no-op was not accepted as freshness evidence. -No complete bundle yet covers the final metrics opt-in, both binaries and wrapper together. Failed runs remain failed. A source, archive or scenario change must be checked against the fingerprint before reuse. +`/tmp/li-wrapper-metrics-qualification` retains the 133-test wrapper result and matching before/after artifact reports. `/tmp/li-release-input-regressions.log` records 80 passing Python tests; `/tmp/li-release-input-real-check.json` is explicitly identity-validation-only. `/tmp/li-native-quota-before.log` preserves the flag-off failure; `/tmp/li-native-quota-after.log` passes after the repair. + +No complete bundle yet covers the quota correction, final metrics opt-in, both binaries and wrapper together. Failed runs remain failed. A source, archive or scenario change must be checked against the fingerprint before reuse. ### Inputs still required before production diff --git a/docs/ops/li-bridge-upgrade.md b/docs/ops/li-bridge-upgrade.md index d0ab9013736c3..75336d12731a8 100644 --- a/docs/ops/li-bridge-upgrade.md +++ b/docs/ops/li-bridge-upgrade.md @@ -23,7 +23,7 @@ limitations under the License. The implementation base is Apache **3.9.2** with the reviewed LI bridge stack. Pin the final internal `3.9.2.N`, matching `3.0.1.N`, wrapper commit, archive checksums, JDKs and ZooKeeper runtime in the release record. A maintenance-baseline change requires a new qualification run; do not substitute a newer tag during rollout. -The current implementation is the split stack through `3.9-li-bridge/config-metrics-gate`, with the companion `3.0-li-bridge/config-metrics-gate` branch. It is not the closed aggregate PR 542. The canonical mergeable runbook is `docs/ops/li-bridge-upgrade.md`; the workspace copy is `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`. The `3.9-li-bridge/review-refresh` branch updates the documentation after the behavior fixes. Historical experiments are evidence, not current acceptance criteria. +The current implementation is the split stack through `3.9-li-bridge/release-input-regressions`, with the companion `3.0-li-bridge/config-metrics-gate` branch. It is not the closed aggregate PR 542. The canonical mergeable runbook is `docs/ops/li-bridge-upgrade.md`; the workspace copy is `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`. The `3.9-li-bridge/review-refresh` branch updates the documentation after the behavior fixes. Historical experiments are evidence, not current acceptance criteria. **Clients do not change.** The supported producer, consumer, transactional client, Streams application, Connect worker, LI AdminClient and operational-tool artifacts/configuration must remain unchanged across every phase. Discovery must name their deployed version floor and owners. One 3.0 test archive is not proof for every externally deployed client. @@ -236,7 +236,7 @@ Automatically stop for unexpected control versions, post-fence API 1001 traffic, ## PR inventory and merge order -All 39 open public PRs are covered below. Upgrade PRs carry `kafka-upgrade-august-2026`; CI foundations 558 and 559 do not. All current diffs are below 1,000 changed lines. These checks do not grant approval to deploy. +All 41 open public PRs are covered below. Upgrade PRs carry `kafka-upgrade-august-2026`; CI foundations 558 and 559 do not. All current diffs are below 1,000 changed lines. These checks do not grant approval to deploy. Closed PRs 542 and 555 are superseded. GitHub automatically closed 563 and 564 during the dependency reorder because their new heads were contained in their former base branches. No release branch was merged. Their restored, separate reviews are 579 and 578. @@ -281,12 +281,14 @@ Closed PRs 542 and 555 are superseded. GitHub automatically closed 563 and 564 d | [586](https://github.com/linkedin/kafka/pull/586) | 3.9 topic identity and scenario-revision-4 qualification — storage/verification | | [587](https://github.com/linkedin/kafka/pull/587) | current findings, evidence limits and completion checklist — operations/review | | [589](https://github.com/linkedin/kafka/pull/589) | 3.9 diagnostic-metrics opt-in and admission checks — observability/verification | +| [590](https://github.com/linkedin/kafka/pull/590) | native quota-window behavior and gated fallback regression — quotas | +| [591](https://github.com/linkedin/kafka/pull/591) | release-input negative tests and final audit records — verification/release | Merge 558 into `3.9-li` and 559 into `3.0-li` first. They have different release bases, so do not put them in one dependent Git stack. Rebase/retarget 575 to `3.0-li` and 543 to `3.9-li`; do not merge feature work into temporary CI branches. The GitHub stack rooted at PR 575 has the 3.0 order **575 → 541 → 577 → 583 → 585 → 588**. The stack rooted at PR 543 has the 3.9 order: -**543 → 544 → 565 → 545 → 546 → 547 → 548 → 560 → 549 → 550 → 561 → 566 → 551 → 567 → 576 → 568 → 578 → 579 → 552 → 569 → 570 → 571 → 553 → 572 → 554 → 573 → 574 → 584 → 586 → 587 → 589**. +**543 → 544 → 565 → 545 → 546 → 547 → 548 → 560 → 549 → 550 → 561 → 566 → 551 → 567 → 576 → 568 → 578 → 579 → 552 → 569 → 570 → 571 → 553 → 572 → 554 → 573 → 574 → 584 → 586 → 587 → 589 → 590 → 591**. Retarget remaining layers after each independent merge. The wrapper branch contains the ACL test repair (`6ddf2a87`) and cleanup mapping/tests (`1a9ecccf`); its source suite passes 132 tests. Add the approved wrapper/dependency/security PR and named deployment-gate owner to the release record. From 3ec910696721f826c87d618ed4f0d90d13da26d8 Mon Sep 17 00:00:00 2001 From: Jon Bringhurst Date: Fri, 11 Sep 2026 12:39:17 -0700 Subject: [PATCH 3/3] docs: retain the failed dormant-backout qualification as a blocker --- docs/ops/li-bridge-review-comments.md | 2 +- docs/ops/li-bridge-review.md | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/ops/li-bridge-review-comments.md b/docs/ops/li-bridge-review-comments.md index 9be4d20202276..1173d98b29daa 100644 --- a/docs/ops/li-bridge-review-comments.md +++ b/docs/ops/li-bridge-review-comments.md @@ -17,7 +17,7 @@ limitations under the License. # Review comment dispositions -All 62 original threads have replies with published source decisions. A fresh GraphQL readback across 37 PRs verified every expected reply, found no mismatches and found no new review threads. Resolved status alone was not accepted as proof. Re-fetch after the final publication and check the actual source/test coverage before closing the review. +All 62 original threads have replies with published source decisions. A fresh GraphQL readback across 41 PRs verified every expected reply, found no mismatches and found no new review threads. Resolved status alone was not accepted as proof. Re-fetch after the final publication and check the actual source/test coverage before closing the review. ## Later qualification findings diff --git a/docs/ops/li-bridge-review.md b/docs/ops/li-bridge-review.md index 006918b2aa47a..078f0e073ca48 100644 --- a/docs/ops/li-bridge-review.md +++ b/docs/ops/li-bridge-review.md @@ -19,7 +19,7 @@ limitations under the License. ## Current verdict -**Do not deploy this candidate. The coverage audit found two offline topic-name-reuse failures after an earlier full verifier passed.** Both paired repairs are published in 583–586, but final qualification remains incomplete. The current inventory has 41 open PRs. The complete revision-4 process run now passes, including all four offline-reuse record checks. The diagnostic-metrics opt-in and matching wrapper pass their scoped suites. The full verifier is running on that pair; the later quota regression repair still needs final-source qualification. An earlier green bundle or CI run does not cover a scenario it never exercised. +**Do not deploy this candidate. The coverage audit found two offline topic-name-reuse failures after an earlier full verifier passed.** Both paired repairs are published in 583–586, but final qualification remains incomplete. The current inventory has 41 open PRs. The complete revision-4 process run now passes, including all four offline-reuse record checks. The diagnostic-metrics opt-in and matching wrapper pass their scoped suites. The full verifier on that pair passed its build/JVM/wrapper stages but failed the all-3.0 dormant-backout churn-progress deadline. This failure and the later quota repair still need final-source qualification. An earlier green bundle or CI run does not cover a scenario it never exercised. The findings below record what the initial review and later tests found. Instructions in an original finding describe the repair that was needed; use the current disposition and evidence sections for status. This is not a line-by-line approval of every Kafka change. @@ -343,7 +343,7 @@ Merge CI 558/559 into their own release branches first. Then retarget the upgrad The oversized original scopes were split. Control wire definitions are in 565 (503 lines), handlers in 545 (845); storage metrics are in 566 (221), broker metrics/watchdog wiring in 551 (910); workload helpers precede runner 553 (754). The new evidence auditor, verifier and release gate are separate layers. All 36 implementation/CI diffs were below 1,000 changed lines at readback. This documentation follow-up stays separate from the 863-line original runbook PR. -The original 62 review threads now have replies with published source decisions and code/test references. See `docs/ops/li-bridge-review-comments.md`. The static `LeaderTransferManager.noOp()` call is valid: javap confirms the forwarder, and the Java builder compiles. Empty, malformed, negative and unordered metric bucket lists are rejected. The earlier claim that empty lists were supported has been corrected against the actual source and test assertion. The latest readback covers 37 PRs and finds all 62 expected replies, no mismatches and no new review threads. The later F18/F19 issue comments are retained and have follow-up code and qualification records. Re-fetch after the final publication; comment status is not proof that the code is correct. +The original 62 review threads now have replies with published source decisions and code/test references. See `docs/ops/li-bridge-review-comments.md`. The static `LeaderTransferManager.noOp()` call is valid: javap confirms the forwarder, and the Java builder compiles. Empty, malformed, negative and unordered metric bucket lists are rejected. The earlier claim that empty lists were supported has been corrected against the actual source and test assertion. The latest readback covers 41 PRs and finds all 62 expected replies, no mismatches and no new review threads. The later F18/F19 issue comments are retained and have follow-up code and qualification records. Re-fetch after the final publication; comment status is not proof that the code is correct. ## Verification and remaining requirements @@ -369,8 +369,8 @@ This prompt-to-artifact checklist separates observed results from open requireme | Qualify pagination with the loaded runtime | `KafkaZkClient`, five vendor-client tests and release runtime probe | Startup rejects an unsupported client. Vendor tests pass. The actual deployed client/Jute/server pairing remains a release gate. | | Follow Google shell style, including comments | Four wrappers, all extracted workflow Bash blocks, ShellCheck, shfmt, syntax/length checks | The refreshed audit checks exact reviewed source revisions: 54 changed-workflow blocks and four wrappers pass ShellCheck, shfmt, syntax, no-tab and 80-column checks. Unmodified upstream Docker workflows are outside these PRs; the broader diagnostic results are retained separately. | | Preserve wrapper/API compatibility | Factory mapping tests, ACL tests, complete wrapper suite and jar comparison | The current 133-test suite passes with matching main jars, stable main/test-classifier hashes and unchanged source. Wrapper commit 1764cc95 is published. The required Mint refresh produced a fresh dependency spec; no TTL or artifact-identity bypass was used. | -| Qualify real archives and reject incomplete evidence | `verify_li_bridge.sh`, `audit_li_bridge_evidence.py`, `verify_li_bridge_release.sh` and negative fixtures | Earlier full verifier passed but predates F18/F19. The metrics-gated full run is active. Eight new release-input tests cover missing inputs, malformed identifiers, mismatched/unknown archive metadata, dirty/wrong checkouts, forced full mode and rejection before launch. A real-archive identity-only check also passes; neither it nor the fixtures grant release approval. | -| Address every review comment with evidence | Comment ledger, source/test decisions, GraphQL readback | All original 62 replies verified across 37 PRs; no new review threads. Later issue findings have published fixes and explicit qualification limits. Re-fetch after final publication. | +| Qualify real archives and reject incomplete evidence | `verify_li_bridge.sh`, `audit_li_bridge_evidence.py`, `verify_li_bridge_release.sh` and negative fixtures | Earlier full verifier passed but predates F18/F19. The metrics-gated full run failed during the process phase after its earlier stages passed. Eight new release-input tests cover missing inputs, malformed identifiers, mismatched/unknown archive metadata, dirty/wrong checkouts, forced full mode and rejection before launch. A real-archive identity-only check also passes; neither it nor the fixtures grant release approval. | +| Address every review comment with evidence | Comment ledger, source/test decisions, GraphQL readback | All original 62 replies verified across 41 PRs; no new review threads. Later issue findings have published fixes and explicit qualification limits. Re-fetch after final publication. | | Keep the three documents consistent | Workspace files and `docs/ops/li-bridge-{upgrade,review,review-comments}.md` | This follow-up synchronizes the records. Verify relative links and actual PR/source/evidence state before calling the review complete. | | Preserve side-task PR 2039 | ADU worktree, commit history and formatting checks | Rebased/pushed on master at `ae007420`; formatting is one separate commit and is idempotent. Functional patches remain unchanged. | @@ -393,12 +393,14 @@ Later evidence supersedes the inventory and coverage limits of those historical - `/tmp/li-scenario-4-batch-fixed`: all four offline-reuse record checks passed, but the complete run failed at the native checkpoint. The old metadata-churn helper exited on `ControllerMovedException` during controller movement. Its progress had advanced to 221 cycles. The unchanged upstream fence and helper retry gap are covered by F21; do not waive this failed run. The summary records `passed=false` and unchanged source. Rotated logs omitted by that older collector are retained separately in `/tmp/li-scenario-4-batch-fixed-rotated-logs.tgz`. - `/tmp/li-log-rotation-before.log`: both new rotation regressions fail before F20. `/tmp/li-log-rotation-restacked.log`: all 70 Python tests pass afterward. - `/tmp/li-churn-retry-before-{3.0,3.9}.log`: real client error classes expose the missing controller retry and incorrectly retryable storage/corruption errors. `/tmp/li-churn-retry-after-{3.0,3.9}.log` and `/tmp/li-churn-retry-restacked-python.log` pass with the repair. -- `/tmp/li-review-readback-result.json`: 37 PRs, 62 original threads, no missing/mismatched replies and no new threads at that readback. +- `/tmp/li-review-readback-result.json`: 41 PRs, 62 original threads, no missing/mismatched replies and no new threads at that readback. The complete revision-4 process run `/tmp/li-scenario-4-churn-fixed` passed on clean `d8f255f8a4` / `1a5d02403b`, with all four name-reuse checks, unchanged source and an issue-free process audit. It includes F20/F21 but predates F22. The real `mint --no-metrics dependency create-dependency-spec --detect-variant --overwrite` command has now refreshed the wrapper metadata successfully. An invocation without `--overwrite` returned success without refreshing the expired file; that no-op was not accepted as freshness evidence. `/tmp/li-wrapper-metrics-qualification` retains the 133-test wrapper result and matching before/after artifact reports. `/tmp/li-release-input-regressions.log` records 80 passing Python tests; `/tmp/li-release-input-real-check.json` is explicitly identity-validation-only. `/tmp/li-native-quota-before.log` preserves the flag-off failure; `/tmp/li-native-quota-after.log` passes after the repair. +`/tmp/li-final-metrics-full` reached the process stage after all required earlier stages passed, then failed during the all-3.0 dormant backout. The ordinary old-client checkpoint completed, but metadata churn stayed at cycle 20: topic creation returned TopicExists while deletion reported a missing ZooKeeper topic. The source was unchanged. This is a failed qualification, not a deadline to waive; the controller/cache mismatch still needs diagnosis. + No complete bundle yet covers the quota correction, final metrics opt-in, both binaries and wrapper together. Failed runs remain failed. A source, archive or scenario change must be checked against the fingerprint before reuse. ### Inputs still required before production