From 96774475aaf67a7dc2d573737faa311768b71e63 Mon Sep 17 00:00:00 2001 From: Jon Bringhurst Date: Fri, 11 Sep 2026 18:07:20 -0700 Subject: [PATCH 1/2] tests: assert bridge opt-ins at request and storage boundaries --- .../LiReassignmentCancellationGateTest.scala | 78 +++++++++++++++++ .../unit/kafka/cluster/PartitionTest.scala | 13 +-- .../server/DynamicBrokerConfigTest.scala | 19 +++-- .../unit/kafka/server/KafkaApisTest.scala | 84 +++++++++++++++++++ .../server/LiProtocolBridgeConfigTest.scala | 10 +++ tests/bin/li_bridge_test_selection.ini | 3 + 6 files changed, 195 insertions(+), 12 deletions(-) create mode 100644 core/src/test/scala/integration/kafka/server/LiReassignmentCancellationGateTest.scala diff --git a/core/src/test/scala/integration/kafka/server/LiReassignmentCancellationGateTest.scala b/core/src/test/scala/integration/kafka/server/LiReassignmentCancellationGateTest.scala new file mode 100644 index 0000000000000..c1355758eb783 --- /dev/null +++ b/core/src/test/scala/integration/kafka/server/LiReassignmentCancellationGateTest.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.api.LeaderAndIsr +import kafka.controller.{LeaderIsrAndControllerEpoch, ReplicaAssignment} +import kafka.utils.TestUtils +import kafka.zk.ZkVersion +import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, NewPartitionReassignment} +import org.apache.kafka.common.{TopicPartition, Uuid} +import org.apache.kafka.common.errors.InvalidReplicaAssignmentException +import org.apache.kafka.server.config.ServerConfigs +import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows, assertTrue} +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource + +import java.util.{Collections, Optional, Properties} +import java.util.concurrent.{ExecutionException, TimeUnit} + +class LiReassignmentCancellationGateTest extends QuorumTestHarness { + private var broker: KafkaServer = _ + + @AfterEach + override def tearDown(): Unit = { + if (broker != null) broker.shutdown() + super.tearDown() + } + + @ParameterizedTest + @ValueSource(booleans = Array(false, true)) + def testCancellationGateAtControllerRequestBoundary(enabled: Boolean): Unit = { + val tp = new TopicPartition("cancel-gate", 0) + val topicId = Some(Uuid.randomUuid()) + // Resume an existing move from [0,1] to [2], with only original replica 0 online. + // Seeding before startup avoids a race with a destination catching up in the fixture. + zkClient.createTopicAssignment(tp.topic, topicId, Map(tp -> Seq(0, 1))) + zkClient.setTopicAssignment(tp.topic, topicId, + Map(tp -> ReplicaAssignment(Seq(2, 0, 1), Seq(2), Seq(0, 1)))) + zkClient.createTopicPartitionStatesRaw( + Map(tp -> LeaderIsrAndControllerEpoch(LeaderAndIsr(0, List(0)), 0)), ZkVersion.MatchAnyVersion) + .foreach(_.maybeThrow()) + val props = TestUtils.createBrokerConfig(0, zkConnect) + props.put(ServerConfigs.CONTROLLED_SHUTDOWN_ENABLE_CONFIG, "false") + props.put(KafkaConfig.LiMinOriginalAliveReplicasProp, "2") + if (enabled) props.put(KafkaConfig.LiProtocolBridgeReassignmentCancellationSafetyEnableProp, "true") + broker = createBroker(KafkaConfig.fromProps(props)).asInstanceOf[KafkaServer] + assertEquals(0, TestUtils.waitUntilControllerElected(zkClient)) + val adminProps = new Properties + adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + TestUtils.bootstrapServers(Seq(broker), broker.config.interBrokerListenerName)) + val admin = Admin.create(adminProps) + try { + val result = admin.alterPartitionReassignments( + Collections.singletonMap(tp, Optional.empty[NewPartitionReassignment]())).all() + if (enabled) { + val error = assertThrows(classOf[ExecutionException], () => result.get(10, TimeUnit.SECONDS)) + assertTrue(error.getCause.isInstanceOf[InvalidReplicaAssignmentException]) + } else result.get(10, TimeUnit.SECONDS) + val assignment = zkClient.getFullReplicaAssignmentForTopics(Set(tp.topic))(tp) + assertEquals(if (enabled) Seq(2) else Seq(0, 1), assignment.targetReplicas) + } finally admin.close() + } +} diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala index 22ff99a87ae36..ebaea0696fc0d 100644 --- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala +++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala @@ -2154,8 +2154,9 @@ class PartitionTest extends AbstractPartitionTest { assertEquals(log.logEndOffset, partition.localLogOrException.highWatermark) } - @Test - def testLeaderTransferSelectsLowestInSyncFollower(): Unit = { + @ParameterizedTest + @ValueSource(booleans = Array(false, true)) + def testLeaderTransferSelectsLowestInSyncFollower(enabled: Boolean): Unit = { configRepository.setTopicConfig(topicPartition.topic, TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "4") val log = logManager.getOrCreateLog(topicPartition, topicId = None) seedLogData(log, numRecords = 10, leaderEpoch = 4) @@ -2177,7 +2178,7 @@ class PartitionTest extends AbstractPartitionTest { metadataCache, logManager, alterPartitionManager, - leaderTransferEnabled = true, + leaderTransferEnabled = enabled, leaderTransferManager = leaderTransferManager) transferPartition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None) assertTrue(transferPartition.makeLeader( @@ -2196,9 +2197,11 @@ class PartitionTest extends AbstractPartitionTest { fetchFollower(transferPartition, replicaId = lowestInSyncFollower, fetchOffset = log.logEndOffset) time.sleep(transferPartition.replicaLagTimeMaxMs + 1) transferPartition.maybeTransferToNewLeader() - - verify(leaderTransferManager).submit(topicPartition, lowestInSyncFollower) + if (enabled) verify(leaderTransferManager).submit(topicPartition, lowestInSyncFollower) verifyNoMoreInteractions(leaderTransferManager) + + transferPartition.maybeShrinkIsr() + assertEquals(if (enabled) 0 else 1, alterPartitionManager.isrUpdates.size) } @Test diff --git a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala index 86f83526a3ed0..ee903ec1dcbce 100755 --- a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala @@ -317,7 +317,7 @@ class DynamicBrokerConfigTest { KafkaConfig.LiProtocolBridgeReassignmentCancellationSafetyEnableProp, KafkaConfig.LiProtocolBridgeProduceRequestInstrumentationEnableProp ) - val allBridgeFlags = dynamicBridgeFlags :+ KafkaConfig.LiProtocolBridgeLeaderTransferEnableProp + val allBridgeFlags = KafkaConfig.LiProtocolBridgeEnableProps allBridgeFlags.foreach(flag => assertFalse(config.getBoolean(flag), s"$flag should be disabled by default")) val dynamicProps = new Properties @@ -325,12 +325,17 @@ class DynamicBrokerConfigTest { config.dynamicConfig.validate(dynamicProps, perBrokerConfig = false) config.dynamicConfig.updateDefaultConfig(dynamicProps) dynamicBridgeFlags.foreach(flag => assertTrue(config.getBoolean(flag), s"$flag should be enabled dynamically")) - assertThrows(classOf[ConfigException], () => config.dynamicConfig.validate(dynamicProps, perBrokerConfig = true)) - - val startupOnlyProps = new Properties - startupOnlyProps.put(KafkaConfig.LiProtocolBridgeLeaderTransferEnableProp, "true") - assertThrows(classOf[ConfigException], - () => config.dynamicConfig.validate(startupOnlyProps, perBrokerConfig = false)) + // Test each flag alone so one rejected key cannot hide an allowed override. + allBridgeFlags.foreach { flag => + val oneFlag = new Properties + oneFlag.put(flag, "true") + assertThrows(classOf[ConfigException], () => config.dynamicConfig.validate(oneFlag, perBrokerConfig = true), + s"$flag must reject a per-broker override") + if (!dynamicBridgeFlags.contains(flag)) { + assertThrows(classOf[ConfigException], () => config.dynamicConfig.validate(oneFlag, perBrokerConfig = false), + s"$flag must require a restart") + } + } } @Test diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 0bb12ddf6cbb9..c384bb6c9bb7e 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -229,6 +229,90 @@ class KafkaApisTest extends Logging { clientMetricsManager = clientMetricsManagerOpt) } + @ParameterizedTest + @CsvSource(Array("false,false,true", "false,true,true", "true,false,true", "true,true,true", + "false,true,false", "true,true,false")) + def testRecommendedElectionRequestRequiresCompatibilityGate(enabled: Boolean, authorized: Boolean, + recommended: Boolean): Unit = { + val authorizer = mock(classOf[Authorizer]) + authorizeResource(authorizer, AclOperation.ALTER, ResourceType.CLUSTER, Resource.CLUSTER_NAME, + if (authorized) AuthorizationResult.ALLOWED else AuthorizationResult.DENIED) + val tp = new TopicPartition("recommended", 0) + val electionType = if (recommended) ElectionType.RECOMMENDED else ElectionType.PREFERRED + val leaders = if (recommended) Map(tp -> 1) else Map.empty[TopicPartition, Int] + val builder = if (recommended) new ElectLeadersRequest.Builder(10L, + Collections.singletonMap(tp, Integer.valueOf(1)), 30000) + else new ElectLeadersRequest.Builder(electionType, Collections.singleton(tp), 30000) + val request = buildRequest(builder.build(2.toShort)) + doAnswer { invocation => + invocation.getArgument[Map[TopicPartition, ApiError] => Unit](4)(Map(tp -> ApiError.NONE)) + null + }.when(replicaManager).electLeaders(ArgumentMatchers.eq(controller), ArgumentMatchers.eq(Set(tp)), + ArgumentMatchers.eq(leaders), ArgumentMatchers.eq(electionType), any(), ArgumentMatchers.eq(30000)) + kafkaApis = createKafkaApis(authorizer = Some(authorizer), overrideProperties = Map( + KafkaConfig.LiProtocolBridgeRecommendedElectionEnableProp -> enabled.toString)) + kafkaApis.handleElectLeaders(request) + val response = verifyNoThrottling[ElectLeadersResponse](request) + val expectedError = if (recommended && !enabled) Errors.INVALID_REQUEST + else if (!authorized) Errors.CLUSTER_AUTHORIZATION_FAILED else Errors.NONE + assertEquals(expectedError.code, response.data.errorCode) + verify(replicaManager, times(if ((!recommended || enabled) && authorized) 1 else 0)) + .electLeaders(any(), any(), any(), any(), any(), anyInt()) + } + + @ParameterizedTest + @CsvSource(Array("false,false", "false,true", "true,false", "true,true")) + def testMetadataExclusionRequiresRequestAndCompatibilityGate(enabled: Boolean, requested: Boolean): Unit = { + val topic = "metadata-exclusion" + addTopicToMetadataCache(topic, numPartitions = 2, numBrokers = 1) + val data = new MetadataRequestData() + .setTopics(Collections.singletonList(new MetadataRequestData.MetadataRequestTopic().setName(topic))) + .setAllowAutoTopicCreation(false) + .setExcludePartitions(requested) + val request = buildRequest(new MetadataRequest(data, 12.toShort)) + kafkaApis = createKafkaApis(overrideProperties = Map( + KafkaConfig.LiProtocolBridgeExcludePartitionsEnableProp -> enabled.toString)) + kafkaApis.handleTopicMetadataRequest(request) + val response = verifyNoThrottling[MetadataResponse](request) + assertEquals(1, response.topicMetadata.size) + val metadata = response.topicMetadata.iterator.next() + assertEquals(Errors.NONE, metadata.error) + assertEquals(topic, metadata.topic) + assertEquals(if (enabled && requested) 0 else 2, metadata.partitionMetadata.size) + } + + @ParameterizedTest + @CsvSource(Array("false,-104,7,false", "true,-104,6,false", "true,-104,7,false", + "true,-104,7,true", "false,-4,8,true", "true,-4,8,true")) + def testFollowerRecoveryRequestAndErrorGates(enabled: Boolean, timestamp: Long, + version: Short, moved: Boolean): Unit = { + val tp = new TopicPartition("recovery-gate", 0) + val fetch = when(replicaManager.fetchOffsetForTimestamp(ArgumentMatchers.eq(tp), ArgumentMatchers.eq(timestamp), + any[Option[IsolationLevel]](), any[Optional[Integer]](), anyBoolean())) + if (moved) fetch.thenThrow(Errors.OFFSET_MOVED_TO_TIERED_STORAGE.exception) + else fetch.thenReturn(Some(new TimestampAndOffset(0L, 12L, Optional.of[Integer](1)))) + val topic = new ListOffsetsTopic().setName(tp.topic).setPartitions(Collections.singletonList( + new ListOffsetsPartition().setPartitionIndex(tp.partition).setTimestamp(timestamp))) + val request = buildRequest(ListOffsetsRequest.Builder.forConsumer(true, IsolationLevel.READ_UNCOMMITTED) + .setTargetTimes(Collections.singletonList(topic)).build(version)) + kafkaApis = createKafkaApis(overrideProperties = Map( + KafkaConfig.LiProtocolBridgeFollowerRecoveryEnableProp -> enabled.toString)) + kafkaApis.handleListOffsetRequest(request) + val response = verifyNoThrottling[ListOffsetsResponse](request) + assertEquals(1, response.topics.size) + val partition = response.topics.get(0).partitions.get(0) + val legacy = timestamp == ListOffsetsRequest.LI_EARLIEST_LOCAL_TIMESTAMP + val admitted = !legacy || (enabled && version >= 7) + val expectedError = if (!admitted) Errors.UNSUPPORTED_VERSION.code + else if (moved && legacy) 1107.toShort + else if (moved) Errors.OFFSET_MOVED_TO_TIERED_STORAGE.code + else Errors.NONE.code + assertEquals(expectedError, partition.errorCode) + assertEquals(if (admitted && !moved) 12L else ListOffsetsResponse.UNKNOWN_OFFSET, partition.offset) + verify(replicaManager, times(if (admitted) 1 else 0)).fetchOffsetForTimestamp( + ArgumentMatchers.eq(tp), ArgumentMatchers.eq(timestamp), any(), any(), anyBoolean()) + } + @Test def testDescribeConfigsWithAuthorizer(): Unit = { val authorizer: Authorizer = mock(classOf[Authorizer]) diff --git a/core/src/test/scala/unit/kafka/server/LiProtocolBridgeConfigTest.scala b/core/src/test/scala/unit/kafka/server/LiProtocolBridgeConfigTest.scala index 89ccaa106b62d..b92f66afb50b4 100644 --- a/core/src/test/scala/unit/kafka/server/LiProtocolBridgeConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/LiProtocolBridgeConfigTest.scala @@ -50,6 +50,16 @@ class LiProtocolBridgeConfigTest { assertEquals(true, enabled.extractLogConfigMap.get(LogConfig.LI_LOG_TRUNCATION_METRICS_CONFIG)) } + @Test + def testNonzeroMinimumLogRollRequiresCompatibilityGate(): Unit = { + val props = new Properties + props.put(ZkConfigs.ZK_CONNECT_CONFIG, "localhost:2181") + props.put(KafkaConfig.LiMinLogRollTimeMillisProp, "60000") + assertEquals(0L, KafkaConfig.fromProps(props).extractLogConfigMap.get(LogConfig.LI_MIN_SEGMENT_MS_CONFIG)) + props.put(KafkaConfig.LiProtocolBridgeMinimumLogRollEnableProp, "true") + assertEquals(60000L, KafkaConfig.fromProps(props).extractLogConfigMap.get(LogConfig.LI_MIN_SEGMENT_MS_CONFIG)) + } + @Test def testBridgeModeRejectsMetadataThatLeaderAndIsrV2CannotRepresent(): Unit = { val props = new Properties diff --git a/tests/bin/li_bridge_test_selection.ini b/tests/bin/li_bridge_test_selection.ini index f1eac1ba17010..b422249cb4222 100644 --- a/tests/bin/li_bridge_test_selection.ini +++ b/tests/bin/li_bridge_test_selection.ini @@ -25,6 +25,8 @@ org.apache.kafka.clients.consumer.internals.NoOpConsumerRebalanceListenerTest org.apache.kafka.common.utils.PoisonPillProcessTest [core] +kafka.api.ElectLeadersRequestOpsTest +kafka.controller.PartitionLeaderElectionAlgorithmsTest kafka.controller.ControllerChannelManagerTest kafka.controller.ControllerContextTest kafka.controller.KafkaControllerTest @@ -55,6 +57,7 @@ kafka.server.KafkaServerTest.testRequestChannelWatchdogIntervalTracksConfiguredT kafka.server.DynamicBrokerConfigTest.testProtocolBridgeFlagDefaultsAndDynamicScope kafka.server.LiControllerOperationsTest kafka.server.LiDynamicTopicDeletionTest +kafka.server.LiReassignmentCancellationGateTest kafka.server.KafkaApisTest kafka.server.LegacyBrokerConstructorTest kafka.server.QuotaFactoryTest From ed3a8690666eeb2b7f32ad9531ac844e12a0c79c Mon Sep 17 00:00:00 2001 From: Jon Bringhurst Date: Fri, 11 Sep 2026 18:13:17 -0700 Subject: [PATCH 2/2] docs: tie gate claims to assertions and record complete runtime qualification --- docs/ops/li-bridge-gate-audit.md | 39 ++++++++++++++++---------- docs/ops/li-bridge-review-comments.md | 4 ++- docs/ops/li-bridge-review.md | 40 +++++++++++++++++---------- docs/ops/li-bridge-upgrade.md | 9 +++--- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/docs/ops/li-bridge-gate-audit.md b/docs/ops/li-bridge-gate-audit.md index b384d7b4b6abf..5bdc42df93bfe 100644 --- a/docs/ops/li-bridge-gate-audit.md +++ b/docs/ops/li-bridge-gate-audit.md @@ -19,9 +19,10 @@ limitations under the License. Scope: upgrade-stack runtime changes relative to `3.9-li`, plus the paired 3.0 bridge changes. This records code paths, not a claim that all release tests passed. -The retained full-verifier pass uses 6ea4d367d2 and predates F25–F27. PRs 596/597 -add interrupted-deletion recovery and mandatory scenario-6 evidence. Scoped tests -pass; the final pair and the separate old-client metadata failure remain open. +The complete scenario-6 verifier passes on runtime source f919812ba4, 3.0 archive +8086d17968 and wrapper 1764cc95, followed by a strict clean/full/archive audit. +PR 599 adds boundary assertions against unchanged runtime code. The separate F28 +client-bootstrap limitation and deployed client-floor decision remain open. All suffixes below use `li.protocol.bridge..enable`. The 3.9 Active getters require empty process.roles as well as the Boolean flag. Every Boolean defaults @@ -33,9 +34,9 @@ listed separately from actions taken by the broker. | mode | ControllerChannelManager snapshots the flag once per batch and chooses v2/v5/v1; false uses the original metadata-version branches. RemoteLeaderEndPoint uses -104 only with follower recovery too. AlterPartition retries copy mutable data only while the flag is active. | ControllerChannelManagerTest; BridgeAlterPartitionRetryTest; protocol fixtures; retained process-selection logs | | config.metrics | LiProtocolBridgeMetrics registers/removes gauges only when opted in. PR 594 extends the same opt-in to the three new KafkaController diagnostic gauges; native gauges remain. | LiProtocolBridgeMetricsTest; KafkaControllerTest.testCompatibilityControllerMetricsRequireOptIn; shutdown ownership tests | | topic.deletion.state.cleanup | New-controller complete metadata images, cache replacement, unhosted-log reconciliation and topic-ID recovery occur only under cleanup. KRaft requests are excluded. 3.0 acknowledgement fencing persists when mode is off; direct native v4 responses echo deletion intent. Marked deletions with missing leader state clear reassignment flags but retain all replicas before ordinary acknowledged deletion. | BridgeMetadataCacheEpochTest; BridgeTopicIdentityTest; BridgeStrayLogDeletionTest; TopicDeletionManagerTest; BridgeInterruptedDeletionTest; KafkaControllerTest; KafkaApisTest native response round trip; scenario revision 6 | -| follower.recovery | KafkaApis admits the private -104 query and emits 1107 only through this flag. Generic timestamp helpers alone do not admit a wire request. | KafkaApisTest; BridgeProtocolConstantsTest; both recovery directions | -| recommended.leader.election | KafkaApis rejects election type 2 when disabled. KRaft ControllerApis always rejects it. The election helper restricts the target to live ISR members. | LiControllerOperationsTest; controller/partition tests | -| metadata.exclude.partitions | KafkaApis requires both the request field and the feature flag before suppressing partition metadata. | KafkaApisTest; request/response wire fixtures | +| follower.recovery | KafkaApis admits the private -104 query and emits 1107 only through this flag. Generic timestamp helpers alone do not admit a wire request. | KafkaApisTest.testFollowerRecoveryRequestAndErrorGates checks disabled admission, v7 minimum, backend calls and native/legacy errors; both recovery directions | +| recommended.leader.election | KafkaApis rejects election type 2 when disabled. KRaft ControllerApis always rejects it. The election helper restricts the target to live ISR members. | KafkaApisTest.testRecommendedElectionRequestRequiresCompatibilityGate (off/on, authorization, native preferred election); ElectLeadersRequestOpsTest; PartitionLeaderElectionAlgorithmsTest | +| metadata.exclude.partitions | KafkaApis requires both the request field and the feature flag before suppressing partition metadata. | KafkaApisTest.testMetadataExclusionRequiresRequestAndCompatibilityGate checks all four combinations and serialized responses; wire fixtures | | move.controller | ApiVersionManager filters advertisement/admission; KafkaApis requires CLUSTER_ACTION and the feature before deleting the controller znode. | ApiVersionManagerTest; LiControllerOperationsTest; unchanged private-API clients | | shutdown.safety.override | Advertisement and handler admission are gated; override grant is broker-epoch fenced. Previously admitted work has its documented lifecycle. | LiShutdownSafetyTest; LiControllerOperationsTest | | preferred.controller | KafkaServer registers/watches preferred IDs and KafkaController changes election/fallback/shutdown behavior only under the flag. ZkAdminManager's broker API filters through it. The raw AdminZkClient fix makes optional config explicitly opt in too. | AdminZkClientTest (none/false/true plus manual assignment); controller and shutdown tests | @@ -45,13 +46,13 @@ listed separately from actions taken by the broker. | produce.request.instrumentation | New per-request collector only while enabled; Disabled does not collect stages. PR 594 makes it ignore partition setters and prevents later activation from logging an uncollected request. Logger also checks the dynamic flag. | ProduceRequestInstrumentationTest; acks=0/callback source checks | | request.metric.buckets | RequestChannel creates size/time buckets only from the gated optional config. Empty maps produce no additional request groups. Empty or malformed configured boundary lists are rejected, not supported as a disabling syntax. | RequestMetricBucketsTest; KafkaConfigTest boundary cases | | request.channel.watchdog | Data-plane histogram, health scheduler and PoisonPill construction/actions are gated. Old constructors default the watchdog off. | RequestChannelWatchdogTest; KafkaServerTest interval case; PoisonPillProcessTest | -| minimum.log.roll | KafkaConfig passes zero when disabled. Storage's explicit li.min.log.roll.ms also defaults zero. Old RollParams constructor supplies zero. Size/index/relative-offset rolling checks remain separate. | LogSegmentTest in full storage suite; configuration tests | -| reassignment.cancellation.safety | Only cancellation with the gate invokes the minimum-live-original-replica check. Ordinary reassignment and flag-off cancellation keep native behavior. | KafkaControllerTest; reassignment cancellation process case | +| minimum.log.roll | KafkaConfig passes zero when disabled. Storage's explicit li.min.log.roll.ms also defaults zero. Old RollParams constructor supplies zero. Size/index/relative-offset rolling checks remain separate. | LiProtocolBridgeConfigTest.testNonzeroMinimumLogRollRequiresCompatibilityGate; LogSegmentTest in the full storage suite | +| reassignment.cancellation.safety | Only cancellation with the gate invokes the minimum-live-original-replica check. Ordinary reassignment and flag-off cancellation keep native behavior. | LiReassignmentCancellationGateTest checks actual controller acceptance/rejection and persisted target with an offline original replica; KafkaControllerTest threshold cases; process cancellation | | list.offsets.instrumentation | Data-plane and flag conjunction reaches a collector with disabled registration/usage early returns otherwise. Snapshot/reset is synchronized. | ListOffsetsRequestInstrumentationTest | | static.default.quotas | QuotaFactory passes Long.MaxValue when disabled. Explicit dynamic/callback limits retain native arithmetic; static fallback is used only for absent limits. | QuotaFactoryTest; ClientQuotaManagerTest including PR 590 regressions; RequestQuotaTest | | replica.request.timeout | effectiveReplicaRequestTimeoutMs selects requestTimeoutMs when disabled. BrokerBlockingSender uses that accessor. | ReplicaRequestTimeoutConfigTest (added to focused verifier selection) | | offsets.topic.config | AutoTopicCreationManager copies the original properties; overrides only under the gate. | AutoTopicCreationManagerTest (added to focused selection) | -| leader.transfer.on.isr.shrink | Partition suppresses shrinking below minimum ISR and submits a live ISR target only while enabled. KafkaServer defaults to NoOp manager and ReplicaManager schedules transfers only under the gate. | PartitionTest; LeaderTransferManagerTest; legacy constructor tests | +| leader.transfer.on.isr.shrink | Partition suppresses shrinking below minimum ISR and submits a live ISR target only while enabled. KafkaServer defaults to NoOp manager and ReplicaManager schedules transfers only under the gate. | PartitionTest checks transfer submission and native ISR shrink with the flag off/on; LeaderTransferManagerTest; legacy constructor tests | | legacy.request.metrics | Constructors receive false by default; additional broker/replica/request counters, metadata egress and topic-name diagnostics are conditional. | LegacyRequestMetricsTest; MetadataOutgoingBytesTest; BrokerTopicMetricsTest; LogDirFailureChannelTest | | log.truncation.metrics | KafkaConfig passes false when disabled; explicit internal log setting defaults false; meters are referenced/updated only when enabled. | UnifiedLogTest.testTruncateTo; full storage suite | @@ -98,10 +99,18 @@ listed separately from actions taken by the broker. - The old reply claiming empty buckets were supported was incorrect. Source and tests reject them; an explicit correction was posted and the ledger updated. -## Audit boundary +## Assertion audit and boundary -This matrix must be checked against final published source and actual test results. -A passing process run covers its configuration and actions, not every disabled-path -claim. The newly added default-off fixes are not covered by the still-running full -verifier on the prior source. Final-source qualification, publication/readback and -remaining requirement checks are still needed before goal completion. +The source audit confirms 24 false defaults and 24 ZooKeeper-only Active getters. +Runtime callers use those getters, not the raw Enable accessors. The strengthened +dynamic-scope test checks every flag individually: ten permit cluster-wide updates, +and fourteen require restart; none permits a per-broker dynamic override. + +PR 599 closes six weak test mappings above. Three mutation runs deliberately broke +seven guard boundaries; every break failed the intended assertion. The runtime files +were restored and checked against the published base. The restored suites pass 441 +tests on Scala 2.12 and 39 scoped tests on Scala 2.13, without failures/errors/skips. + +A passing process run still covers only its configuration and actions. It does not +qualify the deployed client/tool floor, actual production runtime, capacity or +security approvals. F28 and the final prompt-to-artifact audit remain open. diff --git a/docs/ops/li-bridge-review-comments.md b/docs/ops/li-bridge-review-comments.md index ac78681450335..5474ca97a85f9 100644 --- a/docs/ops/li-bridge-review-comments.md +++ b/docs/ops/li-bridge-review-comments.md @@ -17,7 +17,7 @@ limitations under the License. # Review comment dispositions -All 62 original threads have replies with published source decisions. A fresh GraphQL readback across 48 PRs verified every expected reply, found no mismatches and found no new review threads. Resolved status alone was not accepted as proof. Re-fetch after the final publication and check the actual source/test coverage before closing the review. +All 62 original threads have replies with published source decisions. A fresh GraphQL readback across 49 PRs verified every expected reply, found no mismatches and found no new review threads. Resolved status alone was not accepted as proof. Re-fetch after the final publication and check the actual source/test coverage before closing the review. ## Later qualification findings @@ -35,6 +35,8 @@ All 62 original threads have replies with published source decisions. A fresh Gr The verifier isolation finding is tracked in [598](https://github.com/linkedin/kafka/pull/598): all Gradle invocations remain single-use, and the global daemon-stop commands are removed. Its command-plan regression fails before and passes after. No functional qualification check is removed. +[599](https://github.com/linkedin/kafka/pull/599) strengthens the gate evidence rather than relying on helper test names: request admission, native behavior, authorization, cancellation, minimum roll and ISR transfer/shrink now have direct flag-boundary assertions. Deliberately broken guard boundaries failed those assertions; restored runtime code passes the new suites. No runtime or client-profile change is included. + ## PR 541 | Comment | Decision | Code/test evidence | Reply | diff --git a/docs/ops/li-bridge-review.md b/docs/ops/li-bridge-review.md index bb0864ab5903c..dd7bfcc3e63e7 100644 --- a/docs/ops/li-bridge-review.md +++ b/docs/ops/li-bridge-review.md @@ -19,14 +19,15 @@ limitations under the License. ## Current verdict -**Do not deploy this candidate without final-source qualification and the release approvals.** The retained scenario-5 full-verifier pass predates F25–F27. The interrupted-deletion repairs and mandatory scenario-6 checks are now published, but only their scoped before/after process tests have passed. A separate old-client metadata timeout remains open. The inventory covers 48 open PRs. Earlier passing or failed bundles retain their original source and coverage limits. +**Do not deploy without approved published-artifact qualification and the release approvals.** A complete scenario-6 full-verifier run now passes on f919812ba4 / 8086d17968 / wrapper 1764cc95, followed by an independent strict clean/full/archive audit. The test-only gate follow-up adds separately verified assertions against unchanged runtime code. F28's existing client-bootstrap limitation and deployed client-floor decision remain open. The inventory covers 49 open PRs. Earlier passing or failed bundles retain their original source and coverage limits. The findings below record what the initial review and later tests found. Instructions in an original finding describe the repair that was needed; use the current disposition and evidence sections for status. This is not a line-by-line approval of every Kafka change. ### Reviewed revisions - Workspace plan: `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`. Canonical runbook: `docs/ops/li-bridge-upgrade.md`. -- Current 3.9 implementation and qualification: PR 598, `3.9-li-bridge/verifier-process-isolation`, including F25, F27 and mandatory scenario-6 checks. +- Current 3.9 implementation and qualification: PR 599, `3.9-li-bridge/request-gate-regressions`, including F25, F27 and mandatory scenario-6 checks. +- Complete local qualification: runtime source `f919812ba4255f07412aefcfb8d59246174dc266`. Later changes are documentation, tests and test selection, not runtime code. - Current 3.0 source: PR 596, `8086d1796816bd2383844b7b9f9b05ebd8b33c9b`. - CI: PR 558, `3e799b08ea`; PR 559, `a86214e2da`. - Wrapper: `1764cc95bfa21e19d3ff89e0164e5896808b5507`, including the diagnostic opt-in and the earlier ACL test fix. @@ -339,6 +340,12 @@ The verifier ran global `gradlew --stop` commands in both checkouts. Those comma PR 598 removes them and their evidence-row requirement. A regression renders the actual full command plan, requires `--no-daemon`, and rejects `--stop`; it fails before and passes after. All 85 Python tests and actual RAT pass. No functional source, wrapper, archive, phase or record check is removed. This is a verifier-only change, not a broker/client behavior change. +### F30 — P2: Gate evidence did not cover every named boundary + +Some gate-matrix entries cited helper or enabled-path tests rather than the actual disabled request boundary. PR 599 adds direct recommended-election, metadata-exclusion and follower-recovery request tests, including native behavior and authorization. It tests cancellation on a real controller with an offline original replica, nonzero minimum-roll configuration while the gate is off, and both transfer submission and native ISR shrink. Each of the 24 flags is now checked alone for dynamic scope, so one rejected property cannot hide another allowed override. + +Three mutation runs deliberately broke seven guard boundaries; each failed the intended assertion. No mutation was published. Runtime files were restored and compared with the base. The restored suites pass 441 cases on Scala 2.12 and 39 scoped cases on Scala 2.13, with no failures/errors/skips; 85 Python tests and actual RAT pass. The verifier also selects the existing recommended-election decoder/eligibility tests and new cancellation test. Runtime code, client settings and deadlines are unchanged. + ## PR dispositions and dependency audit Every PR below has a distinct migration or CI purpose. Keep these scopes, but do not treat publication, a resolved thread or a green job as release approval. Publication does not mean that a layer is approved. The controller/ZooKeeper, security, storage and operational changes still need the corresponding owners' review. @@ -393,12 +400,13 @@ Every PR below has a distinct migration or CI purpose. Keep these scopes, but do | [594](https://github.com/linkedin/kafka/pull/594) | default-off placement, controller diagnostics and instrumentation audit — operations/verification | | [597](https://github.com/linkedin/kafka/pull/597) | 3.9 interrupted deletion and mandatory scenario-revision-6 checks — controller/verification | | [598](https://github.com/linkedin/kafka/pull/598) | verifier process isolation — verification | +| [599](https://github.com/linkedin/kafka/pull/599) | request, cancellation, storage and flag-scope assertions — verification | Merge CI 558/559 into their own release branches first. Then retarget the upgrade stack bottoms as described in the runbook. Never force a Git dependency between the 3.0 and 3.9 CI branches. When reordering again, change PR bases before pushing a head that becomes an ancestor of its former base; GitHub can otherwise auto-close and delete that branch. -The oversized original scopes were split. Control wire definitions are in 565 (503 lines), handlers in 545 (845); storage metrics are in 566 (221), broker metrics/watchdog wiring in 551 (910); workload helpers precede runner 553 (754). The new evidence auditor, verifier and release gate are separate layers. All 48 PR diffs were below 1,000 changed lines at the latest readback. This documentation follow-up stays separate from the 863-line original runbook PR. +The oversized original scopes were split. Control wire definitions are in 565 (503 lines), handlers in 545 (845); storage metrics are in 566 (221), broker metrics/watchdog wiring in 551 (910); workload helpers precede runner 553 (754). The new evidence auditor, verifier and release gate are separate layers. All 49 PR diffs were below 1,000 changed lines at the latest readback. This documentation follow-up stays separate from the 863-line original runbook PR. -The original 62 review threads now have replies with published source decisions and code/test references. See `docs/ops/li-bridge-review-comments.md`. The static `LeaderTransferManager.noOp()` call is valid: javap confirms the forwarder, and the Java builder compiles. Empty, malformed, negative and unordered metric bucket lists are rejected. The earlier claim that empty lists were supported has been corrected against the actual source and test assertion. The latest readback covers 48 PRs and finds all 62 expected replies, no mismatches and no new review threads. The later F18/F19 issue comments are retained and have follow-up code and qualification records. Re-fetch after the final publication; comment status is not proof that the code is correct. +The original 62 review threads now have replies with published source decisions and code/test references. See `docs/ops/li-bridge-review-comments.md`. The static `LeaderTransferManager.noOp()` call is valid: javap confirms the forwarder, and the Java builder compiles. Empty, malformed, negative and unordered metric bucket lists are rejected. The earlier claim that empty lists were supported has been corrected against the actual source and test assertion. The latest readback covers 49 PRs and finds all 62 expected replies, no mismatches and no new review threads. The later F18/F19 issue comments are retained and have follow-up code and qualification records. Re-fetch after the final publication; comment status is not proof that the code is correct. ## Verification and remaining requirements @@ -408,24 +416,24 @@ This prompt-to-artifact checklist separates observed results from open requireme | Requirement | Artifact and verification surface | Evidence / open work | |---|---|---| -| Review the named plan and every open public PR | `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`; canonical runbook; GitHub inventory | 48 open PRs are listed in both tables. Recheck the exact PR set after any further publication. | -| Explain scope and dependencies | PR responsibility table, heads/bases, stack membership | Separate protocol, handlers, storage metrics, runner, auditor, verifier and release-gate layers. GitHub stack 582 has 37 upgrade PRs; stack 581 has nine. New members were appended and read back. CI 558/559 remain on independent release histories. No release branch was merged. | -| Apply the requested label | GitHub labels | All 46 upgrade PRs have `kafka-upgrade-august-2026`; CI 558/559 do not. | +| Review the named plan and every open public PR | `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`; canonical runbook; GitHub inventory | 49 open PRs are listed in both tables. Recheck the exact PR set after any further publication. | +| Explain scope and dependencies | PR responsibility table, heads/bases, stack membership | Separate protocol, handlers, storage metrics, runner, auditor, verifier and release-gate layers. GitHub stack 582 has 38 upgrade PRs; stack 581 has nine. New members were appended and read back. CI 558/559 remain on independent release histories. No release branch was merged. | +| Apply the requested label | GitHub labels | All 47 upgrade PRs have `kafka-upgrade-august-2026`; CI 558/559 do not. | | Keep PRs below 1,000 changed lines, preferably near 500 | Additions plus deletions, not file length | Established diffs passed the limit; recheck the new metrics follow-ups after publication. The largest established diffs are 981 and 963 lines. | | Use plain, direct English | Plan, review, comment replies and workflow comments | Final wording/link review remains required. Historical findings are not current deployment instructions. | -| Gate every Kafka behavior change | `KafkaConfig`, `DynamicBrokerConfig`, runtime call sites, metrics, wrapper mapping | 24 default-off 3.9 gates. F18/F19 use the cleanup gate; ISR retry repair uses bridge mode; F22 separately gates diagnostic registration. Dedicated disabled/activation tests pass. The gate-audit matrix records runtime call sites, non-Boolean configuration opt-ins and native behavior; verify it against final published source and final-source test results. | +| Gate every Kafka behavior change | `KafkaConfig`, `DynamicBrokerConfig`, runtime call sites, metrics, wrapper mapping | 24 default-off 3.9 gates. F18/F19 use the cleanup gate; ISR retry repair uses bridge mode; F22 separately gates diagnostic registration. Dedicated disabled/activation tests pass. The source audit confirms 24 false defaults and ZooKeeper-only Active getters; F30 adds direct boundary assertions and mutation evidence. The gate matrix records those assertions plus non-Boolean opt-ins. | | Select symmetric v2/v5/v1 control | Schemas, controller selectors, wire fixtures and retained logs | Both generations have fixtures and real-process coverage. F20 makes rotated log checks fail closed too. Final-source process qualification remains required. | | Fence activation and preserve callbacks | `RequestSendThreadBridgeTest` | Blocked dequeue, sustained queue and admitted-deletion callback tests pass. Controller restart remains mandatory. | | Enforce all six phases and unchanged clients | `li_bridge_contract.py`, preflight, persistent client/Streams/Connect and private-API helpers | 85 Python tests pass. F28's cold-bootstrap qualification remains open. Native control stays at IBP 3.0 before the separate IBP roll. One 3.0 archive is not the deployed client/tool floor. | -| Prove persisted rollback and recovery | Process runner, record helper, timings and JUnit | Historical full run covers canary/all-3.9 rollback, cancellation, crashes and truncation. Repeat against the final source; registration or ISR alone is not proof. | -| Prove deletion and name reuse | `TopicDeletionManager`, `ZkMetadataCache`, `ReplicaManager`, `BridgeTopicIdentity` | Gated F12/F13/F15/F17/F18/F19 repairs have tests. Revision 6 retains all four post-promotion record checks and adds both generations' interrupted-deletion startup cases. Scoped before/after checks pass but do not replace complete qualification. | +| Prove persisted rollback and recovery | Process runner, record helper, timings and JUnit | The complete scenario-6 run covers canary/all-3.9 rollback, cancellation, crashes, truncation and exact promoted records on the current runtime. Registration or ISR alone is not proof. | +| Prove deletion and name reuse | `TopicDeletionManager`, `ZkMetadataCache`, `ReplicaManager`, `BridgeTopicIdentity` | Gated F12/F13/F15/F17/F18/F19 repairs have tests. Revision 6 retains all four post-promotion record checks and adds both generations' interrupted-deletion startup cases. Scoped before/after checks and the complete scenario-6 runtime qualification pass. They do not grant deployment approval. | | Handle version-changing ISR retries | `BridgeAlterPartitionRetryTest` | One queued builder crosses versions 3/1 and activation without reusing mutated request data. Earlier failed logs remain failed evidence. | | Collect real runtime/configuration/state | Live inventory, runtime probe and negative-input tests | Disposable-cluster collection passes. Production binaries, settings, state dispositions, owners and client/tool floor remain required inputs. | | Qualify pagination with the loaded runtime | `KafkaZkClient`, five vendor-client tests and release runtime probe | Startup rejects an unsupported client. Vendor tests pass. The actual deployed client/Jute/server pairing remains a release gate. | | Follow Google shell style, including comments | Four wrappers, all extracted workflow Bash blocks, ShellCheck, shfmt, syntax/length checks | The audit covers 54 changed-workflow blocks, four wrappers and nine Bash documentation examples. The broader example check found three continuation-indentation issues and one overlong line; formatting-only fixes preserve their assignment/argument tokens. All examples now pass ShellCheck, shfmt, syntax, no-tab and 80-column checks. Unmodified upstream Docker workflows are outside these PRs. | | Preserve wrapper/API compatibility | Factory mapping tests, ACL tests, complete wrapper suite and jar comparison | The current 133-test suite passes with matching main jars, stable main/test-classifier hashes and unchanged source. Wrapper commit 1764cc95 is published. The required Mint refresh produced a fresh dependency spec; no TTL or artifact-identity bypass was used. | -| Qualify real archives and reject incomplete evidence | `verify_li_bridge.sh`, `audit_li_bridge_evidence.py`, `verify_li_bridge_release.sh` and negative fixtures | Earlier full verifier passed but predates F18/F19. The metrics-gated full run failed during the process phase after its earlier stages passed. Eight new release-input tests cover missing inputs, malformed identifiers, mismatched/unknown archive metadata, dirty/wrong checkouts, forced full mode and rejection before launch. A real-archive identity-only check also passes; neither it nor the fixtures grant release approval. | -| Address every review comment with evidence | Comment ledger, source/test decisions, GraphQL readback | All original 62 replies verified across 48 PRs; no new review threads. Later issue findings have published fixes and explicit qualification limits. Re-fetch after final publication. | +| Qualify real archives and reject incomplete evidence | `verify_li_bridge.sh`, `audit_li_bridge_evidence.py`, `verify_li_bridge_release.sh` and negative fixtures | The f919812ba4 / 8086d17968 / 1764cc95 full run and independent strict audit pass. Earlier failures remain retained, including the F28 client-bootstrap failure. Eight new release-input tests cover missing inputs, malformed identifiers, mismatched/unknown archive metadata, dirty/wrong checkouts, forced full mode and rejection before launch. A real-archive identity-only check also passes; neither it nor the fixtures grant release approval. | +| Address every review comment with evidence | Comment ledger, source/test decisions, GraphQL readback | All original 62 replies verified across 49 PRs; no new review threads. Later issue findings have published fixes and explicit qualification limits. Re-fetch after final publication. | | Keep the three documents consistent | Workspace files and `docs/ops/li-bridge-{upgrade,review,review-comments}.md` | This follow-up synchronizes the records. Verify relative links and actual PR/source/evidence state before calling the review complete. | | Preserve side-task PR 2039 | ADU worktree, commit history and formatting checks | Rebased/pushed on master at `ae007420`; formatting is one separate commit and is idempotent. Functional patches remain unchanged. | @@ -448,7 +456,7 @@ Later evidence supersedes the inventory and coverage limits of those historical - `/tmp/li-scenario-4-batch-fixed`: all four offline-reuse record checks passed, but the complete run failed at the native checkpoint. The old metadata-churn helper exited on `ControllerMovedException` during controller movement. Its progress had advanced to 221 cycles. The unchanged upstream fence and helper retry gap are covered by F21; do not waive this failed run. The summary records `passed=false` and unchanged source. Rotated logs omitted by that older collector are retained separately in `/tmp/li-scenario-4-batch-fixed-rotated-logs.tgz`. - `/tmp/li-log-rotation-before.log`: both new rotation regressions fail before F20. `/tmp/li-log-rotation-restacked.log`: all 70 Python tests pass afterward. - `/tmp/li-churn-retry-before-{3.0,3.9}.log`: real client error classes expose the missing controller retry and incorrectly retryable storage/corruption errors. `/tmp/li-churn-retry-after-{3.0,3.9}.log` and `/tmp/li-churn-retry-restacked-python.log` pass with the repair. -- `/tmp/li-review-readback-result.json`: 48 PRs, 62 original threads, no missing/mismatched replies and no new threads at that readback. +- `/tmp/li-review-readback-result.json`: 49 PRs, 62 original threads, no missing/mismatched replies and no new threads at that readback. The complete revision-4 process run `/tmp/li-scenario-4-churn-fixed` passed on clean `d8f255f8a4` / `1a5d02403b`, with all four name-reuse checks, unchanged source and an issue-free process audit. It includes F20/F21 but predates F22. The real `mint --no-metrics dependency create-dependency-spec --detect-variant --overwrite` command has now refreshed the wrapper metadata successfully. An invocation without `--overwrite` returned success without refreshing the expired file; that no-op was not accepted as freshness evidence. @@ -464,7 +472,11 @@ The complete revision-4 process run `/tmp/li-scenario-4-churn-fixed` passed on c `/tmp/li-verifier-isolation-regression-before.log` preserves the global-stop regression; `/tmp/li-verifier-isolation-after.log` passes all 85 Python tests. The removed evidence rows represented unnecessary daemon cleanup, not functional coverage. `/tmp/li-document-shell-audit/report.json` retains the failing documentation-example audit; `/tmp/li-document-shell-audit-fixed/report.json` passes all nine examples after formatting-only fixes. -No complete bundle yet covers all final default-off, response-fencing and interrupted-deletion repairs together. Failed runs remain failed. A source, archive or scenario change must be checked against the fingerprint before reuse. +`/tmp/li-final-isolated-default-profile-full` now covers the default-off, response-fencing and interrupted-deletion repairs together. Both Scala compiles, focused suites, vendor pagination, full clients/server/storage suites, matching artifacts, all 133 wrapper tests, sixteen unchanged-client checkpoints and final audit pass. `/tmp/li-final-isolated-strict-audit.log` independently checks clean source, full suites and retained archives with no issues/warnings. Archive source IDs match f919812ba4 and 8086d17968. The 58 original focused selectors match actual passed JUnit cases in `/tmp/li-final-f919-junit-audit`; F30's additional cases have separate retained JUnit reports and mutation logs. + +The 15 previously cancelled intermediate static-check jobs were rerun successfully. Their actual checkout trees match their PR heads, recorded in `/tmp/li-cancelled-static-checks.json`; the cancelled unit/integration jobs are not relabelled as passes. PRs 597 and 598 also pass public process CI. These successful runs do not disprove F28's deterministic client-liveness failure or approve the separate config-zero profile. + +Failed runs remain failed. A source, archive or scenario change must be checked against the fingerprint before reuse. The post-f919 documentation/test-only delta is checked separately; it is not a claim that commit identities are identical. ### Inputs still required before production diff --git a/docs/ops/li-bridge-upgrade.md b/docs/ops/li-bridge-upgrade.md index 55d721742bea4..321cd2f5c70ee 100644 --- a/docs/ops/li-bridge-upgrade.md +++ b/docs/ops/li-bridge-upgrade.md @@ -23,9 +23,9 @@ limitations under the License. The implementation base is Apache **3.9.2** with the reviewed LI bridge stack. Pin the final internal `3.9.2.N`, matching `3.0.1.N`, wrapper commit, archive checksums, JDKs and ZooKeeper runtime in the release record. A maintenance-baseline change requires a new qualification run; do not substitute a newer tag during rollout. -The current implementation is the split stack through `3.9-li-bridge/verifier-process-isolation`, with the companion `3.0-li-bridge/interrupted-deletion-recovery` branch. It is not the closed aggregate PR 542. The canonical mergeable runbook is `docs/ops/li-bridge-upgrade.md`; the workspace copy is `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`. The `3.9-li-bridge/review-refresh` branch updates the documentation after the behavior fixes. Historical experiments are evidence, not current acceptance criteria. +The current implementation is the split stack through `3.9-li-bridge/request-gate-regressions`, with the companion `3.0-li-bridge/interrupted-deletion-recovery` branch. It is not the closed aggregate PR 542. The canonical mergeable runbook is `docs/ops/li-bridge-upgrade.md`; the workspace copy is `LI-3.0-TO-3.9-ROLLING-UPGRADE-PLAN.md`. The `3.9-li-bridge/review-refresh` branch updates the documentation after the behavior fixes. Historical experiments are evidence, not current acceptance criteria. -**Clients do not change.** The supported producer, consumer, transactional client, Streams application, Connect worker, LI AdminClient and operational-tool artifacts/configuration must remain unchanged across every phase. Discovery must name their deployed version floor and owners. One 3.0 test archive is not proof for every externally deployed client. The latest qualification still has an unresolved cold-producer metadata timeout with an offline bootstrap broker. Record the deployed LI clients' `li.client.cluster.metadata.expire.time.ms` setting and test cold startup with unavailable bootstrap entries. Diagnostic replays and an existing config opt-out are not release approval or permission to change clients during the roll. +**Clients do not change.** The supported producer, consumer, transactional client, Streams application, Connect worker, LI AdminClient and operational-tool artifacts/configuration must remain unchanged across every phase. Discovery must name their deployed version floor and owners. One 3.0 test archive is not proof for every externally deployed client. A retained qualification run exposed a cold-producer metadata timeout with an offline bootstrap broker. The later full run passes, but does not remove that deterministic client limitation. Record the deployed LI clients' `li.client.cluster.metadata.expire.time.ms` setting and test cold startup with unavailable bootstrap entries. Diagnostic replays and an existing config opt-out are not release approval or permission to change clients during the roll. ## Why we need two bridge-capable binaries @@ -240,7 +240,7 @@ Automatically stop for unexpected control versions, post-fence API 1001 traffic, ## PR inventory and merge order -All 48 open public PRs are covered below. Upgrade PRs carry `kafka-upgrade-august-2026`; CI foundations 558 and 559 do not. All current diffs are below 1,000 changed lines. These checks do not grant approval to deploy. +All 49 open public PRs are covered below. Upgrade PRs carry `kafka-upgrade-august-2026`; CI foundations 558 and 559 do not. All current diffs are below 1,000 changed lines. These checks do not grant approval to deploy. Closed PRs 542 and 555 are superseded. GitHub automatically closed 563 and 564 during the dependency reorder because their new heads were contained in their former base branches. No release branch was merged. Their restored, separate reviews are 579 and 578. @@ -294,12 +294,13 @@ Closed PRs 542 and 555 are superseded. GitHub automatically closed 563 and 564 d | [594](https://github.com/linkedin/kafka/pull/594) | default-off placement, controller diagnostics and instrumentation audit — operations/verification | | [597](https://github.com/linkedin/kafka/pull/597) | 3.9 interrupted deletion and mandatory scenario-revision-6 checks — controller/verification | | [598](https://github.com/linkedin/kafka/pull/598) | verifier process isolation — verification | +| [599](https://github.com/linkedin/kafka/pull/599) | request, cancellation, storage and flag-scope assertions — verification | Merge 558 into `3.9-li` and 559 into `3.0-li` first. They have different release bases, so do not put them in one dependent Git stack. Rebase/retarget 575 to `3.0-li` and 543 to `3.9-li`; do not merge feature work into temporary CI branches. The GitHub stack rooted at PR 575 has the 3.0 order **575 → 541 → 577 → 583 → 585 → 588 → 592 → 595 → 596**. The stack rooted at PR 543 has the 3.9 order: -**543 → 544 → 565 → 545 → 546 → 547 → 548 → 560 → 549 → 550 → 561 → 566 → 551 → 567 → 576 → 568 → 578 → 579 → 552 → 569 → 570 → 571 → 553 → 572 → 554 → 573 → 574 → 584 → 586 → 587 → 589 → 590 → 591 → 593 → 594 → 597 → 598**. +**543 → 544 → 565 → 545 → 546 → 547 → 548 → 560 → 549 → 550 → 561 → 566 → 551 → 567 → 576 → 568 → 578 → 579 → 552 → 569 → 570 → 571 → 553 → 572 → 554 → 573 → 574 → 584 → 586 → 587 → 589 → 590 → 591 → 593 → 594 → 597 → 598 → 599**. Retarget remaining layers after each independent merge. Wrapper `1764cc95` contains the diagnostic opt-in, ACL test repair (`6ddf2a87`) and cleanup mapping/tests (`1a9ecccf`); its source suite passes 133 tests. Add the approved wrapper/dependency/security PR and named deployment-gate owner to the release record.