Add Cassandra 6.0 support

.circleci/config.yml

@@ -173,6 +173,9 @@ jobs:
             JDK_VERSION: "11"
             SPARK_VERSION: "3"
             SCALA_VERSION: "2.12"
+            # No downstream job of this one tests Cassandra 6.0, so skip its dtest jar build
+            # (build-dtest-jars.sh reads this var to filter CANDIDATE_BRANCHES).
+            BRANCHES: "cassandra-4.0 cassandra-4.1 cassandra-5.0"
           command: |
             ./scripts/build-dependencies.sh
             ./gradlew --no-daemon --max-workers=2 codeCheckTasks
@@ -201,7 +204,8 @@ jobs:
             SCALA_VERSION: "2.13"
             # JDK17 only targets Cassandra 5.0+; skip 4.0 / 4.1 dtest jar builds
             # (build-dtest-jars.sh reads this var to filter CANDIDATE_BRANCHES).
-            BRANCHES: "cassandra-5.0"
+            # Only the Spark 4 matrix tests Cassandra 6.0, so its dtest jar is built here and nowhere else.
+            BRANCHES: "cassandra-5.0 cassandra-6.0"
           command: |
             ./scripts/build-dependencies.sh
             ./gradlew --no-daemon --max-workers=2 codeCheckTasks
@@ -351,6 +355,20 @@ workflows:
               sstable_format: ["bti"]
               cassandra: ["5.0"]
 
+      # Cassandra 6.0 on Spark 4 / Scala 2.13 / JDK 17 + BTI sstable format.
+      # Cassandra 6.0 needs JDK17 or later, so it has no JDK11 job.
+      - unit:
+          requires:
+            - build-deps-jdk17
+          matrix:
+            alias: unit-c6-spark4
+            parameters:
+              spark: ["4"]
+              scala: ["2.13"]
+              jdk: ["17"]
+              sstable_format: ["bti"]
+              cassandra: ["6.0"]
+
       # ---- Integration tests ----
       # Cassandra 4.x: Scala 2.12, full patch versions for dtest jar lookup
       - int-test:
@@ -387,3 +405,16 @@ workflows:
               scala: ["2.13"]
               jdk: ["17"]
               cassandra: ["5.0.7"]
+
+      # Cassandra 6.0 on Spark 4 / Scala 2.13 / JDK 17.
+      # The value is the dtest jar version, which is the Cassandra base.version at the pinned tag.
+      - int-test:
+          requires:
+            - build-deps-jdk17
+          matrix:
+            alias: int-c6-spark4
+            parameters:
+              spark: ["4"]
+              scala: ["2.13"]
+              jdk: ["17"]
+              cassandra: ["6.0-alpha2"]

CHANGES.txt

@@ -1,5 +1,6 @@
 0.5.0
 -----
+ * Add Cassandra 6.0 support (CASSANALYTICS-37)
  * CDC reader stats silently dropped in SidecarCdcBuilder (CASSANALYTICS-191)
  * Add CapturePublishedSchema metric to SidecarCdcStats (CASSANALYTICS-189)
  * Expand list of architecture that supports unaligned access in FastByteOperations (CASSANALYTICS-188)

DEV-README.md

@@ -90,6 +90,36 @@ create a temporary alias for every node except the first:
  for i in {2..20}; do sudo ifconfig lo0 alias "127.0.0.${i}"; done
 ```
 
+### Host names
+
+A multi-node test passes `localhost2` through `localhost20` to the bulk reader as sidecar contact points, so each
+name must resolve.  `LocalhostResolver` maps the name inside Sidecar only; the Spark side uses the system resolver.
+Add an entry for every node except the first:
+
+```shell
+sudo bash -c 'for i in {2..20}; do echo "127.0.0.${i}  localhost${i}"; done >> /etc/hosts'
+```
+
+Without these entries an upstream DNS server can answer `localhost2` with a public address.  The bulk reader then
+connects to that address and the test fails with `java.net.ConnectException: Operation timed out` in
+`CassandraDataLayer.initialize`.
+
+### Topology-change tests skip on Cassandra 6.0
+
+Twenty-eight test classes under `expansion`, `shrink`, `replacement` and `movement` pause a topology change with a
+ByteBuddy hook, then run the bulk writer while the node is in the transitional state.  The hooks target
+`StorageService.bootstrap(Collection, long)`, `StorageService.unbootstrap()` and `RangeRelocator.stream()`.
+
+CEP-21 Transactional Cluster Metadata removed all three in Cassandra 6.0.  The work now belongs to
+`org.apache.cassandra.tcm.sequences`: `BootstrapAndJoin.bootstrap(...)` for a join, `BootstrapAndReplace` for a
+replacement, `UnbootstrapAndLeave.executeNext()` with `LeaveStreams` for a decommission, and `Move` for a token move.
+A hook that fails to install is silent, so each class waited two minutes for a latch that never counted down.
+
+`ResiliencyTestBase.assumeTopologyChangeHooksSupported()` now skips these classes on 6.0 and later.  The four base
+classes call it from `beforeClusterProvisioning()`.  Bulk write during a topology change is therefore untested on
+6.0.  To close the gap, retarget each hook at the sequence types named above, and keep the 4.0 and 5.0 targets for
+the older runs.
+
 ## IntelliJ
 
 The project is well-supported in IntelliJ.
@@ -106,4 +136,19 @@ Major Cassandra versions require dedicated bridge modules (e.g. `cassandra-five-
 Cassandra as a library, and correctly read sstable files. Significant part of code may be based on previous module
 version. To minimise code duplication, Gradle build copies all source files from previous dependent module.
 If implementation of given component has to be updated, copy the file with the same name to a target module and
-modify its body.
+modify its body.
+
+Each major version owns five modules.  The table shows the current set:
+
+| Cassandra | Shaded library      | Bridge                     | Types                     | Avro converter                      | Spark SQL converter                          |
+|-----------|---------------------|----------------------------|---------------------------|-------------------------------------|----------------------------------------------|
+| 4.0, 4.1  | `cassandra-four-zero` | `cassandra-four-zero-bridge` | `cassandra-four-zero-types` | `cassandra-four-zero-avro-converter` | `cassandra-analytics-spark-four-zero-converter` |
+| 5.0       | `cassandra-five-zero` | `cassandra-five-zero-bridge` | `cassandra-five-zero-types` | `cassandra-five-zero-avro-converter` | `cassandra-analytics-spark-five-zero-converter` |
+| 6.0       | `cassandra-six-zero`  | `cassandra-six-zero-bridge`  | `cassandra-six-zero-types`  | `cassandra-six-zero-avro-converter`  | `cassandra-analytics-spark-six-zero-converter`  |
+
+The copy-forward chain runs four-zero, then five-zero, then six-zero.  A six-zero module therefore inherits the
+compiled classes of both earlier majors, and Gradle never recompiles an inherited class.  A Cassandra API that
+changed between two majors is silent at build time and fails at runtime with `NoSuchMethodError`.  After you add a
+new major version, compile the union of the inherited sources against the new shaded jar and confirm that the
+resulting error set matches the error set that the previous major produces.  Every error the new version adds names
+a file you must copy forward and override.

build.gradle

@@ -66,8 +66,8 @@ ext.dependencyLocation = (System.getenv("CASSANDRA_DEP_DIR") ?: "${rootDir}/depe
 //   - cassandraVersionEnumMap values must match the implemented_versions default
 //   - cassandraFullVersionMap values must match the supported_versions default
 // NOTE: Both maps must ALSO stay in sync with the values in build-dtest-jars.sh
-ext.cassandraVersionEnumMap = ["4.0": "FOURZERO", "4.1": "FOURONE", "5.0": "FIVEZERO"]
-ext.cassandraFullVersionMap = ["4.0": "4.0.17", "4.1": "4.1.4", "5.0": "5.0.7"]
+ext.cassandraVersionEnumMap = ["4.0": "FOURZERO", "4.1": "FOURONE", "5.0": "FIVEZERO", "6.0": "SIXZERO"]
+ext.cassandraFullVersionMap = ["4.0": "4.0.17", "4.1": "4.1.4", "5.0": "5.0.7", "6.0": "6.0-alpha2"]
 
 // Shared helper: sets implemented_versions and supported_versions system properties on a Test task.
 // When majorMinor is provided (e.g. "4.0"), uses that version directly.
@@ -77,7 +77,12 @@ ext.applyCassandraVersionFilter = { Test task, String majorMinor = null ->
     if (!majorMinor) {
         def ver = System.getenv("CASSANDRA_VERSION")
         if (!ver) return null
-        majorMinor = ver.split("\\.").take(2).join(".")
+        // Match the leading major.minor pair only, so that a pre-release such as 6.0-alpha2 resolves to 6.0
+        def versionMatcher = ver =~ /^(\d+)\.(\d+)/
+        if (!versionMatcher) {
+            throw new GradleException("Cannot parse CASSANDRA_VERSION: ${ver}")
+        }
+        majorMinor = "${versionMatcher[0][1]}.${versionMatcher[0][2]}"
     }
 
     def enumName = cassandraVersionEnumMap[majorMinor]

cassandra-analytics-cdc-sidecar/build.gradle

@@ -59,6 +59,16 @@ configurations {
     fivezeroTypes {
         description = 'Cassandra 5.0 Data Types'
     }
+
+    sixzero {
+        description = 'Cassandra 6.0 dependency'
+    }
+    sixzeroBridge {
+        description = 'Bridge to Cassandra 6.0'
+    }
+    sixzeroTypes {
+        description = 'Cassandra 6.0 Data Types'
+    }
 }
 
 dependencies {
@@ -72,6 +82,10 @@ dependencies {
     fivezeroBridge(project(path: ':cassandra-five-zero-bridge'))
     fivezeroTypes(project(path: ':cassandra-five-zero-types'))
 
+    sixzero(project(path: ':cassandra-six-zero', configuration: 'shadow'))
+    sixzeroBridge(project(path: ':cassandra-six-zero-bridge'))
+    sixzeroTypes(project(path: ':cassandra-six-zero-types'))
+
     implementation project(":cassandra-analytics-common")
     implementation project(':cassandra-analytics-cdc')
     implementation project(":cassandra-analytics-cdc-codec")
@@ -118,6 +132,10 @@ jar {
     dependsOn(configurations.fivezeroBridge)
     dependsOn(configurations.fivezeroTypes)
 
+    dependsOn(configurations.sixzero)
+    dependsOn(configurations.sixzeroBridge)
+    dependsOn(configurations.sixzeroTypes)
+
     exclude('META-INF/*')
     into('bridges') {
         from(configurations.fourzero.singleFile)
@@ -127,6 +145,10 @@ jar {
         from(configurations.fivezero.singleFile)
         from(configurations.fivezeroBridge.singleFile)
         from(configurations.fivezeroTypes.singleFile)
+
+        from(configurations.sixzero.singleFile)
+        from(configurations.sixzeroBridge.singleFile)
+        from(configurations.sixzeroTypes.singleFile)
     }
 }
 

cassandra-analytics-cdc/build.gradle

@@ -60,6 +60,19 @@ configurations {
     fivezeroAvroConverter {
         description = 'Avro converter for Cassandra 5.0 Data Types'
     }
+
+    sixzero {
+        description = 'Cassandra 6.0 dependency'
+    }
+    sixzeroBridge {
+        description = 'Bridge to Cassandra 6.0'
+    }
+    sixzeroTypes {
+        description = 'Cassandra 6.0 Data Types'
+    }
+    sixzeroAvroConverter {
+        description = 'Avro converter for Cassandra 6.0 Data Types'
+    }
 }
 
 dependencies {
@@ -78,6 +91,11 @@ dependencies {
     fivezeroTypes(project(path: ':cassandra-five-zero-types'))
     fivezeroAvroConverter(project(":cassandra-five-zero-avro-converter"))
 
+    sixzero(project(path: ':cassandra-six-zero', configuration: 'shadow'))
+    sixzeroBridge(project(path: ':cassandra-six-zero-bridge'))
+    sixzeroTypes(project(path: ':cassandra-six-zero-types'))
+    sixzeroAvroConverter(project(":cassandra-six-zero-avro-converter"))
+
     implementation "com.google.guava:guava:${guavaVersion}"
 
     implementation "org.apache.commons:commons-lang3:${project.commonsLang3Version}"
@@ -127,6 +145,11 @@ jar {
     dependsOn(configurations.fivezeroTypes)
     dependsOn(configurations.fivezeroAvroConverter)
 
+    dependsOn(configurations.sixzero)
+    dependsOn(configurations.sixzeroBridge)
+    dependsOn(configurations.sixzeroTypes)
+    dependsOn(configurations.sixzeroAvroConverter)
+
     exclude('META-INF/*')
     into('bridges') {
         from(configurations.fourzero.singleFile)
@@ -138,6 +161,11 @@ jar {
         from(configurations.fivezeroBridge.singleFile)
         from(configurations.fivezeroTypes.singleFile)
         from(configurations.fivezeroAvroConverter.singleFile)
+
+        from(configurations.sixzero.singleFile)
+        from(configurations.sixzeroBridge.singleFile)
+        from(configurations.sixzeroTypes.singleFile)
+        from(configurations.sixzeroAvroConverter.singleFile)
     }
 }
 
@@ -159,7 +187,7 @@ def configureCdcTestTask = { Test task, String majorMinor = null ->
         // Full version format to match CDC's TestVersionSupplier; tests both versions for backward compat.
         // 4.1 intentionally excluded from gradlew defaults to keep local iteration fast;
         // use testCassandra41 for targeted 4.1 runs. CI covers 4.1 via CASSANDRA_VERSION env var.
-        task.systemProperty "cassandra.sidecar.versions_to_test", "4.0.17,5.0.7"
+        task.systemProperty "cassandra.sidecar.versions_to_test", "4.0.17,5.0.7,6.0-alpha2"
     }
 
     task.minHeapSize = '1024m'

cassandra-analytics-cdc/src/test/java/org/apache/cassandra/cdc/test/TestVersionSupplier.java

@@ -32,7 +32,9 @@ private TestVersionSupplier()
 
     public static Stream<CassandraVersion> testVersions()
     {
-        String versions = System.getProperty("cassandra.sidecar.versions_to_test", "4.0.17,5.0.7");
+        // Keep in sync with configureCdcTestTask in cassandra-analytics-cdc/build.gradle, which sets the same
+        // property; this default applies only outside Gradle, for example in an IDE
+        String versions = System.getProperty("cassandra.sidecar.versions_to_test", "4.0.17,5.0.7,6.0-alpha2");
         return Arrays.stream(versions.split(","))
                      .map(String::trim)
                      .map(v -> CassandraVersion.fromVersion(v).orElseThrow(() -> new IllegalArgumentException("Unsupported version: " + v)));

cassandra-analytics-common/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java

@@ -37,13 +37,14 @@
  *
  * NOTE: The following values need to stay in sync with:
  * - build.gradle:
- *   - ext.cassandraVersionEnumMap = ["4.0": "FOURZERO", "4.1": "FOURONE", "5.0": "FIVEZERO"]
- *   - ext.cassandraFullVersionMap = ["4.0": "4.0.17", "4.1": "4.1.4", "5.0": "5.0.7"]
+ *   - ext.cassandraVersionEnumMap = ["4.0": "FOURZERO", "4.1": "FOURONE", "5.0": "FIVEZERO", "6.0": "SIXZERO"]
+ *   - ext.cassandraFullVersionMap = ["4.0": "4.0.17", "4.1": "4.1.4", "5.0": "5.0.7", "6.0": "6.0-alpha2"]
  * - build-dtest-jars.sh:
  *   - CANDIDATE_BRANCHES=(
  *      "cassandra-4.0:cassandra-4.0.17"
  *      "cassandra-4.1:99d9faeef57c9cf5240d11eac9db5b283e45a4f9"
  *      "cassandra-5.0:cassandra-5.0.7"
+ *      "cassandra-6.0:cassandra-6.0-alpha2"
  */
 public enum CassandraVersion
 {
@@ -72,7 +73,13 @@ public enum CassandraVersion
              // Cassandra 5.0 native sstable versions
              "big-oa",
              "bti-da",
-             }, 40);
+             }, 40),
+    SIXZERO(60, "6.0", "six-zero", new String[]{"big", "bti"},
+            new String[] {
+            // Cassandra 6.0 native sstable versions
+            "big-pa",
+            "bti-ea",
+            }, 40);
 
     private final int number;
     private final String name;
@@ -177,15 +184,15 @@ public boolean canRead(CassandraVersion other)
         // FOURONE is intentionally excluded from local-dev defaults to keep iteration fast;
         // CI covers 4.1 via explicit CASSANDRA_VERSION env var or per-version Gradle tasks (e.g. testCassandra41).
         String providedVersionsOrDefault = System.getProperty("cassandra.analytics.bridges.implemented_versions",
-                                                              String.join(",", FOURZERO.name(), FIVEZERO.name()));
+                                                              String.join(",", FOURZERO.name(), FIVEZERO.name(), SIXZERO.name()));
         implementedVersions = Arrays.stream(providedVersionsOrDefault.split(","))
                                     .map(CassandraVersion::valueOf)
                                     .filter(v -> v.sstableFormats().contains(configuredSSTableFormat))
                                     .toArray(CassandraVersion[]::new);
 
         // NOTE: These default versions must stay in sync with cassandraFullVersionMap in build.gradle.
         String providedSupportedVersionsOrDefault = System.getProperty("cassandra.analytics.bridges.supported_versions",
-                                                                       "cassandra-4.0.17,cassandra-5.0.7");
+                                                                       "cassandra-4.0.17,cassandra-5.0.7,cassandra-6.0-alpha2");
         supportedVersions = Arrays.stream(providedSupportedVersionsOrDefault.split(","))
                                   .filter(version -> CassandraVersion.fromVersion(version)
                                                                      .filter(v -> v.sstableFormats().contains(configuredSSTableFormat))