From 5a251e7d18e46e9bdb2f287c3d76a5dc83426af1 Mon Sep 17 00:00:00 2001 From: Yaniv Michael Kaul Date: Thu, 26 Mar 2026 18:28:20 +0200 Subject: [PATCH] (fix) tests: fix flaky test_no_connection_refused_on_timeout The test asserted that at least one WriteTimeout/WriteFailure must occur during 2000 concurrent LWT operations, but this is non-deterministic -- if the cluster handles all ops without timeout, the assertion fails. The actual purpose (PYTHON-91) is to verify the connection remains functional after LWT timeout stress. Replace the timeout assertion with a post-execution liveness check that verifies the session can still execute queries, and log a warning when the timeout path is not exercised. --- tests/integration/standard/test_query.py | 40 ++++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 5f1d5bfc19..28459c1215 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -920,8 +920,11 @@ def tearDown(self): """ Shutdown cluster """ - self.session.execute("DROP TABLE test3rf.lwt") - self.session.execute("DROP TABLE test3rf.lwt_clustering") + try: + self.session.execute("DROP TABLE test3rf.lwt") + self.session.execute("DROP TABLE test3rf.lwt_clustering") + except Exception as exc: + log.warning("tearDown failed to drop tables: %s", exc) self.cluster.shutdown() @xfail_scylla_version_lt(reason='scylladb/scylladb#18068 - LWT is not yet supported with tablets', @@ -931,6 +934,18 @@ def test_no_connection_refused_on_timeout(self): """ Test for PYTHON-91 "Connection closed after LWT timeout" Verifies that connection to the cluster is not shut down when timeout occurs. + + Fires a large batch of concurrent, conflicting LWT statements at the + server -- the same genuine contention used to reproduce PYTHON-91 -- + so that a real WriteTimeout/WriteFailure response (including its CAS + write_type) is parsed by the driver whenever the cluster is slow + enough to produce one. Whether that actually happens is inherently + non-deterministic (it depends on cluster load/scheduling), so unlike + the original version of this test we do not hard-assert on it -- that + assertion was the source of the flakiness. The deterministic + regression check is the post-stress liveness probe below: the + session must still be able to execute queries afterwards. + Number of iterations can be specified with LWT_ITERATIONS environment variable. Default value is 1000 """ @@ -955,15 +970,28 @@ def test_no_connection_refused_on_timeout(self): exception_type = type(result).__name__ if exception_type == "NoHostAvailable": pytest.fail("PYTHON-91: Disconnected from Cassandra: %s" % result.message) - if exception_type in ["WriteTimeout", "WriteFailure", "ReadTimeout", "ReadFailure", "ErrorMessageSub"]: - if type(result).__name__ in ["WriteTimeout", "WriteFailure"]: + if exception_type in ["WriteTimeout", "WriteFailure", "ReadTimeout", "ReadFailure", + "ErrorMessageSub", "OperationTimedOut"]: + if exception_type in ["WriteTimeout", "WriteFailure", "OperationTimedOut"]: received_timeout = True continue pytest.fail("Unexpected exception %s: %s" % (exception_type, result.message)) - # Make sure test passed - assert received_timeout + if not received_timeout: + # The cluster handled every op without a timeout this run -- that's + # fine (this is what actually caused the original flakiness when it + # was a hard assertion), just note it so the timeout path not being + # exercised is visible without failing the test. + log.warning("LWT stress of %d iterations completed without inducing a " + "WriteTimeout/WriteFailure/OperationTimedOut -- timeout path " + "not exercised this run.", iterations) + + # The actual PYTHON-91 regression check: can we still talk to the cluster? + try: + self.session.execute("SELECT key FROM system.local") + except NoHostAvailable: + pytest.fail("PYTHON-91: Connection to cluster was lost after LWT timeout stress") @xfail_scylla('Fails on Scylla with error `SERIAL/LOCAL_SERIAL consistency may only be requested for one partition at a time`') def test_was_applied_batch_stmt(self):