From 42d0c965c639c35849bcee503050a23ab09fafab Mon Sep 17 00:00:00 2001 From: Haoyan Geng Date: Mon, 21 Sep 2026 21:04:34 +0000 Subject: [PATCH 1/2] [SPARK-59699][CONNECT] Make DataFrameReader options case-insensitive --- python/pyspark/sql/connect/readwriter.py | 4 ++++ .../sql/tests/connect/test_connect_plan.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/python/pyspark/sql/connect/readwriter.py b/python/pyspark/sql/connect/readwriter.py index 561447a437e6d..0db09b8ae1599 100644 --- a/python/pyspark/sql/connect/readwriter.py +++ b/python/pyspark/sql/connect/readwriter.py @@ -111,6 +111,10 @@ def schema(self, schema: Union[StructType, str]) -> "DataFrameReader": schema.__doc__ = PySparkDataFrameReader.schema.__doc__ def option(self, key: str, value: "OptionalPrimitiveType") -> "DataFrameReader": + normalized_key = key.lower() + for existing_key in list(self._options): + if existing_key.lower() == normalized_key: + del self._options[existing_key] self._options[key] = cast(str, to_str(value)) return self diff --git a/python/pyspark/sql/tests/connect/test_connect_plan.py b/python/pyspark/sql/tests/connect/test_connect_plan.py index 685ea3656f5cb..7917ad8487c48 100644 --- a/python/pyspark/sql/tests/connect/test_connect_plan.py +++ b/python/pyspark/sql/tests/connect/test_connect_plan.py @@ -772,6 +772,22 @@ def test_datasource_read(self): self.assertEqual(len(data_source.paths), 1) self.assertEqual(data_source.paths[0], "test_path") + def test_reader_options_case_insensitive(self): + reader = DataFrameReader(self.connect) + df = ( + reader.option("versionAsOf", 0) + .option("versionasof", 1) + .option("versionAsOf", 2) + .table("myTable") + ) + options = df._plan.to_proto(self.connect).root.read.named_table.options + self.assertEqual(dict(options), {"versionAsOf": "2"}) + + reader = DataFrameReader(self.connect) + df = reader.option("header", False).option("header", True).load(format="csv") + options = df._plan.to_proto(self.connect).root.read.data_source.options + self.assertEqual(dict(options), {"header": "true"}) + def test_relation_changes(self): reader = DataFrameReader(self.connect) df = reader.option("startingVersion", "1").option("endingVersion", "5").changes("myTable") From e892e72ac1a530af31ee714e8a4a79203be93366 Mon Sep 17 00:00:00 2001 From: Haoyan Geng Date: Mon, 21 Sep 2026 21:21:40 +0000 Subject: [PATCH 2/2] [SPARK-59699][CONNECT] Explain reader option deduplication --- python/pyspark/sql/connect/readwriter.py | 1 + python/pyspark/sql/tests/connect/test_connect_plan.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/connect/readwriter.py b/python/pyspark/sql/connect/readwriter.py index 0db09b8ae1599..6da2982e92c40 100644 --- a/python/pyspark/sql/connect/readwriter.py +++ b/python/pyspark/sql/connect/readwriter.py @@ -111,6 +111,7 @@ def schema(self, schema: Union[StructType, str]) -> "DataFrameReader": schema.__doc__ = PySparkDataFrameReader.schema.__doc__ def option(self, key: str, value: "OptionalPrimitiveType") -> "DataFrameReader": + # Remove case-insensitive matches so the latest spelling and value win. normalized_key = key.lower() for existing_key in list(self._options): if existing_key.lower() == normalized_key: diff --git a/python/pyspark/sql/tests/connect/test_connect_plan.py b/python/pyspark/sql/tests/connect/test_connect_plan.py index 7917ad8487c48..6069b703b72f2 100644 --- a/python/pyspark/sql/tests/connect/test_connect_plan.py +++ b/python/pyspark/sql/tests/connect/test_connect_plan.py @@ -784,9 +784,9 @@ def test_reader_options_case_insensitive(self): self.assertEqual(dict(options), {"versionAsOf": "2"}) reader = DataFrameReader(self.connect) - df = reader.option("header", False).option("header", True).load(format="csv") + df = reader.option("header", False).option("HEADER", True).load(format="csv") options = df._plan.to_proto(self.connect).root.read.data_source.options - self.assertEqual(dict(options), {"header": "true"}) + self.assertEqual(dict(options), {"HEADER": "true"}) def test_relation_changes(self): reader = DataFrameReader(self.connect)