From 011f5a5b346e5dbda82f70d4c11209494f0f257b Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Mon, 7 Sep 2026 20:50:56 +0000
Subject: [PATCH 01/22] feat: [SPARK-59277][SQL][HIVE] Support first-class
 CHAR/VARCHAR in Hive extensions

---
 .../BaseScriptTransformationExec.scala        |  2 +-
 .../BaseScriptTransformationSuite.scala       | 18 ++++
 .../spark/sql/hive/HiveInspectors.scala       | 96 ++++++++++++++-----
 .../spark/sql/hive/HiveInspectorSuite.scala   | 75 ++++++++++++++-
 .../HiveScriptTransformationSuite.scala       | 22 +++++
 .../sql/hive/execution/HiveUDAFSuite.scala    | 20 ++++
 .../sql/hive/execution/HiveUDFSuite.scala     | 26 ++++-
 7 files changed, 230 insertions(+), 29 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index a313e2c671bec..f6685be7aab87 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -201,7 +201,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
   private lazy val outputFieldWriters: Seq[String => Any] = output.map { attr =>
     val converter = CatalystTypeConverters.createToCatalystConverter(attr.dataType)
     attr.dataType match {
-      case StringType => wrapperConvertException(data => data, converter)
+      case _: StringType => wrapperConvertException(data => data, converter)
       case BooleanType => wrapperConvertException(data => data.toBoolean, converter)
       case ByteType => wrapperConvertException(data => data.toByte, converter)
       case BinaryType =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index 62c5f5631776b..8bd5e9855b4bf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -86,6 +86,24 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
 
+  test("SPARK-59277: TRANSFORM output supports first-class CHAR/VARCHAR without SerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val input = Seq(("ab", "xyz")).toDF("c", "v")
+      checkAnswer(
+        input,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("c", CharType(4, "UTF8_LCASE"))(),
+            AttributeReference("v", VarcharType(5, "UNICODE_CI"))()),
+          child = child,
+          ioschema = defaultIOSchema),
+        Seq(Row("ab  ", "xyz")))
+    }
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
   test("script transformation should not swallow errors from upstream operators (no serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 76d9a08f603d0..3d3def913e4bf 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.common.`type`.{HiveChar, HiveDecimal, HiveInterval
 import org.apache.hadoop.hive.serde2.{io => hiveIo}
 import org.apache.hadoop.hive.serde2.objectinspector.{StructField => HiveStructField, _}
 import org.apache.hadoop.hive.serde2.objectinspector.primitive._
-import org.apache.hadoop.hive.serde2.typeinfo.{DecimalTypeInfo, TypeInfoFactory}
+import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, DecimalTypeInfo, TypeInfoFactory, VarcharTypeInfo}
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.AnalysisException
@@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.errors.DataTypeErrors.toSQLType
 import org.apache.spark.sql.execution.datasources.DaysWritable
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.TimestampNanosVal
@@ -103,10 +104,6 @@ import org.apache.spark.unsafe.types.UTF8String
  *   Struct: Object[] / java.util.List / java POJO
  *   Union: class StandardUnion { byte tag; Object object }
  *
- * NOTICE: HiveVarchar/HiveChar is not supported by catalyst, it will be simply considered as
- *  String type.
- *
- *
  * 2. Hive ObjectInspector is a group of flexible APIs to inspect value in different data
  *  representation, and developers can extend those API as needed, so technically,
  *  object inspector supports arbitrary data type in java.
@@ -280,7 +277,6 @@ private[hive] trait HiveInspectors {
       (o: Any) =>
         x.getWritableConstantValue
     case x: PrimitiveObjectInspector => x match {
-      // TODO we don't support the HiveVarcharObjectInspector yet.
       case _: StringObjectInspector if x.preferWritable() =>
         withNullSafe(o => getStringWritable(o))
       case _: StringObjectInspector =>
@@ -313,21 +309,28 @@ private[hive] trait HiveInspectors {
         withNullSafe(o => getByteWritable(o))
       case _: ByteObjectInspector =>
         withNullSafe(o => o.asInstanceOf[java.lang.Byte])
-        // To spark HiveVarchar and HiveChar are same as string
-      case _: HiveVarcharObjectInspector if x.preferWritable() =>
-        withNullSafe(o => getStringWritable(o))
-      case _: HiveVarcharObjectInspector =>
+      case hvoi: HiveVarcharObjectInspector if x.preferWritable() =>
+        val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
         withNullSafe { o =>
-            val s = o.asInstanceOf[UTF8String].toString
-            new HiveVarchar(s, s.length)
+          val varchar = new HiveVarchar(o.asInstanceOf[UTF8String].toString, length)
+          new hiveIo.HiveVarcharWritable(varchar)
         }
-      case _: HiveCharObjectInspector if x.preferWritable() =>
-        withNullSafe(o => getStringWritable(o))
-      case _: HiveCharObjectInspector =>
+      case hvoi: HiveVarcharObjectInspector =>
+        val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
         withNullSafe { o =>
-            val s = o.asInstanceOf[UTF8String].toString
-            new HiveChar(s, s.length)
-          }
+          new HiveVarchar(o.asInstanceOf[UTF8String].toString, length)
+        }
+      case hcoi: HiveCharObjectInspector if x.preferWritable() =>
+        val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
+        withNullSafe { o =>
+          val char = new HiveChar(o.asInstanceOf[UTF8String].toString, length)
+          new hiveIo.HiveCharWritable(char)
+        }
+      case hcoi: HiveCharObjectInspector =>
+        val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
+        withNullSafe { o =>
+          new HiveChar(o.asInstanceOf[UTF8String].toString, length)
+        }
       case _: JavaHiveDecimalObjectInspector =>
         withNullSafe(o =>
           HiveDecimal.create(o.asInstanceOf[Decimal].toJavaBigDecimal))
@@ -939,7 +942,14 @@ private[hive] trait HiveInspectors {
     case MapType(keyType, valueType, _) =>
       ObjectInspectorFactory.getStandardMapObjectInspector(
         toInspector(keyType), toInspector(valueType))
-    case StringType => PrimitiveObjectInspectorFactory.javaStringObjectInspector
+    // Hive object inspectors preserve the CHAR/VARCHAR length but have no collation metadata.
+    case c: CharType =>
+      PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
+        TypeInfoFactory.getCharTypeInfo(c.length))
+    case v: VarcharType =>
+      PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
+        TypeInfoFactory.getVarcharTypeInfo(v.length))
+    case _: StringType => PrimitiveObjectInspectorFactory.javaStringObjectInspector
     case IntegerType => PrimitiveObjectInspectorFactory.javaIntObjectInspector
     case DoubleType => PrimitiveObjectInspectorFactory.javaDoubleObjectInspector
     case BooleanType => PrimitiveObjectInspectorFactory.javaBooleanObjectInspector
@@ -986,7 +996,11 @@ private[hive] trait HiveInspectors {
    * @return Hive java objectinspector (recursively).
    */
   def toInspector(expr: Expression): ObjectInspector = expr match {
-    case Literal(value, StringType) =>
+    case Literal(value, c: CharType) =>
+      getHiveCharWritableConstantObjectInspector(value, c.length)
+    case Literal(value, v: VarcharType) =>
+      getHiveVarcharWritableConstantObjectInspector(value, v.length)
+    case Literal(value, _: StringType) =>
       getStringWritableConstantObjectInspector(value)
     case Literal(value, IntegerType) =>
       getIntWritableConstantObjectInspector(value)
@@ -1088,10 +1102,14 @@ private[hive] trait HiveInspectors {
         inspectorToDataType(m.getMapValueObjectInspector))
     case _: WritableStringObjectInspector => StringType
     case _: JavaStringObjectInspector => StringType
-    case _: WritableHiveVarcharObjectInspector => StringType
-    case _: JavaHiveVarcharObjectInspector => StringType
-    case _: WritableHiveCharObjectInspector => StringType
-    case _: JavaHiveCharObjectInspector => StringType
+    // Hive object inspectors cannot represent collations, so Hive function results use the
+    // default collation while preserving CHAR/VARCHAR length under first-class semantics.
+    case hvoi: HiveVarcharObjectInspector if SQLConf.get.charVarcharFirstClassTypes =>
+      VarcharType(hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength)
+    case _: HiveVarcharObjectInspector => StringType
+    case hcoi: HiveCharObjectInspector if SQLConf.get.charVarcharFirstClassTypes =>
+      CharType(hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength)
+    case _: HiveCharObjectInspector => StringType
     case _: WritableIntObjectInspector => IntegerType
     case _: JavaIntObjectInspector => IntegerType
     case _: WritableDoubleObjectInspector => DoubleType
@@ -1131,6 +1149,32 @@ private[hive] trait HiveInspectors {
     PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
       TypeInfoFactory.stringTypeInfo, getStringWritable(value))
 
+  private def getHiveCharWritableConstantObjectInspector(
+      value: Any,
+      length: Int): ObjectInspector = {
+    val writable = if (value == null) {
+      null
+    } else {
+      new hiveIo.HiveCharWritable(
+        new HiveChar(value.asInstanceOf[UTF8String].toString, length))
+    }
+    PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
+      TypeInfoFactory.getCharTypeInfo(length), writable)
+  }
+
+  private def getHiveVarcharWritableConstantObjectInspector(
+      value: Any,
+      length: Int): ObjectInspector = {
+    val writable = if (value == null) {
+      null
+    } else {
+      new hiveIo.HiveVarcharWritable(
+        new HiveVarchar(value.asInstanceOf[UTF8String].toString, length))
+    }
+    PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
+      TypeInfoFactory.getVarcharTypeInfo(length), writable)
+  }
+
   private def getIntWritableConstantObjectInspector(value: Any): ObjectInspector =
     PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
       TypeInfoFactory.intTypeInfo, getIntWritable(value))
@@ -1296,7 +1340,9 @@ private[hive] trait HiveInspectors {
       case IntegerType => intTypeInfo
       case LongType => longTypeInfo
       case ShortType => shortTypeInfo
-      case StringType => stringTypeInfo
+      case c: CharType => getCharTypeInfo(c.length)
+      case v: VarcharType => getVarcharTypeInfo(v.length)
+      case _: StringType => stringTypeInfo
       case d: DecimalType => decimalTypeInfo(d)
       case DateType => dateTypeInfo
       case TimestampType => timestampTypeInfo
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index a88dcd0ef56aa..acf6da8c06def 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -21,10 +21,10 @@ import java.util
 
 import org.apache.hadoop.hive.ql.udf.UDAFPercentile
 import org.apache.hadoop.hive.serde2.io.DoubleWritable
-import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectInspectorFactory, PrimitiveObjectInspector, StructObjectInspector}
+import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, ObjectInspector, ObjectInspectorFactory, PrimitiveObjectInspector, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
-import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo
+import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, DecimalTypeInfo, VarcharTypeInfo}
 import org.apache.hadoop.io.LongWritable
 
 import org.apache.spark.SparkFunSuite
@@ -32,10 +32,18 @@ import org.apache.spark.sql.{AnalysisException, Row, TestUserClassUDT}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Literal
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 
 class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
 
+  private def withFirstClassCharVarchar(enabled: Boolean)(f: => Unit): Unit = {
+    val conf = new SQLConf
+    conf.setConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS, enabled)
+    SQLConf.withExistingConf(conf)(f)
+  }
+
   def unwrap(data: Any, oi: ObjectInspector): Any = {
     val unwrapper = unwrapperFor(oi)
     unwrapper(data)
@@ -292,6 +300,69 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
     assert(typeInfo2.scale() === 10)
   }
 
+  test("SPARK-59277: Hive object inspectors preserve CHAR/VARCHAR type information") {
+    withFirstClassCharVarchar(enabled = true) {
+      Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
+        val inspector = toInspector(dataType).asInstanceOf[PrimitiveObjectInspector]
+        assert(inspectorToDataType(inspector) === dataType)
+        dataType match {
+          case c: CharType =>
+            assert(inspector.getTypeInfo.asInstanceOf[CharTypeInfo].getLength === c.length)
+          case v: VarcharType =>
+            assert(inspector.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength === v.length)
+        }
+      }
+    }
+  }
+
+  test("SPARK-59277: Hive object inspectors accept collated CHAR/VARCHAR values") {
+    withFirstClassCharVarchar(enabled = true) {
+      Seq[DataType](
+        CharType(5, "UTF8_LCASE"),
+        VarcharType(7, "UNICODE_CI")).foreach { dataType =>
+        val inspector = toInspector(dataType)
+        val value = UTF8String.fromString(dataType match {
+          case _: CharType => "ab   "
+          case _: VarcharType => "abc"
+        })
+        val expectedType = dataType match {
+          case c: CharType => CharType(c.length)
+          case v: VarcharType => VarcharType(v.length)
+        }
+        assert(inspectorToDataType(inspector) === expectedType)
+        assert(unwrap(wrap(value, inspector, dataType), inspector) === value)
+      }
+    }
+  }
+
+  test("SPARK-59277: Hive object inspectors support nested CHAR/VARCHAR") {
+    withFirstClassCharVarchar(enabled = true) {
+      val dataType = StructType(Seq(
+        StructField("chars", ArrayType(CharType(4))),
+        StructField("varchars", MapType(IntegerType, VarcharType(8)))))
+      assert(inspectorToDataType(toInspector(dataType)) === dataType)
+    }
+  }
+
+  test("SPARK-59277: Hive constant inspectors preserve CHAR/VARCHAR type information") {
+    withFirstClassCharVarchar(enabled = true) {
+      Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
+        val value = UTF8String.fromString("abc")
+        val inspector = toInspector(Literal.create(value, dataType))
+        assert(inspector.isInstanceOf[ConstantObjectInspector])
+        assert(inspectorToDataType(inspector) === dataType)
+      }
+    }
+  }
+
+  test("SPARK-59277: Hive CHAR/VARCHAR inspectors remain STRING under legacy semantics") {
+    withFirstClassCharVarchar(enabled = false) {
+      Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
+        assert(inspectorToDataType(toInspector(dataType)) === StringType)
+      }
+    }
+  }
+
   test("SPARK-57556: TIME type is unsupported in Hive object inspectors") {
     val timeType = TimeType()
     val expectedParams = Map("typeName" -> s"\"${timeType.sql}\"")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
index b1ff05b8c1b06..58241fd27dd08 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.types.DayTimeIntervalType._
 import org.apache.spark.sql.types.YearMonthIntervalType._
@@ -372,6 +373,27 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
     }
   }
 
+  test("SPARK-59277: TRANSFORM supports nested collated CHAR/VARCHAR with Hive SerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val query = sql(
+        """
+          |SELECT TRANSFORM(
+          |  array(CAST('ab' AS CHAR(4) COLLATE UTF8_LCASE)),
+          |  named_struct('value', CAST('xyz' AS VARCHAR(6) COLLATE UNICODE_CI)))
+          |USING 'cat'
+          |AS (
+          |  chars ARRAY<CHAR(4) COLLATE UTF8_LCASE>,
+          |  nested STRUCT<value: VARCHAR(6) COLLATE UNICODE_CI>)
+          |FROM VALUES (1) input(dummy)
+          |""".stripMargin)
+      assert(query.schema.map(_.dataType) === Seq(
+        ArrayType(CharType(4, "UTF8_LCASE")),
+        StructType(Seq(StructField("value", VarcharType(6, "UNICODE_CI"))))))
+      checkAnswer(query, Row(Seq("ab  "), Row("xyz")))
+    }
+  }
+
   test("SPARK-32400: TRANSFORM doesn't support CalendarIntervalType/UserDefinedType (hive serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
     withTempView("v") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
index 9fec3e58de255..a84b1d806cc62 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.ObjectHashAggregateExec
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{CharType, VarcharType}
 import org.apache.spark.tags.SlowHiveTest
 
 @SlowHiveTest
@@ -200,6 +201,25 @@ class HiveUDAFSuite extends QueryTest
     }
   }
 
+  test("SPARK-59277: Hive UDAF supports first-class CHAR/VARCHAR") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      Seq(
+        ("CHAR(5) COLLATE UTF8_LCASE", CharType(5), Row("def  ")),
+        ("VARCHAR(7) COLLATE UNICODE_CI", VarcharType(7), Row("def"))
+      ).foreach { case (dataType, expectedType, expectedRow) =>
+        val aggregate = sql(
+          s"""SELECT hive_max(value)
+             |FROM VALUES
+             |  (CAST('abc' AS $dataType)),
+             |  (CAST('def' AS $dataType))
+             |AS input(value)
+             |""".stripMargin)
+        assert(aggregate.schema.head.dataType === expectedType)
+        checkAnswer(aggregate, expectedRow)
+      }
+    }
+  }
+
   test("non-deterministic children expressions of UDAF") {
     withTempView("view1") {
       spark.range(1).selectExpr("id as x", "id as y").createTempView("view1")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 942172d1411c3..1a1fdd6c97865 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -46,7 +46,7 @@ import org.apache.spark.sql.hive.HiveGenericUDF
 import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
 import org.apache.spark.sql.hive.test.{TestHiveSingleton, TestUDTFJar}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{TimestampType, TimeType}
+import org.apache.spark.sql.types.{CharType, TimestampType, TimeType, VarcharType}
 import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.util.Utils
 
@@ -892,6 +892,30 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
     hiveContext.reset()
   }
 
+  test("SPARK-59277: Hive UDF and UDTF support first-class CHAR/VARCHAR") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      withUserDefinedFunction("hive_upper" -> true, "hive_explode" -> true) {
+        sql(s"CREATE TEMPORARY FUNCTION hive_upper AS '${classOf[GenericUDFUpper].getName}'")
+        sql(s"CREATE TEMPORARY FUNCTION hive_explode AS '${classOf[GenericUDTFExplode].getName}'")
+
+        val scalar = sql(
+          "SELECT hive_upper(CAST('Ab' AS CHAR(5) COLLATE UTF8_LCASE)) AS value")
+        assert(scalar.schema.head.dataType === CharType(5))
+        checkAnswer(scalar, Row("AB   "))
+
+        val table = sql(
+          """SELECT value
+            |FROM (
+            |  SELECT array(CAST('abc' AS VARCHAR(7) COLLATE UNICODE_CI)) AS values
+            |) input
+            |LATERAL VIEW hive_explode(values) exploded AS value
+            |""".stripMargin)
+        assert(table.schema.head.dataType === VarcharType(7))
+        checkAnswer(table, Row("abc"))
+      }
+    }
+  }
+
   test("SPARK-58792: copied HiveGenericUDF nodes must not share a mutable GenericUDF") {
     val tsAttr = AttributeReference("ts", TimestampType, nullable = false)()
     val constTs = Literal(

From 97ebb3fb82d4de0e7ca8f028ae4e53e010db6b04 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Tue, 8 Sep 2026 00:09:44 +0000
Subject: [PATCH 02/22] fix: [SPARK-59277][SQL][HIVE] Enforce CHAR/VARCHAR
 boundaries

---
 .../spark/sql/hive/HiveInspectors.scala       | 114 ++++++++++++++----
 .../HiveScriptTransformationExec.scala        |   6 +-
 .../spark/sql/hive/hiveUDFEvaluators.scala    |   7 +-
 .../org/apache/spark/sql/hive/hiveUDFs.scala  |   7 +-
 .../spark/sql/hive/HiveInspectorSuite.scala   |  69 ++++++++++-
 .../sql/hive/execution/HiveUDFSuite.scala     |  16 ++-
 6 files changed, 184 insertions(+), 35 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 3d3def913e4bf..fcf4cccc3db10 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -311,25 +311,57 @@ private[hive] trait HiveInspectors {
         withNullSafe(o => o.asInstanceOf[java.lang.Byte])
       case hvoi: HiveVarcharObjectInspector if x.preferWritable() =>
         val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
-        withNullSafe { o =>
-          val varchar = new HiveVarchar(o.asInstanceOf[UTF8String].toString, length)
-          new hiveIo.HiveVarcharWritable(varchar)
+        dataType match {
+          case v: VarcharType if SQLConf.get.charVarcharFirstClassTypes =>
+            withNullSafe { o =>
+              val checked = CharVarcharCodegenUtils.varcharTypeWriteSideCheck(
+                o.asInstanceOf[UTF8String], v.length)
+              new hiveIo.HiveVarcharWritable(new HiveVarchar(checked.toString, length))
+            }
+          case _ =>
+            withNullSafe(o => getStringWritable(o))
         }
       case hvoi: HiveVarcharObjectInspector =>
         val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
-        withNullSafe { o =>
-          new HiveVarchar(o.asInstanceOf[UTF8String].toString, length)
+        dataType match {
+          case v: VarcharType if SQLConf.get.charVarcharFirstClassTypes =>
+            withNullSafe { o =>
+              val checked = CharVarcharCodegenUtils.varcharTypeWriteSideCheck(
+                o.asInstanceOf[UTF8String], v.length)
+              new HiveVarchar(checked.toString, length)
+            }
+          case _ =>
+            withNullSafe { o =>
+              val value = o.asInstanceOf[UTF8String].toString
+              new HiveVarchar(value, value.length)
+            }
         }
       case hcoi: HiveCharObjectInspector if x.preferWritable() =>
         val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
-        withNullSafe { o =>
-          val char = new HiveChar(o.asInstanceOf[UTF8String].toString, length)
-          new hiveIo.HiveCharWritable(char)
+        dataType match {
+          case c: CharType if SQLConf.get.charVarcharFirstClassTypes =>
+            withNullSafe { o =>
+              val checked = CharVarcharCodegenUtils.charTypeWriteSideCheck(
+                o.asInstanceOf[UTF8String], c.length)
+              new hiveIo.HiveCharWritable(new HiveChar(checked.toString, length))
+            }
+          case _ =>
+            withNullSafe(o => getStringWritable(o))
         }
       case hcoi: HiveCharObjectInspector =>
         val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
-        withNullSafe { o =>
-          new HiveChar(o.asInstanceOf[UTF8String].toString, length)
+        dataType match {
+          case c: CharType if SQLConf.get.charVarcharFirstClassTypes =>
+            withNullSafe { o =>
+              val checked = CharVarcharCodegenUtils.charTypeWriteSideCheck(
+                o.asInstanceOf[UTF8String], c.length)
+              new HiveChar(checked.toString, length)
+            }
+          case _ =>
+            withNullSafe { o =>
+              val value = o.asInstanceOf[UTF8String].toString
+              new HiveChar(value, value.length)
+            }
         }
       case _: JavaHiveDecimalObjectInspector =>
         withNullSafe(o =>
@@ -832,6 +864,26 @@ private[hive] trait HiveInspectors {
             null
           }
         }
+      case (_, c: CharType) =>
+        val unwrapper = unwrapperFor(objectInspector)
+        data: Any => {
+          val value = unwrapper(data).asInstanceOf[UTF8String]
+          if (value == null) {
+            null
+          } else {
+            CharVarcharCodegenUtils.charTypeReadSideCheck(value, c.length)
+          }
+        }
+      case (_, v: VarcharType) =>
+        val unwrapper = unwrapperFor(objectInspector)
+        data: Any => {
+          val value = unwrapper(data).asInstanceOf[UTF8String]
+          if (value == null) {
+            null
+          } else {
+            CharVarcharCodegenUtils.varcharTypeReadSideCheck(value, v.length)
+          }
+        }
       case (li: ListObjectInspector, ArrayType(elementType, _)) =>
         val unwrapper = unwrapperFor(li.getListElementObjectInspector, elementType)
         data: Any => {
@@ -945,10 +997,10 @@ private[hive] trait HiveInspectors {
     // Hive object inspectors preserve the CHAR/VARCHAR length but have no collation metadata.
     case c: CharType =>
       PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
-        TypeInfoFactory.getCharTypeInfo(c.length))
+        toHiveCharTypeInfo(c))
     case v: VarcharType =>
       PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
-        TypeInfoFactory.getVarcharTypeInfo(v.length))
+        toHiveVarcharTypeInfo(v))
     case _: StringType => PrimitiveObjectInspectorFactory.javaStringObjectInspector
     case IntegerType => PrimitiveObjectInspectorFactory.javaIntObjectInspector
     case DoubleType => PrimitiveObjectInspectorFactory.javaDoubleObjectInspector
@@ -988,6 +1040,20 @@ private[hive] trait HiveInspectors {
       messageParameters = Map("typeName" -> toSQLType(dataType)))
   }
 
+  private def toHiveCharTypeInfo(dataType: CharType): CharTypeInfo = {
+    if (dataType.length < 1 || dataType.length > HiveChar.MAX_CHAR_LENGTH) {
+      throw unsupportedHiveType(dataType)
+    }
+    TypeInfoFactory.getCharTypeInfo(dataType.length)
+  }
+
+  private def toHiveVarcharTypeInfo(dataType: VarcharType): VarcharTypeInfo = {
+    if (dataType.length < 1 || dataType.length > HiveVarchar.MAX_VARCHAR_LENGTH) {
+      throw unsupportedHiveType(dataType)
+    }
+    TypeInfoFactory.getVarcharTypeInfo(dataType.length)
+  }
+
   /**
    * Map the catalyst expression to ObjectInspector, however,
    * if the expression is `Literal` or foldable, a constant writable object inspector returns;
@@ -997,9 +1063,9 @@ private[hive] trait HiveInspectors {
    */
   def toInspector(expr: Expression): ObjectInspector = expr match {
     case Literal(value, c: CharType) =>
-      getHiveCharWritableConstantObjectInspector(value, c.length)
+      getHiveCharWritableConstantObjectInspector(value, c)
     case Literal(value, v: VarcharType) =>
-      getHiveVarcharWritableConstantObjectInspector(value, v.length)
+      getHiveVarcharWritableConstantObjectInspector(value, v)
     case Literal(value, _: StringType) =>
       getStringWritableConstantObjectInspector(value)
     case Literal(value, IntegerType) =>
@@ -1151,28 +1217,32 @@ private[hive] trait HiveInspectors {
 
   private def getHiveCharWritableConstantObjectInspector(
       value: Any,
-      length: Int): ObjectInspector = {
+      dataType: CharType): ObjectInspector = {
     val writable = if (value == null) {
       null
     } else {
+      val checked = CharVarcharCodegenUtils.charTypeWriteSideCheck(
+        value.asInstanceOf[UTF8String], dataType.length)
       new hiveIo.HiveCharWritable(
-        new HiveChar(value.asInstanceOf[UTF8String].toString, length))
+        new HiveChar(checked.toString, dataType.length))
     }
     PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
-      TypeInfoFactory.getCharTypeInfo(length), writable)
+      toHiveCharTypeInfo(dataType), writable)
   }
 
   private def getHiveVarcharWritableConstantObjectInspector(
       value: Any,
-      length: Int): ObjectInspector = {
+      dataType: VarcharType): ObjectInspector = {
     val writable = if (value == null) {
       null
     } else {
+      val checked = CharVarcharCodegenUtils.varcharTypeWriteSideCheck(
+        value.asInstanceOf[UTF8String], dataType.length)
       new hiveIo.HiveVarcharWritable(
-        new HiveVarchar(value.asInstanceOf[UTF8String].toString, length))
+        new HiveVarchar(checked.toString, dataType.length))
     }
     PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
-      TypeInfoFactory.getVarcharTypeInfo(length), writable)
+      toHiveVarcharTypeInfo(dataType), writable)
   }
 
   private def getIntWritableConstantObjectInspector(value: Any): ObjectInspector =
@@ -1340,8 +1410,8 @@ private[hive] trait HiveInspectors {
       case IntegerType => intTypeInfo
       case LongType => longTypeInfo
       case ShortType => shortTypeInfo
-      case c: CharType => getCharTypeInfo(c.length)
-      case v: VarcharType => getVarcharTypeInfo(v.length)
+      case c: CharType => toHiveCharTypeInfo(c)
+      case v: VarcharType => toHiveVarcharTypeInfo(v)
       case _: StringType => stringTypeInfo
       case d: DecimalType => decimalTypeInfo(d)
       case DateType => dateTypeInfo
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
index de2d15415837a..6d9861a67d62d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
@@ -75,7 +75,9 @@ private[hive] case class HiveScriptTransformationExec(
       val mutableRow = new SpecificInternalRow(output.map(_.dataType))
 
       @transient
-      lazy val unwrappers = outputSoi.getAllStructFieldRefs.asScala.map(unwrapperFor)
+      lazy val unwrappers = outputSoi.getAllStructFieldRefs.asScala.zip(output).map {
+        case (field, attr) => unwrapperFor(field.getFieldObjectInspector, attr.dataType)
+      }
 
       override def hasNext: Boolean = {
         if (completed) {
@@ -130,7 +132,7 @@ private[hive] case class HiveScriptTransformationExec(
           if (dataList.get(i) == null) {
             mutableRow.setNullAt(i)
           } else {
-            unwrappers(i)(dataList.get(i), mutableRow, i)
+            mutableRow.update(i, unwrappers(i)(dataList.get(i)))
           }
           i += 1
         }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
index f09ee7eed93ef..969d66ff10f74 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
@@ -162,9 +162,12 @@ class HiveGenericUDFEvaluator(
   }
 
   @transient
-  private lazy val unwrapper: Any => Any = unwrapperFor(returnInspector)
+  private lazy val catalystReturnType = inspectorToDataType(returnInspector)
 
-  override def returnType: DataType = inspectorToDataType(returnInspector)
+  @transient
+  private lazy val unwrapper: Any => Any = unwrapperFor(returnInspector, catalystReturnType)
+
+  override def returnType: DataType = catalystReturnType
 
   def setArg(index: Int, arg: Any): Unit =
     deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(() => arg)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index e5db4d9c7384a..c70124d8994c1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -245,7 +245,7 @@ private[hive] case class HiveGenericUDTF(
   private lazy val wrappers = children.map(x => wrapperFor(toInspector(x), x.dataType)).toArray
 
   @transient
-  private lazy val unwrapper = unwrapperFor(outputInspector)
+  private lazy val unwrapper = unwrapperFor(outputInspector, elementSchema)
 
   @transient
   private lazy val inputProjection = new InterpretedProjection(children)
@@ -399,7 +399,7 @@ private[hive] case class HiveUDAFFunction(
   // Unwrapper function used to unwrap final aggregation result objects returned by Hive UDAFs into
   // Spark SQL specific format.
   @transient
-  private lazy val resultUnwrapper = unwrapperFor(finalHiveEvaluator.objectInspector)
+  private lazy val resultUnwrapper = unwrapperFor(finalHiveEvaluator.objectInspector, dataType)
 
   @transient
   private lazy val cached: Array[AnyRef] = new Array[AnyRef](children.length)
@@ -507,7 +507,8 @@ private[hive] case class HiveUDAFFunction(
 
   // Helper class used to de/serialize Hive UDAF `AggregationBuffer` objects
   private class AggregationBufferSerDe {
-    private val partialResultUnwrapper = unwrapperFor(partial1HiveEvaluator.objectInspector)
+    private val partialResultUnwrapper =
+      unwrapperFor(partial1HiveEvaluator.objectInspector, partialResultDataType)
 
     private val partialResultWrapper =
       wrapperFor(partial1HiveEvaluator.objectInspector, partialResultDataType)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index acf6da8c06def..91edad20e5cf9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
 import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, DecimalTypeInfo, VarcharTypeInfo}
 import org.apache.hadoop.io.LongWritable
 
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkRuntimeException}
 import org.apache.spark.sql.{AnalysisException, Row, TestUserClassUDT}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Literal
@@ -322,15 +322,19 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
         VarcharType(7, "UNICODE_CI")).foreach { dataType =>
         val inspector = toInspector(dataType)
         val value = UTF8String.fromString(dataType match {
-          case _: CharType => "ab   "
+          case _: CharType => "ab"
           case _: VarcharType => "abc"
         })
+        val expectedValue = dataType match {
+          case _: CharType => UTF8String.fromString("ab   ")
+          case _: VarcharType => value
+        }
         val expectedType = dataType match {
           case c: CharType => CharType(c.length)
           case v: VarcharType => VarcharType(v.length)
         }
         assert(inspectorToDataType(inspector) === expectedType)
-        assert(unwrap(wrap(value, inspector, dataType), inspector) === value)
+        assert(unwrap(wrap(value, inspector, dataType), inspector) === expectedValue)
       }
     }
   }
@@ -340,7 +344,18 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
       val dataType = StructType(Seq(
         StructField("chars", ArrayType(CharType(4))),
         StructField("varchars", MapType(IntegerType, VarcharType(8)))))
-      assert(inspectorToDataType(toInspector(dataType)) === dataType)
+      val inspector = toInspector(dataType)
+      assert(inspectorToDataType(inspector) === dataType)
+
+      val input = InternalRow(
+        new GenericArrayData(Array[Any](UTF8String.fromString("a"))),
+        ArrayBasedMapData(
+          Array[Any](1),
+          Array[Any](UTF8String.fromString("value"))))
+      val result = unwrapperFor(inspector, dataType)(
+        wrap(input, inspector, dataType)).asInstanceOf[InternalRow]
+      assert(result.getArray(0).getUTF8String(0) === UTF8String.fromString("a   "))
+      assert(result.getMap(1).valueArray().getUTF8String(0) === UTF8String.fromString("value"))
     }
   }
 
@@ -351,6 +366,12 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
         val inspector = toInspector(Literal.create(value, dataType))
         assert(inspector.isInstanceOf[ConstantObjectInspector])
         assert(inspectorToDataType(inspector) === dataType)
+        val expected = dataType match {
+          case _: CharType => UTF8String.fromString("abc  ")
+          case _: VarcharType => value
+        }
+        assert(unwrapperFor(inspector, dataType)(
+          inspector.asInstanceOf[ConstantObjectInspector].getWritableConstantValue) === expected)
       }
     }
   }
@@ -363,6 +384,46 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
     }
   }
 
+  test("SPARK-59277: Hive CHAR/VARCHAR boundaries enforce Spark length semantics") {
+    withFirstClassCharVarchar(enabled = true) {
+      val varchar = VarcharType(3)
+      val varcharInspector = toInspector(varchar)
+      checkError(
+        exception = intercept[SparkRuntimeException] {
+          wrap(UTF8String.fromString("abcd"), varcharInspector, varchar)
+        },
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "3"))
+
+      val char = CharType(5)
+      val charUnwrapper =
+        unwrapperFor(PrimitiveObjectInspectorFactory.javaStringObjectInspector, char)
+      assert(charUnwrapper("ab") === UTF8String.fromString("ab   "))
+      checkError(
+        exception = intercept[SparkRuntimeException] {
+          charUnwrapper("abcdef")
+        },
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "5"))
+    }
+  }
+
+  test("SPARK-59277: Hive object inspectors reject unsupported CHAR/VARCHAR lengths") {
+    withFirstClassCharVarchar(enabled = true) {
+      Seq[DataType](CharType(0), CharType(256), VarcharType(65536)).foreach { dataType =>
+        val expectedParams = Map("typeName" -> s"\"${dataType.sql}\"")
+        checkError(
+          exception = intercept[AnalysisException](toInspector(dataType)),
+          condition = "UNSUPPORTED_DATATYPE",
+          parameters = expectedParams)
+        checkError(
+          exception = intercept[AnalysisException](dataType.toTypeInfo),
+          condition = "UNSUPPORTED_DATATYPE",
+          parameters = expectedParams)
+      }
+    }
+  }
+
   test("SPARK-57556: TIME type is unsupported in Hive object inspectors") {
     val timeType = TimeType()
     val expectedParams = Map("typeName" -> s"\"${timeType.sql}\"")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 1a1fdd6c97865..aa7e91f98c6ae 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -46,7 +46,7 @@ import org.apache.spark.sql.hive.HiveGenericUDF
 import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
 import org.apache.spark.sql.hive.test.{TestHiveSingleton, TestUDTFJar}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{CharType, TimestampType, TimeType, VarcharType}
+import org.apache.spark.sql.types.{CharType, StringType, TimestampType, TimeType, VarcharType}
 import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.util.Utils
 
@@ -894,10 +894,22 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
 
   test("SPARK-59277: Hive UDF and UDTF support first-class CHAR/VARCHAR") {
     withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      withUserDefinedFunction("hive_upper" -> true, "hive_explode" -> true) {
+      withUserDefinedFunction(
+          "hive_simple_concat" -> true,
+          "hive_upper" -> true,
+          "hive_explode" -> true) {
+        sql(s"CREATE TEMPORARY FUNCTION hive_simple_concat AS " +
+          s"'${classOf[UDFStringString].getName}'")
         sql(s"CREATE TEMPORARY FUNCTION hive_upper AS '${classOf[GenericUDFUpper].getName}'")
         sql(s"CREATE TEMPORARY FUNCTION hive_explode AS '${classOf[GenericUDTFExplode].getName}'")
 
+        val simple = sql(
+          """SELECT hive_simple_concat(
+            |  CAST('A' AS CHAR(3)),
+            |  CAST('b' AS VARCHAR(2))) AS value""".stripMargin)
+        assert(simple.schema.head.dataType === StringType)
+        checkAnswer(simple, Row("A b"))
+
         val scalar = sql(
           "SELECT hive_upper(CAST('Ab' AS CHAR(5) COLLATE UTF8_LCASE)) AS value")
         assert(scalar.schema.head.dataType === CharType(5))

From 3b611b326d435ff606f858f688f4d194b782b9e6 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Wed, 9 Sep 2026 14:34:02 +0000
Subject: [PATCH 03/22] fix: [SPARK-59277] preserve analyzed Hive conversion
 types

---
 .../spark/sql/hive/HiveInspectors.scala       | 26 +++++++---
 .../HiveScriptTransformationExec.scala        |  4 +-
 .../spark/sql/hive/hiveUDFEvaluators.scala    |  8 +--
 .../org/apache/spark/sql/hive/hiveUDFs.scala  | 41 ++++++++++-----
 .../sql/hive/execution/HiveUDFSuite.scala     | 52 +++++++++++++++++++
 5 files changed, 106 insertions(+), 25 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index fcf4cccc3db10..7a074266708c7 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -312,7 +312,7 @@ private[hive] trait HiveInspectors {
       case hvoi: HiveVarcharObjectInspector if x.preferWritable() =>
         val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
         dataType match {
-          case v: VarcharType if SQLConf.get.charVarcharFirstClassTypes =>
+          case v: VarcharType =>
             withNullSafe { o =>
               val checked = CharVarcharCodegenUtils.varcharTypeWriteSideCheck(
                 o.asInstanceOf[UTF8String], v.length)
@@ -324,7 +324,7 @@ private[hive] trait HiveInspectors {
       case hvoi: HiveVarcharObjectInspector =>
         val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
         dataType match {
-          case v: VarcharType if SQLConf.get.charVarcharFirstClassTypes =>
+          case v: VarcharType =>
             withNullSafe { o =>
               val checked = CharVarcharCodegenUtils.varcharTypeWriteSideCheck(
                 o.asInstanceOf[UTF8String], v.length)
@@ -339,7 +339,7 @@ private[hive] trait HiveInspectors {
       case hcoi: HiveCharObjectInspector if x.preferWritable() =>
         val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
         dataType match {
-          case c: CharType if SQLConf.get.charVarcharFirstClassTypes =>
+          case c: CharType =>
             withNullSafe { o =>
               val checked = CharVarcharCodegenUtils.charTypeWriteSideCheck(
                 o.asInstanceOf[UTF8String], c.length)
@@ -351,7 +351,7 @@ private[hive] trait HiveInspectors {
       case hcoi: HiveCharObjectInspector =>
         val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
         dataType match {
-          case c: CharType if SQLConf.get.charVarcharFirstClassTypes =>
+          case c: CharType =>
             withNullSafe { o =>
               val checked = CharVarcharCodegenUtils.charTypeWriteSideCheck(
                 o.asInstanceOf[UTF8String], c.length)
@@ -841,8 +841,9 @@ private[hive] trait HiveInspectors {
    * Catalyst `dataType` to preserve nanosecond timestamp precision. The plain
    * `unwrapperFor(ObjectInspector)` cannot do this because a Hive `TimestampObjectInspector`
    * maps to micros by default; here the nanos timestamp types are produced as `TimestampNanosVal`,
-   * recursing through array/map/struct so nested nanos timestamps round-trip correctly. Any other
-   * type is delegated to the `ObjectInspector`-only overload.
+   * recursing through array/map/struct so nested nanos timestamps round-trip correctly. CHAR and
+   * VARCHAR targets also apply their read-side length and padding checks. Any other type is
+   * delegated to the `ObjectInspector`-only overload.
    */
   def unwrapperFor(objectInspector: ObjectInspector, dataType: DataType): Any => Any =
     (objectInspector, dataType) match {
@@ -952,6 +953,19 @@ private[hive] trait HiveInspectors {
         (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
     }
 
+  /**
+   * Builds an in-place unwrapper that also honors target-type-specific conversions.
+   */
+  def unwrapperFor(
+      field: HiveStructField,
+      dataType: DataType): (Any, InternalRow, Int) => Unit = dataType match {
+    case _: CharType | _: VarcharType =>
+      val unwrapper = unwrapperFor(field.getFieldObjectInspector, dataType)
+      (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
+    case _ =>
+      unwrapperFor(field)
+  }
+
   def wrap(a: Any, oi: ObjectInspector, dataType: DataType): AnyRef = {
     wrapperFor(oi, dataType)(a).asInstanceOf[AnyRef]
   }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
index 6d9861a67d62d..ee6b9b358279a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
@@ -76,7 +76,7 @@ private[hive] case class HiveScriptTransformationExec(
 
       @transient
       lazy val unwrappers = outputSoi.getAllStructFieldRefs.asScala.zip(output).map {
-        case (field, attr) => unwrapperFor(field.getFieldObjectInspector, attr.dataType)
+        case (field, attr) => unwrapperFor(field, attr.dataType)
       }
 
       override def hasNext: Boolean = {
@@ -132,7 +132,7 @@ private[hive] case class HiveScriptTransformationExec(
           if (dataList.get(i) == null) {
             mutableRow.setNullAt(i)
           } else {
-            mutableRow.update(i, unwrappers(i)(dataList.get(i)))
+            unwrappers(i)(dataList.get(i), mutableRow, i)
           }
           i += 1
         }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
index 969d66ff10f74..bafec1575f047 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
@@ -112,7 +112,9 @@ class HiveSimpleUDFEvaluator(
 }
 
 class HiveGenericUDFEvaluator(
-    funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
+    funcWrapper: HiveFunctionWrapper,
+    children: Seq[Expression],
+    resolvedReturnType: Option[DataType])
   extends HiveUDFEvaluatorBase[GenericUDF](funcWrapper, children) {
 
   // SPARK-58792: copied expression nodes (e.g. via withNewChildrenInternal) share one
@@ -161,8 +163,8 @@ class HiveGenericUDFEvaluator(
     case (inspect, child) => new DeferredObjectAdapter(inspect, child.dataType)
   }
 
-  @transient
-  private lazy val catalystReturnType = inspectorToDataType(returnInspector)
+  private lazy val catalystReturnType =
+    resolvedReturnType.getOrElse(inspectorToDataType(returnInspector))
 
   @transient
   private lazy val unwrapper: Any => Any = unwrapperFor(returnInspector, catalystReturnType)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index c70124d8994c1..b5532c919f7db 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -114,7 +114,10 @@ private[hive] case class HiveSimpleUDF(
 }
 
 private[hive] case class HiveGenericUDF(
-    name: String, funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
+    name: String,
+    funcWrapper: HiveFunctionWrapper,
+    children: Seq[Expression],
+    resolvedDataType: Option[DataType] = None)
   extends Expression
   with HiveInspectors
   with UserDefinedExpression {
@@ -130,10 +133,12 @@ private[hive] case class HiveGenericUDF(
   override def foldable: Boolean = evaluator.isUDFDeterministic &&
     evaluator.returnInspector.isInstanceOf[ConstantObjectInspector]
 
-  override lazy val dataType: DataType = inspectorToDataType(evaluator.returnInspector)
+  override lazy val dataType: DataType = resolvedDataType.getOrElse(evaluator.returnType)
 
-  @transient
-  private lazy val evaluator = new HiveGenericUDFEvaluator(funcWrapper, children)
+  // The evaluator carries the return type resolved during analysis. Its mutable Hive state is
+  // transient and is rebuilt on the executor, but the resolved Catalyst type must not be.
+  private lazy val evaluator =
+    new HiveGenericUDFEvaluator(funcWrapper, children, resolvedDataType)
 
   override def eval(input: InternalRow): Any = {
     children.zipWithIndex.foreach {
@@ -155,7 +160,7 @@ private[hive] case class HiveGenericUDF(
   }
 
   override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
-    copy(children = newChildren)
+    copy(children = newChildren, resolvedDataType = Some(dataType))
 
   protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val refEvaluator = ctx.addReferenceObj("evaluator", evaluator)
@@ -333,7 +338,8 @@ private[hive] case class HiveUDAFFunction(
     children: Seq[Expression],
     isUDAFBridgeRequired: Boolean = false,
     mutableAggBufferOffset: Int = 0,
-    inputAggBufferOffset: Int = 0)
+    inputAggBufferOffset: Int = 0,
+    resolvedDataTypes: Option[(DataType, DataType)] = None)
   extends TypedImperativeAggregate[HiveUDAFBuffer]
   with HiveInspectors
   with UserDefinedExpression {
@@ -341,10 +347,14 @@ private[hive] case class HiveUDAFFunction(
   final override val nodePatterns: Seq[TreePattern] = Seq(USER_DEFINED_AGGREGATION)
 
   override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
-    copy(mutableAggBufferOffset = newMutableAggBufferOffset)
+    copy(
+      mutableAggBufferOffset = newMutableAggBufferOffset,
+      resolvedDataTypes = Some(catalystDataTypes))
 
   override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate =
-    copy(inputAggBufferOffset = newInputAggBufferOffset)
+    copy(
+      inputAggBufferOffset = newInputAggBufferOffset,
+      resolvedDataTypes = Some(catalystDataTypes))
 
   // Hive `ObjectInspector`s for all child expressions (input parameters of the function).
   @transient
@@ -387,10 +397,13 @@ private[hive] case class HiveUDAFFunction(
       evaluator.init(GenericUDAFEvaluator.Mode.FINAL, Array(partial1HiveEvaluator.objectInspector)))
   }
 
-  // Spark SQL data type of partial aggregation results
-  @transient
-  private lazy val partialResultDataType =
-    inspectorToDataType(partial1HiveEvaluator.objectInspector)
+  // Resolve both Catalyst types together during analysis so the partial type is serialized with
+  // the expression instead of being rebuilt from the executor's SQLConf.
+  private lazy val catalystDataTypes = resolvedDataTypes.getOrElse((
+    inspectorToDataType(partial1HiveEvaluator.objectInspector),
+    inspectorToDataType(finalHiveEvaluator.objectInspector)))
+
+  private def partialResultDataType: DataType = catalystDataTypes._1
 
   // Wrapper functions used to wrap Spark SQL input arguments into Hive specific format.
   @transient
@@ -409,7 +422,7 @@ private[hive] case class HiveUDAFFunction(
 
   override def nullable: Boolean = true
 
-  override lazy val dataType: DataType = inspectorToDataType(finalHiveEvaluator.objectInspector)
+  override lazy val dataType: DataType = catalystDataTypes._2
 
   override def prettyName: String = name
 
@@ -552,7 +565,7 @@ private[hive] case class HiveUDAFFunction(
   }
 
   override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
-    copy(children = newChildren)
+    copy(children = newChildren, resolvedDataTypes = Some(catalystDataTypes))
 }
 
 case class HiveUDAFBuffer(buf: AggregationBuffer, canDoMerge: Boolean)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index aa7e91f98c6ae..b7b6687131080 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -928,6 +928,58 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
     }
   }
 
+  test("SPARK-59277: Hive UDF supports preserved CHAR without standard semantics") {
+    withSQLConf(
+        SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true",
+        SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "false") {
+      withUserDefinedFunction("hive_upper" -> true) {
+        sql(s"CREATE TEMPORARY FUNCTION hive_upper AS '${classOf[GenericUDFUpper].getName}'")
+
+        val result = sql("SELECT hive_upper(CAST('Ab' AS CHAR(5))) AS value")
+        assert(result.schema.head.dataType === CharType(5))
+        checkAnswer(result, Row("AB   "))
+      }
+    }
+  }
+
+  test("SPARK-59277: persisted views use their captured Hive conversion types") {
+    withUserDefinedFunction("hive_upper" -> false, "hive_max" -> false) {
+      withView("first_class_hive_view", "legacy_hive_view") {
+        sql(s"CREATE FUNCTION hive_upper AS '${classOf[GenericUDFUpper].getName}'")
+        sql(s"CREATE FUNCTION hive_max AS '${classOf[GenericUDAFMax].getName}'")
+        val query =
+          """SELECT
+            |  hive_upper(CAST('Ab' AS CHAR(5))) AS scalar_value,
+            |  hive_max(CAST('cd' AS CHAR(4))) AS aggregate_value""".stripMargin
+
+        withSQLConf(
+            SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true",
+            SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") {
+          sql(s"CREATE VIEW first_class_hive_view AS $query")
+        }
+        withSQLConf(
+            SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "false",
+            SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "false") {
+          val result = sql("SELECT * FROM first_class_hive_view")
+          assert(result.schema.map(_.dataType) === Seq(StringType, StringType))
+          checkAnswer(result, Row("AB   ", "cd  "))
+        }
+
+        withSQLConf(
+            SQLConf.LEGACY_CHAR_VARCHAR_AS_STRING.key -> "true",
+            SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "false",
+            SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "false") {
+          sql(s"CREATE VIEW legacy_hive_view AS $query")
+        }
+        withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+          val result = sql("SELECT * FROM legacy_hive_view")
+          assert(result.schema.map(_.dataType) === Seq(StringType, StringType))
+          checkAnswer(result, Row("AB", "cd"))
+        }
+      }
+    }
+  }
+
   test("SPARK-58792: copied HiveGenericUDF nodes must not share a mutable GenericUDF") {
     val tsAttr = AttributeReference("ts", TimestampType, nullable = false)()
     val constTs = Literal(

From 2294f32bc8b392b88018bd03f1330c43f946fc9f Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Thu, 10 Sep 2026 00:50:42 +0000
Subject: [PATCH 04/22] test: [SPARK-59277] update Hive reflection expectation

---
 .../org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala
index ce46baae9e468..c90d31faa52a8 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala
@@ -29,7 +29,7 @@ class HiveScalaReflectionSuite extends SparkFunSuite {
 
   test("SPARK-38510: ScalaReflection.getConstructorParameterNames should work for classes with " +
     "cyclic annotation references") {
-    assert(Seq("name", "funcWrapper", "children") ===
+    assert(Seq("name", "funcWrapper", "children", "resolvedDataType") ===
       ScalaReflection.getConstructorParameterNames(classOf[HiveGenericUDF]))
   }
 }

From c77094331c527f3688ecd13972098665d74ed042 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Fri, 11 Sep 2026 23:37:33 +0000
Subject: [PATCH 05/22] fix: [SPARK-59277] preserve nested Hive conversion
 types

---
 .../spark/sql/hive/HiveInspectors.scala       |  2 +-
 .../org/apache/spark/sql/hive/hiveUDFs.scala  | 15 ++++++++------
 .../spark/sql/hive/HiveInspectorSuite.scala   | 12 ++++++++++-
 .../sql/hive/execution/HiveUDFSuite.scala     | 20 +++++++++++++++++--
 4 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 7a074266708c7..18f4b210769c8 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -959,7 +959,7 @@ private[hive] trait HiveInspectors {
   def unwrapperFor(
       field: HiveStructField,
       dataType: DataType): (Any, InternalRow, Int) => Unit = dataType match {
-    case _: CharType | _: VarcharType =>
+    case dt if CharVarcharUtils.hasCharVarchar(dt) =>
       val unwrapper = unwrapperFor(field.getFieldObjectInspector, dataType)
       (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
     case _ =>
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index b5532c919f7db..6d93ad39f6824 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -212,7 +212,8 @@ private[hive] case class HiveGenericUDF(
 private[hive] case class HiveGenericUDTF(
     name: String,
     funcWrapper: HiveFunctionWrapper,
-    children: Seq[Expression])
+    children: Seq[Expression],
+    resolvedElementSchema: Option[StructType] = None)
   extends Generator with HiveInspectors with CodegenFallback with UserDefinedExpression {
 
   @transient
@@ -238,10 +239,12 @@ private[hive] case class HiveGenericUDTF(
   @transient
   protected lazy val collector = new UDTFCollector
 
-  override lazy val elementSchema = StructType(outputInspector.getAllStructFieldRefs.asScala.map {
-    field => StructField(field.getFieldName, inspectorToDataType(field.getFieldObjectInspector),
-      nullable = true)
-  }.toArray)
+  override lazy val elementSchema = resolvedElementSchema.getOrElse {
+    StructType(outputInspector.getAllStructFieldRefs.asScala.map {
+      field => StructField(field.getFieldName, inspectorToDataType(field.getFieldObjectInspector),
+        nullable = true)
+    }.toArray)
+  }
 
   @transient
   private lazy val inputDataTypes: Array[DataType] = children.map(_.dataType).toArray
@@ -291,7 +294,7 @@ private[hive] case class HiveGenericUDTF(
   override def prettyName: String = name
 
   override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
-    copy(children = newChildren)
+    copy(children = newChildren, resolvedElementSchema = Some(elementSchema))
 }
 
 /**
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 91edad20e5cf9..33e525ed9c6e7 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -30,7 +30,7 @@ import org.apache.hadoop.io.LongWritable
 import org.apache.spark.{SparkFunSuite, SparkRuntimeException}
 import org.apache.spark.sql.{AnalysisException, Row, TestUserClassUDT}
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.{Literal, SpecificInternalRow}
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -356,6 +356,16 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
         wrap(input, inspector, dataType)).asInstanceOf[InternalRow]
       assert(result.getArray(0).getUTF8String(0) === UTF8String.fromString("a   "))
       assert(result.getMap(1).valueArray().getUTF8String(0) === UTF8String.fromString("value"))
+
+      val outerType = StructType(Seq(StructField("nested", dataType)))
+      val outerInspector = toInspector(outerType).asInstanceOf[StructObjectInspector]
+      val field = outerInspector.getAllStructFieldRefs.get(0)
+      val targetRow = new SpecificInternalRow(Seq(dataType))
+      unwrapperFor(field, dataType)(wrap(input, inspector, dataType), targetRow, 0)
+      val nestedResult = targetRow.getStruct(0, dataType.length)
+      assert(nestedResult.getArray(0).getUTF8String(0) === UTF8String.fromString("a   "))
+      assert(
+        nestedResult.getMap(1).valueArray().getUTF8String(0) === UTF8String.fromString("value"))
     }
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index b7b6687131080..b4180055d32d0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -943,19 +943,31 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
   }
 
   test("SPARK-59277: persisted views use their captured Hive conversion types") {
-    withUserDefinedFunction("hive_upper" -> false, "hive_max" -> false) {
-      withView("first_class_hive_view", "legacy_hive_view") {
+    withUserDefinedFunction(
+        "hive_upper" -> false,
+        "hive_max" -> false,
+        "hive_explode" -> false) {
+      withView("first_class_hive_view", "first_class_hive_udtf_view", "legacy_hive_view") {
         sql(s"CREATE FUNCTION hive_upper AS '${classOf[GenericUDFUpper].getName}'")
         sql(s"CREATE FUNCTION hive_max AS '${classOf[GenericUDAFMax].getName}'")
+        sql(s"CREATE FUNCTION hive_explode AS '${classOf[GenericUDTFExplode].getName}'")
         val query =
           """SELECT
             |  hive_upper(CAST('Ab' AS CHAR(5))) AS scalar_value,
             |  hive_max(CAST('cd' AS CHAR(4))) AS aggregate_value""".stripMargin
+        val udtfQuery =
+          """SELECT value
+            |FROM (
+            |  SELECT array(CAST('xy' AS CHAR(5))) AS values
+            |) input
+            |LATERAL VIEW hive_explode(values) exploded AS value
+            |""".stripMargin
 
         withSQLConf(
             SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true",
             SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") {
           sql(s"CREATE VIEW first_class_hive_view AS $query")
+          sql(s"CREATE VIEW first_class_hive_udtf_view AS $udtfQuery")
         }
         withSQLConf(
             SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "false",
@@ -963,6 +975,10 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
           val result = sql("SELECT * FROM first_class_hive_view")
           assert(result.schema.map(_.dataType) === Seq(StringType, StringType))
           checkAnswer(result, Row("AB   ", "cd  "))
+
+          val udtfResult = sql("SELECT * FROM first_class_hive_udtf_view")
+          assert(udtfResult.schema.head.dataType === StringType)
+          checkAnswer(udtfResult, Row("xy   "))
         }
 
         withSQLConf(

From 3b0a50408fb31aed8befcd110e782bbe08bf2e59 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Wed, 16 Sep 2026 15:35:43 +0000
Subject: [PATCH 06/22] fix: [SPARK-59277] cover Hive CHAR conversion review
 gaps

---
 .../spark/sql/hive/HiveInspectors.scala       |  4 +-
 .../spark/sql/hive/HiveInspectorSuite.scala   |  6 ++
 .../sql/hive/execution/HiveUDAFSuite.scala    | 85 ++++++++++++++++++-
 3 files changed, 91 insertions(+), 4 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 18f4b210769c8..b5ec9d061087c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -954,7 +954,9 @@ private[hive] trait HiveInspectors {
     }
 
   /**
-   * Builds an in-place unwrapper that also honors target-type-specific conversions.
+   * Builds an in-place unwrapper that honors CHAR/VARCHAR conversion when `dataType`
+   * contains those types. Other targets, including nanosecond timestamps, still use
+   * `unwrapperFor(field)`.
    */
   def unwrapperFor(
       field: HiveStructField,
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 33e525ed9c6e7..70fdf41b6869e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -382,6 +382,12 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
         }
         assert(unwrapperFor(inspector, dataType)(
           inspector.asInstanceOf[ConstantObjectInspector].getWritableConstantValue) === expected)
+
+        val nullInspector = toInspector(Literal.create(null, dataType))
+        assert(nullInspector.isInstanceOf[ConstantObjectInspector])
+        assert(inspectorToDataType(nullInspector) === dataType)
+        assert(unwrapperFor(nullInspector, dataType)(
+          nullInspector.asInstanceOf[ConstantObjectInspector].getWritableConstantValue) === null)
       }
     }
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
index a84b1d806cc62..cb93e53f47dcc 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
@@ -19,13 +19,14 @@ package org.apache.spark.sql.hive.execution
 
 import scala.jdk.CollectionConverters._
 
+import org.apache.hadoop.hive.common.`type`.HiveChar
 import org.apache.hadoop.hive.ql.udf.UDAFPercentile
 import org.apache.hadoop.hive.ql.udf.generic.{AbstractGenericUDAFResolver, GenericUDAFEvaluator, GenericUDAFMax}
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.{AggregationBuffer, Mode}
 import org.apache.hadoop.hive.ql.util.JavaDataModel
-import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectInspectorFactory}
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo
+import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectInspectorFactory, PrimitiveObjectInspector}
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.{PrimitiveObjectInspectorFactory, PrimitiveObjectInspectorUtils}
+import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, TypeInfo}
 import test.org.apache.spark.sql.MyDoubleAvg
 
 import org.apache.spark.SPARK_DOC_ROOT
@@ -220,6 +221,22 @@ class HiveUDAFSuite extends QueryTest
     }
   }
 
+  test("SPARK-59277: Hive UDAF partial buffer type can differ from the final result") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      withUserDefinedFunction("char_max" -> true) {
+        sql(
+          s"CREATE TEMPORARY FUNCTION char_max AS " +
+            s"'${classOf[MockPartialStringFinalCharUDAF].getName}'")
+        withTempView("cv_udaf") {
+          Seq("abc", "def").toDF("value").repartition(2).createOrReplaceTempView("cv_udaf")
+          val aggregate = sql("SELECT char_max(CAST(value AS VARCHAR(7))) FROM cv_udaf")
+          assert(aggregate.schema.head.dataType === CharType(5))
+          checkAnswer(aggregate, Row("def  "))
+        }
+      }
+    }
+  }
+
   test("non-deterministic children expressions of UDAF") {
     withTempView("view1") {
       spark.range(1).selectExpr("id as x", "id as y").createTempView("view1")
@@ -425,3 +442,65 @@ class MockUDAFEvaluator2 extends GenericUDAFEvaluator {
     Array[Object](buffer.nonNullCount: java.lang.Long, buffer.nullCount: java.lang.Long)
   }
 }
+
+// PARTIAL1/PARTIAL2 expose a STRING inspector; FINAL/COMPLETE expose CHAR(5). This keeps the
+// (partial, final) Catalyst type pair distinct so shuffle serde cannot silently use the result
+// type for the aggregation buffer.
+class MockPartialStringFinalCharUDAF extends AbstractGenericUDAFResolver {
+  override def getEvaluator(info: Array[TypeInfo]): GenericUDAFEvaluator =
+    new MockPartialStringFinalCharEvaluator
+}
+
+class MockPartialStringFinalCharBuffer(var max: String)
+    extends GenericUDAFEvaluator.AbstractAggregationBuffer {
+  override def estimate(): Int = 16
+}
+
+class MockPartialStringFinalCharEvaluator extends GenericUDAFEvaluator {
+  private var inputOI: PrimitiveObjectInspector = _
+  private val partialOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector
+  private val finalOI =
+    PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(new CharTypeInfo(5))
+
+  override def init(mode: Mode, parameters: Array[ObjectInspector]): ObjectInspector = {
+    if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
+      inputOI = parameters.head.asInstanceOf[PrimitiveObjectInspector]
+    }
+    if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) partialOI else finalOI
+  }
+
+  override def getNewAggregationBuffer: AggregationBuffer =
+    new MockPartialStringFinalCharBuffer(null)
+
+  override def reset(agg: AggregationBuffer): Unit = {
+    agg.asInstanceOf[MockPartialStringFinalCharBuffer].max = null
+  }
+
+  override def iterate(agg: AggregationBuffer, parameters: Array[AnyRef]): Unit = {
+    if (parameters.head != null) {
+      val value = PrimitiveObjectInspectorUtils.getString(parameters.head, inputOI)
+      val buffer = agg.asInstanceOf[MockPartialStringFinalCharBuffer]
+      if (buffer.max == null || value > buffer.max) {
+        buffer.max = value
+      }
+    }
+  }
+
+  override def merge(agg: AggregationBuffer, partial: Object): Unit = {
+    if (partial != null) {
+      val value = partial.asInstanceOf[String]
+      val buffer = agg.asInstanceOf[MockPartialStringFinalCharBuffer]
+      if (buffer.max == null || value > buffer.max) {
+        buffer.max = value
+      }
+    }
+  }
+
+  override def terminatePartial(agg: AggregationBuffer): AnyRef =
+    agg.asInstanceOf[MockPartialStringFinalCharBuffer].max
+
+  override def terminate(agg: AggregationBuffer): AnyRef = {
+    val max = agg.asInstanceOf[MockPartialStringFinalCharBuffer].max
+    if (max == null) null else new HiveChar(max, 5)
+  }
+}

From 89619956e90ed2ce631a8784816fdeabce4a8c0c Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Wed, 16 Sep 2026 16:10:24 +0000
Subject: [PATCH 07/22] test: [SPARK-59277] cover Hive inspector edge cases

---
 .../spark/sql/hive/HiveInspectorSuite.scala   | 46 ++++++++++++++++++-
 .../sql/hive/execution/HiveUDAFSuite.scala    |  8 ++--
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 70fdf41b6869e..4bd040705a4e5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.hive
 import java.util
 
 import org.apache.hadoop.hive.ql.udf.UDAFPercentile
-import org.apache.hadoop.hive.serde2.io.DoubleWritable
+import org.apache.hadoop.hive.serde2.io.{DoubleWritable, HiveCharWritable, HiveVarcharWritable}
 import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, ObjectInspector, ObjectInspectorFactory, PrimitiveObjectInspector, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
@@ -339,6 +339,33 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
     }
   }
 
+  test("SPARK-59277: writable CHAR/VARCHAR inspectors round-trip values and nulls") {
+    withFirstClassCharVarchar(enabled = true) {
+      val charType = CharType(5)
+      val charInspector = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+        new CharTypeInfo(charType.length))
+      val charValue = UTF8String.fromString("abc")
+      val wrappedChar = wrap(charValue, charInspector, charType)
+      assert(wrappedChar.isInstanceOf[HiveCharWritable])
+      assert(wrappedChar.asInstanceOf[HiveCharWritable].getHiveChar.getPaddedValue === "abc  ")
+      assert(unwrapperFor(charInspector, charType)(wrappedChar) ===
+        UTF8String.fromString("abc  "))
+      assert(wrap(null, charInspector, charType) === null)
+      assert(unwrapperFor(charInspector, charType)(null) === null)
+
+      val varcharType = VarcharType(7)
+      val varcharInspector = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+        new VarcharTypeInfo(varcharType.length))
+      val varcharValue = UTF8String.fromString("abc")
+      val wrappedVarchar = wrap(varcharValue, varcharInspector, varcharType)
+      assert(wrappedVarchar.isInstanceOf[HiveVarcharWritable])
+      assert(wrappedVarchar.asInstanceOf[HiveVarcharWritable].getHiveVarchar.getValue === "abc")
+      assert(unwrapperFor(varcharInspector, varcharType)(wrappedVarchar) === varcharValue)
+      assert(wrap(null, varcharInspector, varcharType) === null)
+      assert(unwrapperFor(varcharInspector, varcharType)(null) === null)
+    }
+  }
+
   test("SPARK-59277: Hive object inspectors support nested CHAR/VARCHAR") {
     withFirstClassCharVarchar(enabled = true) {
       val dataType = StructType(Seq(
@@ -412,6 +439,23 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
         parameters = Map("limit" -> "3"))
 
       val char = CharType(5)
+      val charInspector = toInspector(char)
+      checkError(
+        exception = intercept[SparkRuntimeException] {
+          wrap(UTF8String.fromString("abcdef"), charInspector, char)
+        },
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "5"))
+
+      val varcharUnwrapper =
+        unwrapperFor(PrimitiveObjectInspectorFactory.javaStringObjectInspector, varchar)
+      checkError(
+        exception = intercept[SparkRuntimeException] {
+          varcharUnwrapper("abcd")
+        },
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "3"))
+
       val charUnwrapper =
         unwrapperFor(PrimitiveObjectInspectorFactory.javaStringObjectInspector, char)
       assert(charUnwrapper("ab") === UTF8String.fromString("ab   "))
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
index cb93e53f47dcc..2a5d072039fd3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
@@ -443,9 +443,11 @@ class MockUDAFEvaluator2 extends GenericUDAFEvaluator {
   }
 }
 
-// PARTIAL1/PARTIAL2 expose a STRING inspector; FINAL/COMPLETE expose CHAR(5). This keeps the
-// (partial, final) Catalyst type pair distinct so shuffle serde cannot silently use the result
-// type for the aggregation buffer.
+/**
+ * PARTIAL1/PARTIAL2 expose a STRING inspector; FINAL/COMPLETE expose CHAR(5). This keeps the
+ * (partial, final) Catalyst type pair distinct so shuffle serde cannot silently use the result
+ * type for the aggregation buffer.
+ */
 class MockPartialStringFinalCharUDAF extends AbstractGenericUDAFResolver {
   override def getEvaluator(info: Array[TypeInfo]): GenericUDAFEvaluator =
     new MockPartialStringFinalCharEvaluator

From e2396060bf44ef966ab648a43e91bb12cce8aadc Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Wed, 16 Sep 2026 16:43:24 +0000
Subject: [PATCH 08/22] refactor: [SPARK-59277] simplify Hive CHAR conversion

---
 .../spark/sql/hive/HiveInspectors.scala       | 106 +++++++++---------
 .../spark/sql/hive/hiveUDFEvaluators.scala    |   1 +
 .../org/apache/spark/sql/hive/hiveUDFs.scala  |   5 +-
 .../spark/sql/hive/HiveInspectorSuite.scala   |  12 ++
 .../sql/hive/execution/HiveUDFSuite.scala     |   5 +-
 5 files changed, 75 insertions(+), 54 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index b5ec9d061087c..2dfac7990ddbc 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -266,6 +266,39 @@ private[hive] trait HiveInspectors {
     input => if (input == null) null else f(input)
   }
 
+  private def charVarcharWrapper(
+      targetLength: Option[Int],
+      inspectorLength: Int,
+      preferWritable: Boolean,
+      writeSideCheck: (UTF8String, Int) => UTF8String,
+      toJava: (String, Int) => Any,
+      toWritable: (String, Int) => Any): Any => Any = {
+    if (preferWritable) {
+      targetLength match {
+        case Some(length) =>
+          withNullSafe { value =>
+            val checked = writeSideCheck(value.asInstanceOf[UTF8String], length)
+            toWritable(checked.toString, inspectorLength)
+          }
+        case None =>
+          withNullSafe(value => getStringWritable(value))
+      }
+    } else {
+      targetLength match {
+        case Some(length) =>
+          withNullSafe { value =>
+            val checked = writeSideCheck(value.asInstanceOf[UTF8String], length)
+            toJava(checked.toString, inspectorLength)
+          }
+        case None =>
+          withNullSafe { value =>
+            val string = value.asInstanceOf[UTF8String].toString
+            toJava(string, string.length)
+          }
+      }
+    }
+  }
+
   /**
    * Wraps with Hive types based on object inspector.
    */
@@ -309,60 +342,30 @@ private[hive] trait HiveInspectors {
         withNullSafe(o => getByteWritable(o))
       case _: ByteObjectInspector =>
         withNullSafe(o => o.asInstanceOf[java.lang.Byte])
-      case hvoi: HiveVarcharObjectInspector if x.preferWritable() =>
-        val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
-        dataType match {
-          case v: VarcharType =>
-            withNullSafe { o =>
-              val checked = CharVarcharCodegenUtils.varcharTypeWriteSideCheck(
-                o.asInstanceOf[UTF8String], v.length)
-              new hiveIo.HiveVarcharWritable(new HiveVarchar(checked.toString, length))
-            }
-          case _ =>
-            withNullSafe(o => getStringWritable(o))
-        }
       case hvoi: HiveVarcharObjectInspector =>
         val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
-        dataType match {
-          case v: VarcharType =>
-            withNullSafe { o =>
-              val checked = CharVarcharCodegenUtils.varcharTypeWriteSideCheck(
-                o.asInstanceOf[UTF8String], v.length)
-              new HiveVarchar(checked.toString, length)
-            }
-          case _ =>
-            withNullSafe { o =>
-              val value = o.asInstanceOf[UTF8String].toString
-              new HiveVarchar(value, value.length)
-            }
-        }
-      case hcoi: HiveCharObjectInspector if x.preferWritable() =>
-        val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
-        dataType match {
-          case c: CharType =>
-            withNullSafe { o =>
-              val checked = CharVarcharCodegenUtils.charTypeWriteSideCheck(
-                o.asInstanceOf[UTF8String], c.length)
-              new hiveIo.HiveCharWritable(new HiveChar(checked.toString, length))
-            }
-          case _ =>
-            withNullSafe(o => getStringWritable(o))
-        }
+        charVarcharWrapper(
+          dataType match {
+            case v: VarcharType => Some(v.length)
+            case _ => None
+          },
+          length,
+          hvoi.preferWritable(),
+          CharVarcharCodegenUtils.varcharTypeWriteSideCheck,
+          (value, size) => new HiveVarchar(value, size),
+          (value, size) => new hiveIo.HiveVarcharWritable(new HiveVarchar(value, size)))
       case hcoi: HiveCharObjectInspector =>
         val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
-        dataType match {
-          case c: CharType =>
-            withNullSafe { o =>
-              val checked = CharVarcharCodegenUtils.charTypeWriteSideCheck(
-                o.asInstanceOf[UTF8String], c.length)
-              new HiveChar(checked.toString, length)
-            }
-          case _ =>
-            withNullSafe { o =>
-              val value = o.asInstanceOf[UTF8String].toString
-              new HiveChar(value, value.length)
-            }
-        }
+        charVarcharWrapper(
+          dataType match {
+            case c: CharType => Some(c.length)
+            case _ => None
+          },
+          length,
+          hcoi.preferWritable(),
+          CharVarcharCodegenUtils.charTypeWriteSideCheck,
+          (value, size) => new HiveChar(value, size),
+          (value, size) => new hiveIo.HiveCharWritable(new HiveChar(value, size)))
       case _: JavaHiveDecimalObjectInspector =>
         withNullSafe(o =>
           HiveDecimal.create(o.asInstanceOf[Decimal].toJavaBigDecimal))
@@ -618,7 +621,8 @@ private[hive] trait HiveInspectors {
         val constant = ym.getWritableConstantValue.asInstanceOf[HiveIntervalYearMonth]
         _ => constant.getTotalMonths
       case pi: PrimitiveObjectInspector => pi match {
-        // We think HiveVarchar/HiveChar is also a String
+        // Untyped conversion extracts the raw value. The typed overload applies CHAR/VARCHAR
+        // padding and overflow checks.
         case hvoi: HiveVarcharObjectInspector if hvoi.preferWritable() =>
           data: Any => {
             if (data != null) {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
index bafec1575f047..585f41a0301b0 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
@@ -163,6 +163,7 @@ class HiveGenericUDFEvaluator(
     case (inspect, child) => new DeferredObjectAdapter(inspect, child.dataType)
   }
 
+  @transient
   private lazy val catalystReturnType =
     resolvedReturnType.getOrElse(inspectorToDataType(returnInspector))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 6d93ad39f6824..396cddbbd63aa 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -135,8 +135,9 @@ private[hive] case class HiveGenericUDF(
 
   override lazy val dataType: DataType = resolvedDataType.getOrElse(evaluator.returnType)
 
-  // The evaluator carries the return type resolved during analysis. Its mutable Hive state is
-  // transient and is rebuilt on the executor, but the resolved Catalyst type must not be.
+  // resolvedDataType is the stable Catalyst return type captured during analysis. The evaluator
+  // contains only runtime Hive state and is rebuilt from that type after serialization.
+  @transient
   private lazy val evaluator =
     new HiveGenericUDFEvaluator(funcWrapper, children, resolvedDataType)
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 4bd040705a4e5..2e22dced8d235 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -352,6 +352,12 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
         UTF8String.fromString("abc  "))
       assert(wrap(null, charInspector, charType) === null)
       assert(unwrapperFor(charInspector, charType)(null) === null)
+      checkError(
+        exception = intercept[SparkRuntimeException] {
+          wrap(UTF8String.fromString("abcdef"), charInspector, charType)
+        },
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "5"))
 
       val varcharType = VarcharType(7)
       val varcharInspector = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
@@ -363,6 +369,12 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
       assert(unwrapperFor(varcharInspector, varcharType)(wrappedVarchar) === varcharValue)
       assert(wrap(null, varcharInspector, varcharType) === null)
       assert(unwrapperFor(varcharInspector, varcharType)(null) === null)
+      checkError(
+        exception = intercept[SparkRuntimeException] {
+          wrap(UTF8String.fromString("abcdefgh"), varcharInspector, varcharType)
+        },
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "7"))
     }
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index b4180055d32d0..9a9c8967808c9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -971,7 +971,10 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
         }
         withSQLConf(
             SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "false",
-            SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "false") {
+            SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "false",
+            SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
+            SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString) {
+          // Interpreted task execution rebuilds the transient evaluator from the analyzed type.
           val result = sql("SELECT * FROM first_class_hive_view")
           assert(result.schema.map(_.dataType) === Seq(StringType, StringType))
           checkAnswer(result, Row("AB   ", "cd  "))

From 484bd6be6c7de415cffa3d04d6b8f6584b3a869e Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Wed, 16 Sep 2026 17:36:44 +0000
Subject: [PATCH 09/22] fix: [SPARK-59277] snapshot HiveGenericUDF CHAR result
 types

Serialize a concrete Catalyst type for runtime evaluation so executor
Hive UDF conversion does not re-infer CHAR/VARCHAR from SQLConf.
---
 .../spark/sql/hive/HiveInspectors.scala       |  80 ++++++-------
 .../spark/sql/hive/hiveUDFEvaluators.scala    |  65 ++++++----
 .../org/apache/spark/sql/hive/hiveUDFs.scala  |  25 +++-
 .../spark/sql/hive/HiveInspectorSuite.scala   |  21 +++-
 .../sql/hive/execution/HiveUDFSuite.scala     | 112 +++++++++++++++++-
 5 files changed, 227 insertions(+), 76 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 2dfac7990ddbc..3d2cbe3775a32 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -266,39 +266,6 @@ private[hive] trait HiveInspectors {
     input => if (input == null) null else f(input)
   }
 
-  private def charVarcharWrapper(
-      targetLength: Option[Int],
-      inspectorLength: Int,
-      preferWritable: Boolean,
-      writeSideCheck: (UTF8String, Int) => UTF8String,
-      toJava: (String, Int) => Any,
-      toWritable: (String, Int) => Any): Any => Any = {
-    if (preferWritable) {
-      targetLength match {
-        case Some(length) =>
-          withNullSafe { value =>
-            val checked = writeSideCheck(value.asInstanceOf[UTF8String], length)
-            toWritable(checked.toString, inspectorLength)
-          }
-        case None =>
-          withNullSafe(value => getStringWritable(value))
-      }
-    } else {
-      targetLength match {
-        case Some(length) =>
-          withNullSafe { value =>
-            val checked = writeSideCheck(value.asInstanceOf[UTF8String], length)
-            toJava(checked.toString, inspectorLength)
-          }
-        case None =>
-          withNullSafe { value =>
-            val string = value.asInstanceOf[UTF8String].toString
-            toJava(string, string.length)
-          }
-      }
-    }
-  }
-
   /**
    * Wraps with Hive types based on object inspector.
    */
@@ -528,6 +495,39 @@ private[hive] trait HiveInspectors {
       identity[Any]
   }
 
+  private def charVarcharWrapper(
+      targetLength: Option[Int],
+      inspectorLength: Int,
+      preferWritable: Boolean,
+      writeSideCheck: (UTF8String, Int) => UTF8String,
+      toJava: (String, Int) => Any,
+      toWritable: (String, Int) => Any): Any => Any = {
+    if (preferWritable) {
+      targetLength match {
+        case Some(length) =>
+          withNullSafe { value =>
+            val checked = writeSideCheck(value.asInstanceOf[UTF8String], length)
+            toWritable(checked.toString, inspectorLength)
+          }
+        case None =>
+          withNullSafe(value => getStringWritable(value))
+      }
+    } else {
+      targetLength match {
+        case Some(length) =>
+          withNullSafe { value =>
+            val checked = writeSideCheck(value.asInstanceOf[UTF8String], length)
+            toJava(checked.toString, inspectorLength)
+          }
+        case None =>
+          withNullSafe { value =>
+            val string = value.asInstanceOf[UTF8String].toString
+            toJava(string, string.length)
+          }
+      }
+    }
+  }
+
   /**
    * Builds unwrappers ahead of time according to object inspector
    * types to avoid pattern matching and branching costs per row.
@@ -958,18 +958,14 @@ private[hive] trait HiveInspectors {
     }
 
   /**
-   * Builds an in-place unwrapper that honors CHAR/VARCHAR conversion when `dataType`
-   * contains those types. Other targets, including nanosecond timestamps, still use
-   * `unwrapperFor(field)`.
+   * Builds an in-place unwrapper using the target Catalyst `dataType`. This preserves all
+   * target-aware conversion, including CHAR/VARCHAR checks and nanosecond timestamp precision.
    */
   def unwrapperFor(
       field: HiveStructField,
-      dataType: DataType): (Any, InternalRow, Int) => Unit = dataType match {
-    case dt if CharVarcharUtils.hasCharVarchar(dt) =>
-      val unwrapper = unwrapperFor(field.getFieldObjectInspector, dataType)
-      (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
-    case _ =>
-      unwrapperFor(field)
+      dataType: DataType): (Any, InternalRow, Int) => Unit = {
+    val unwrapper = unwrapperFor(field.getFieldObjectInspector, dataType)
+    (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
   }
 
   def wrap(a: Any, oi: ObjectInspector, dataType: DataType): AnyRef = {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
index 585f41a0301b0..853476704bee2 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
@@ -111,26 +111,24 @@ class HiveSimpleUDFEvaluator(
   }
 }
 
-class HiveGenericUDFEvaluator(
-    funcWrapper: HiveFunctionWrapper,
-    children: Seq[Expression],
-    resolvedReturnType: Option[DataType])
-  extends HiveUDFEvaluatorBase[GenericUDF](funcWrapper, children) {
-
-  // SPARK-58792: copied expression nodes (e.g. via withNewChildrenInternal) share one
-  // HiveFunctionWrapper, whose cached GenericUDF instance is mutable: initialize()
-  // rewrites its converters and output holders based on the arguments of whichever
-  // copy initialized it last. Give every evaluator its own clone so copied nodes
-  // cannot corrupt each other.
-  @transient
-  override lazy val function: GenericUDF =
-    HiveFunctionRegistryUtils.cloneGenericUDF(funcWrapper.createFunction[GenericUDF]())
-
-  @transient
-  private lazy val argumentInspectors = children.map(toInspector).toArray
+private[hive] object HiveGenericUDFEvaluator extends HiveInspectors {
+
+  def inferReturnType(
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression]): DataType = {
+    val function =
+      HiveFunctionRegistryUtils.cloneGenericUDF(funcWrapper.createFunction[GenericUDF]())
+    val argumentInspectors = children.map(toInspector).toArray
+    val udfType = function.getClass.getAnnotation(classOf[HiveUDFType])
+    val isDeterministic =
+      udfType != null && udfType.deterministic() && !udfType.stateful()
+    inspectorToDataType(initialize(function, argumentInspectors, isDeterministic))
+  }
 
-  @transient
-  lazy val returnInspector = {
+  def initialize(
+      function: GenericUDF,
+      argumentInspectors: Array[ObjectInspector],
+      isDeterministic: Boolean): ObjectInspector = {
     // Inline o.a.h.hive.ql.udf.generic.GenericUDF#initializeAndFoldConstants, but
     // eliminate calls o.a.h.hive.ql.exec.FunctionRegistry to avoid initializing Hive
     // built-in UDFs.
@@ -140,7 +138,7 @@ class HiveGenericUDFEvaluator(
     if (function.getRequiredFiles == null && function.getRequiredJars == null &&
       argumentInspectors.forall(ObjectInspectorUtils.isConstantObjectInspector) &&
       !ObjectInspectorUtils.isConstantObjectInspector(oi) &&
-      isUDFDeterministic &&
+      isDeterministic &&
       ObjectInspectorUtils.supportsConstantObjectInspector(oi)) {
       val argumentValues: Array[DeferredObject] = argumentInspectors.map { argumentInspector =>
         new GenericUDF.DeferredJavaObject(
@@ -157,16 +155,35 @@ class HiveGenericUDFEvaluator(
       oi
     }
   }
+}
+
+private[hive] class HiveGenericUDFEvaluator(
+    funcWrapper: HiveFunctionWrapper,
+    children: Seq[Expression],
+    catalystReturnType: DataType)
+  extends HiveUDFEvaluatorBase[GenericUDF](funcWrapper, children) {
+
+  // SPARK-58792: copied expression nodes (e.g. via withNewChildrenInternal) share one
+  // HiveFunctionWrapper, whose cached GenericUDF instance is mutable: initialize()
+  // rewrites its converters and output holders based on the arguments of whichever
+  // copy initialized it last. Give every evaluator its own clone so copied nodes
+  // cannot corrupt each other.
+  @transient
+  override lazy val function: GenericUDF =
+    HiveFunctionRegistryUtils.cloneGenericUDF(funcWrapper.createFunction[GenericUDF]())
+
+  @transient
+  private lazy val argumentInspectors = children.map(toInspector).toArray
+
+  @transient
+  lazy val returnInspector =
+    HiveGenericUDFEvaluator.initialize(function, argumentInspectors, isUDFDeterministic)
 
   @transient
   private lazy val deferredObjects: Array[DeferredObject] = argumentInspectors.zip(children).map {
     case (inspect, child) => new DeferredObjectAdapter(inspect, child.dataType)
   }
 
-  @transient
-  private lazy val catalystReturnType =
-    resolvedReturnType.getOrElse(inspectorToDataType(returnInspector))
-
   @transient
   private lazy val unwrapper: Any => Any = unwrapperFor(returnInspector, catalystReturnType)
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 396cddbbd63aa..407b745c11222 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -113,6 +113,16 @@ private[hive] case class HiveSimpleUDF(
   }
 }
 
+/**
+ * A Hive GenericUDF expression whose Catalyst return type is fixed during analysis.
+ *
+ * @param resolvedDataType the return type already resolved by an earlier copy of this expression.
+ *   This is load-bearing runtime state for first-class types. For example, if
+ *   `hive_udf(CAST('a' AS CHAR(3)))` resolves to `CHAR(3)`, executor-side Hive conversion must
+ *   keep that type even when the executor's SQLConf has first-class types disabled. When absent
+ *   on an initial expression, analysis materializes `dataType`; subsequent copies retain the
+ *   concrete type here.
+ */
 private[hive] case class HiveGenericUDF(
     name: String,
     funcWrapper: HiveFunctionWrapper,
@@ -133,13 +143,18 @@ private[hive] case class HiveGenericUDF(
   override def foldable: Boolean = evaluator.isUDFDeterministic &&
     evaluator.returnInspector.isInstanceOf[ConstantObjectInspector]
 
-  override lazy val dataType: DataType = resolvedDataType.getOrElse(evaluator.returnType)
+  // This non-transient lazy val is materialized when analysis asks for dataType and is serialized
+  // with the expression. Type inference uses separate driver-side Hive state so the transient
+  // runtime evaluator can only be constructed from one concrete Catalyst type.
+  private lazy val catalystDataType: DataType = resolvedDataType.getOrElse {
+    HiveGenericUDFEvaluator.inferReturnType(funcWrapper, children)
+  }
+
+  override lazy val dataType: DataType = catalystDataType
 
-  // resolvedDataType is the stable Catalyst return type captured during analysis. The evaluator
-  // contains only runtime Hive state and is rebuilt from that type after serialization.
   @transient
   private lazy val evaluator =
-    new HiveGenericUDFEvaluator(funcWrapper, children, resolvedDataType)
+    new HiveGenericUDFEvaluator(funcWrapper, children, catalystDataType)
 
   override def eval(input: InternalRow): Any = {
     children.zipWithIndex.foreach {
@@ -161,7 +176,7 @@ private[hive] case class HiveGenericUDF(
   }
 
   override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
-    copy(children = newChildren, resolvedDataType = Some(dataType))
+    copy(children = newChildren, resolvedDataType = Some(catalystDataType))
 
   protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val refEvaluator = ctx.addReferenceObj("evaluator", evaluator)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 2e22dced8d235..d32f1c612a50e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.hive
 
+import java.sql.Timestamp
 import java.util
 
 import org.apache.hadoop.hive.ql.udf.UDAFPercentile
@@ -31,7 +32,7 @@ import org.apache.spark.{SparkFunSuite, SparkRuntimeException}
 import org.apache.spark.sql.{AnalysisException, Row, TestUserClassUDT}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Literal, SpecificInternalRow}
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeUtils, GenericArrayData, MapData}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -408,6 +409,24 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
     }
   }
 
+  test("SPARK-59277: typed field unwrappers preserve nanosecond timestamps") {
+    val value = Timestamp.valueOf("2026-09-16 12:34:56.123456789")
+    Seq(
+      TimestampNTZNanosType(9) ->
+        DateTimeUtils.localDateTimeToTimestampNanos(value.toLocalDateTime, 9),
+      TimestampLTZNanosType(9) ->
+        DateTimeUtils.instantToTimestampNanos(value.toInstant, 9)).foreach {
+      case (dataType, expected) =>
+        val inspector = ObjectInspectorFactory.getStandardStructObjectInspector(
+          util.Arrays.asList("value"),
+          util.Arrays.asList(PrimitiveObjectInspectorFactory.javaTimestampObjectInspector))
+        val field = inspector.getAllStructFieldRefs.get(0)
+        val targetRow = new SpecificInternalRow(Seq(dataType))
+        unwrapperFor(field, dataType)(value, targetRow, 0)
+        assert(targetRow.get(0, dataType) === expected)
+    }
+  }
+
   test("SPARK-59277: Hive constant inspectors preserve CHAR/VARCHAR type information") {
     withFirstClassCharVarchar(enabled = true) {
       Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 9a9c8967808c9..7b617e1f20ff3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -17,13 +17,15 @@
 
 package org.apache.spark.sql.hive.execution
 
-import java.io.{DataInput, DataOutput, File, PrintWriter}
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInput, DataOutput, File}
+import java.io.{ObjectInputStream, ObjectOutputStream, PrintWriter}
 import java.sql.{Date, Timestamp}
 import java.util.{ArrayList, Arrays, Properties}
 
 import scala.jdk.CollectionConverters._
 
 import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hive.common.`type`.HiveChar
 import org.apache.hadoop.hive.ql.exec.UDF
 import org.apache.hadoop.hive.ql.metadata.HiveException
 import org.apache.hadoop.hive.ql.udf.{UDAFPercentile, UDFType}
@@ -32,9 +34,10 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject
 import org.apache.hadoop.hive.serde2.{AbstractSerDe, SerDeStats}
 import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectInspectorFactory}
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory
 import org.apache.hadoop.io.{LongWritable, Writable}
 
-import org.apache.spark.{SparkException, SparkFiles, TestUtils}
+import org.apache.spark.{SparkException, SparkFiles, SparkRuntimeException, TestUtils}
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BindReferences, CodegenObjectFactoryMode, Literal}
@@ -46,8 +49,9 @@ import org.apache.spark.sql.hive.HiveGenericUDF
 import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
 import org.apache.spark.sql.hive.test.{TestHiveSingleton, TestUDTFJar}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{CharType, StringType, TimestampType, TimeType, VarcharType}
+import org.apache.spark.sql.types.{CharType, DataType, StringType, TimestampType, TimeType, VarcharType}
 import org.apache.spark.tags.SlowHiveTest
+import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
 
 case class Fields(f1: Int, f2: Int, f3: Int, f4: Int, f5: Int)
@@ -974,7 +978,6 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
             SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "false",
             SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
             SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString) {
-          // Interpreted task execution rebuilds the transient evaluator from the analyzed type.
           val result = sql("SELECT * FROM first_class_hive_view")
           assert(result.schema.map(_.dataType) === Seq(StringType, StringType))
           checkAnswer(result, Row("AB   ", "cd  "))
@@ -999,6 +1002,86 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
     }
   }
 
+  test("SPARK-59277: HiveGenericUDF Java serialization preserves its Catalyst type") {
+    def serialize(expression: HiveGenericUDF): Array[Byte] = {
+      val bytes = new ByteArrayOutputStream()
+      val output = new ObjectOutputStream(bytes)
+      try {
+        output.writeObject(expression)
+      } finally {
+        output.close()
+      }
+      bytes.toByteArray
+    }
+
+    def deserialize(bytes: Array[Byte]): HiveGenericUDF = {
+      val input = new ObjectInputStream(new ByteArrayInputStream(bytes))
+      try {
+        input.readObject().asInstanceOf[HiveGenericUDF]
+      } finally {
+        input.close()
+      }
+    }
+
+    val inferredBytes = withSQLConf(
+        SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true",
+        SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") {
+      val expression = HiveGenericUDF(
+        "return_char",
+        HiveFunctionWrapper(classOf[ReturnCharGenericUDF].getName),
+        Seq(Literal("ab")))
+      assert(expression.resolvedDataType.isEmpty)
+      assert(expression.dataType === CharType(5))
+      val copied = expression.withNewChildren(Seq(Literal("cd"))).asInstanceOf[HiveGenericUDF]
+      assert(copied.resolvedDataType.contains(CharType(5)))
+      serialize(expression)
+    }
+
+    withSQLConf(
+        SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "false",
+        SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "false") {
+      val expression = deserialize(inferredBytes)
+      assert(expression.resolvedDataType.isEmpty)
+      assert(expression.dataType === CharType(5))
+      assert(expression.eval(InternalRow.empty) === UTF8String.fromString("ab   "))
+    }
+
+    val (charBytes, varcharBytes) = withSQLConf(
+        SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "false",
+        SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "false") {
+      def resolvedExpression(value: String, dataType: DataType) = {
+        val expression = HiveGenericUDF(
+          "return_string",
+          HiveFunctionWrapper(classOf[ReturnStringGenericUDF].getName),
+          Seq(Literal(value)),
+          Some(dataType))
+        assert(expression.dataType === dataType)
+        serialize(expression)
+      }
+      (
+        resolvedExpression("ab", CharType(5)),
+        resolvedExpression("abcd", VarcharType(3)))
+    }
+
+    withSQLConf(
+        SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true",
+        SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "true") {
+      val charExpression = deserialize(charBytes)
+      assert(charExpression.dataType === CharType(5))
+      assert(charExpression.eval(InternalRow.empty) === UTF8String.fromString("ab   "))
+
+      val varcharExpression = deserialize(varcharBytes)
+      assert(varcharExpression.dataType === VarcharType(3))
+      val exception = intercept[SparkException] {
+        varcharExpression.eval(InternalRow.empty)
+      }
+      checkError(
+        exception = exception.getCause.asInstanceOf[SparkRuntimeException],
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "3"))
+    }
+  }
+
   test("SPARK-58792: copied HiveGenericUDF nodes must not share a mutable GenericUDF") {
     val tsAttr = AttributeReference("ts", TimestampType, nullable = false)()
     val constTs = Literal(
@@ -1132,6 +1215,27 @@ class PairUDF extends GenericUDF {
   override def getDisplayString(p1: Array[String]): String = ""
 }
 
+class ReturnCharGenericUDF extends GenericUDF {
+  override def initialize(arguments: Array[ObjectInspector]): ObjectInspector =
+    PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
+      TypeInfoFactory.getCharTypeInfo(5))
+
+  override def evaluate(arguments: Array[DeferredObject]): AnyRef =
+    new HiveChar(arguments(0).get.toString, 5)
+
+  override def getDisplayString(children: Array[String]): String = "return_char"
+}
+
+class ReturnStringGenericUDF extends GenericUDF {
+  override def initialize(arguments: Array[ObjectInspector]): ObjectInspector =
+    PrimitiveObjectInspectorFactory.javaStringObjectInspector
+
+  override def evaluate(arguments: Array[DeferredObject]): AnyRef =
+    arguments(0).get.toString
+
+  override def getDisplayString(children: Array[String]): String = "return_string"
+}
+
 @UDFType(stateful = true)
 class StatefulUDF extends UDF {
   private val result = new LongWritable(0)

From 21df1f65a33c496578cbdb804ab05b8481307714 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Wed, 16 Sep 2026 18:37:10 +0000
Subject: [PATCH 10/22] fix: [SPARK-59277] snapshot Hive types on the product
 and fail TRANSFORM overflow

Store inferred Hive UDF/UDTF/UDAF result types as required case-class fields so
Java serialization cannot drop them, and deserialize Hive TRANSFORM CHAR/VARCHAR
as STRING so first-class length checks raise EXCEED_LIMIT_LENGTH instead of truncating.
---
 .../BaseScriptTransformationExec.scala        |   8 +
 .../BaseScriptTransformationSuite.scala       |  27 +++
 .../HiveScriptTransformationExec.scala        |  23 ++-
 .../spark/sql/hive/hiveUDFEvaluators.scala    |  18 +-
 .../org/apache/spark/sql/hive/hiveUDFs.scala  | 159 ++++++++++++------
 .../sql/hive/HiveScalaReflectionSuite.scala   |   2 +-
 .../HiveScriptTransformationSuite.scala       |  24 +++
 .../sql/hive/execution/HiveUDFSuite.scala     |   6 +-
 8 files changed, 200 insertions(+), 67 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index f6685be7aab87..5e29f174a293f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -201,6 +201,14 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
   private lazy val outputFieldWriters: Seq[String => Any] = output.map { attr =>
     val converter = CatalystTypeConverters.createToCatalystConverter(attr.dataType)
     attr.dataType match {
+      case _: CharType | _: VarcharType =>
+        // First-class CHAR/VARCHAR must not use Hive LazySimpleSerde's null-on-error path.
+        (data: String) =>
+          if (data == ioschema.outputRowFormatMap("TOK_TABLEROWFORMATNULL")) {
+            null
+          } else {
+            converter(data)
+          }
       case _: StringType => wrapperConvertException(data => data, converter)
       case BooleanType => wrapperConvertException(data => data.toBoolean, converter)
       case ByteType => wrapperConvertException(data => data.toByte, converter)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index 8bd5e9855b4bf..f84dd2dda0c40 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -104,6 +104,33 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
 
+  test("SPARK-59277: TRANSFORM CHAR/VARCHAR overflow without SerDe raises EXCEED_LIMIT_LENGTH") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val input = Seq(("abcdef", "abcdefgh")).toDF("c", "v")
+      val exception = intercept[Exception] {
+        QueryTest.executePlan(
+          createScriptTransformationExec(
+            script = "cat",
+            output = Seq(
+              AttributeReference("c", CharType(4))(),
+              AttributeReference("v", VarcharType(5))()),
+            child = input.queryExecution.sparkPlan,
+            ioschema = defaultIOSchema),
+          spark.sqlContext)
+      }
+      val runtimeException = exception match {
+        case s: org.apache.spark.SparkRuntimeException => s
+        case other =>
+          other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
+      }
+      checkError(
+        exception = runtimeException,
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "4"))
+    }
+  }
+
   test("script transformation should not swallow errors from upstream operators (no serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
index ee6b9b358279a..2037f2a50458b 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
@@ -36,7 +36,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.hive.HiveInspectors
 import org.apache.spark.sql.hive.HiveShim._
-import org.apache.spark.sql.types.DataType
+import org.apache.spark.sql.types._
 import org.apache.spark.util.{CircularBuffer, Utils}
 
 /**
@@ -258,7 +258,11 @@ object HiveScriptIOSchema extends HiveInspectors {
       output: Seq[Attribute]): Option[(AbstractSerDe, StructObjectInspector)] = {
     ioschema.outputSerdeClass.map { serdeClass =>
       val (columns, columnTypes) = parseAttrs(output)
-      val serde = initSerDe(serdeClass, columns, columnTypes, ioschema.outputSerdeProps)
+      // Hive CHAR/VARCHAR SerDe truncates silently. Always deserialize as STRING
+      // (even under first-class CHAR/VARCHAR) so unwrapperFor can raise
+      // EXCEED_LIMIT_LENGTH instead of returning a truncated value.
+      val serdeTypes = columnTypes.map(toHiveSerdePhysicalType)
+      val serde = initSerDe(serdeClass, columns, serdeTypes, ioschema.outputSerdeProps)
       val structObjectInspector = serde.getObjectInspector().asInstanceOf[StructObjectInspector]
       (serde, structObjectInspector)
     }
@@ -270,6 +274,21 @@ object HiveScriptIOSchema extends HiveInspectors {
     (columns, columnTypes)
   }
 
+  /**
+   * Hive SerDe CHAR/VARCHAR types truncate on deserialize. Map them to STRING so Spark
+   * applies first-class length checks. Unlike `replaceCharVarcharWithString`, this must
+   * run even when first-class CHAR/VARCHAR is enabled.
+   */
+  private def toHiveSerdePhysicalType(dt: DataType): DataType = dt match {
+    case ArrayType(et, n) => ArrayType(toHiveSerdePhysicalType(et), n)
+    case MapType(kt, vt, n) =>
+      MapType(toHiveSerdePhysicalType(kt), toHiveSerdePhysicalType(vt), n)
+    case StructType(fields) =>
+      StructType(fields.map(f => f.copy(dataType = toHiveSerdePhysicalType(f.dataType))))
+    case _: CharType | _: VarcharType => StringType
+    case other => other
+  }
+
   def initSerDe(
       serdeClassName: String,
       columns: Seq[String],
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
index 853476704bee2..1dead6c3bf2d0 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
@@ -113,26 +113,28 @@ class HiveSimpleUDFEvaluator(
 
 private[hive] object HiveGenericUDFEvaluator extends HiveInspectors {
 
+  /**
+   * Driver-side Hive initialize for `SELECT hive_udf(...)`. Stores the Catalyst type on
+   * `HiveGenericUDF.dataType` (for example CHAR(5) from a CHAR inspector).
+   */
   def inferReturnType(
       funcWrapper: HiveFunctionWrapper,
       children: Seq[Expression]): DataType = {
     val function =
       HiveFunctionRegistryUtils.cloneGenericUDF(funcWrapper.createFunction[GenericUDF]())
-    val argumentInspectors = children.map(toInspector).toArray
-    val udfType = function.getClass.getAnnotation(classOf[HiveUDFType])
-    val isDeterministic =
-      udfType != null && udfType.deterministic() && !udfType.stateful()
-    inspectorToDataType(initialize(function, argumentInspectors, isDeterministic))
+    inspectorToDataType(initialize(function, children.map(toInspector).toArray))
   }
 
   def initialize(
       function: GenericUDF,
-      argumentInspectors: Array[ObjectInspector],
-      isDeterministic: Boolean): ObjectInspector = {
+      argumentInspectors: Array[ObjectInspector]): ObjectInspector = {
     // Inline o.a.h.hive.ql.udf.generic.GenericUDF#initializeAndFoldConstants, but
     // eliminate calls o.a.h.hive.ql.exec.FunctionRegistry to avoid initializing Hive
     // built-in UDFs.
     val oi = function.initialize(argumentInspectors)
+    val udfType = function.getClass.getAnnotation(classOf[HiveUDFType])
+    val isDeterministic =
+      udfType != null && udfType.deterministic() && !udfType.stateful()
     // If the UDF depends on any external resources, we can't fold because the
     // resources may not be available at compile time.
     if (function.getRequiredFiles == null && function.getRequiredJars == null &&
@@ -177,7 +179,7 @@ private[hive] class HiveGenericUDFEvaluator(
 
   @transient
   lazy val returnInspector =
-    HiveGenericUDFEvaluator.initialize(function, argumentInspectors, isUDFDeterministic)
+    HiveGenericUDFEvaluator.initialize(function, argumentInspectors)
 
   @transient
   private lazy val deferredObjects: Array[DeferredObject] = argumentInspectors.zip(children).map {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 407b745c11222..935647d90dac3 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -114,20 +114,16 @@ private[hive] case class HiveSimpleUDF(
 }
 
 /**
- * A Hive GenericUDF expression whose Catalyst return type is fixed during analysis.
+ * A Hive GenericUDF whose Catalyst return type is snapshotted at analysis.
  *
- * @param resolvedDataType the return type already resolved by an earlier copy of this expression.
- *   This is load-bearing runtime state for first-class types. For example, if
- *   `hive_udf(CAST('a' AS CHAR(3)))` resolves to `CHAR(3)`, executor-side Hive conversion must
- *   keep that type even when the executor's SQLConf has first-class types disabled. When absent
- *   on an initial expression, analysis materializes `dataType`; subsequent copies retain the
- *   concrete type here.
+ * Example: `SELECT hive_upper(CAST('Ab' AS CHAR(5)))` stores `CharType(5)` on `dataType` so
+ * executor conversion still pads even if first-class types are disabled there.
  */
 private[hive] case class HiveGenericUDF(
     name: String,
     funcWrapper: HiveFunctionWrapper,
     children: Seq[Expression],
-    resolvedDataType: Option[DataType] = None)
+    override val dataType: DataType)
   extends Expression
   with HiveInspectors
   with UserDefinedExpression {
@@ -143,18 +139,9 @@ private[hive] case class HiveGenericUDF(
   override def foldable: Boolean = evaluator.isUDFDeterministic &&
     evaluator.returnInspector.isInstanceOf[ConstantObjectInspector]
 
-  // This non-transient lazy val is materialized when analysis asks for dataType and is serialized
-  // with the expression. Type inference uses separate driver-side Hive state so the transient
-  // runtime evaluator can only be constructed from one concrete Catalyst type.
-  private lazy val catalystDataType: DataType = resolvedDataType.getOrElse {
-    HiveGenericUDFEvaluator.inferReturnType(funcWrapper, children)
-  }
-
-  override lazy val dataType: DataType = catalystDataType
-
   @transient
   private lazy val evaluator =
-    new HiveGenericUDFEvaluator(funcWrapper, children, catalystDataType)
+    new HiveGenericUDFEvaluator(funcWrapper, children, dataType)
 
   override def eval(input: InternalRow): Any = {
     children.zipWithIndex.foreach {
@@ -176,7 +163,7 @@ private[hive] case class HiveGenericUDF(
   }
 
   override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
-    copy(children = newChildren, resolvedDataType = Some(catalystDataType))
+    copy(children = newChildren)
 
   protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val refEvaluator = ctx.addReferenceObj("evaluator", evaluator)
@@ -214,6 +201,19 @@ private[hive] case class HiveGenericUDF(
   }
 }
 
+object HiveGenericUDF {
+  def apply(
+      name: String,
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression]): HiveGenericUDF = {
+    HiveGenericUDF(
+      name,
+      funcWrapper,
+      children,
+      HiveGenericUDFEvaluator.inferReturnType(funcWrapper, children))
+  }
+}
+
 /**
  * Converts a Hive Generic User Defined Table Generating Function (UDTF) to a
  * `Generator`. Note that the semantics of Generators do not allow
@@ -229,7 +229,7 @@ private[hive] case class HiveGenericUDTF(
     name: String,
     funcWrapper: HiveFunctionWrapper,
     children: Seq[Expression],
-    resolvedElementSchema: Option[StructType] = None)
+    override val elementSchema: StructType)
   extends Generator with HiveInspectors with CodegenFallback with UserDefinedExpression {
 
   @transient
@@ -255,13 +255,6 @@ private[hive] case class HiveGenericUDTF(
   @transient
   protected lazy val collector = new UDTFCollector
 
-  override lazy val elementSchema = resolvedElementSchema.getOrElse {
-    StructType(outputInspector.getAllStructFieldRefs.asScala.map {
-      field => StructField(field.getFieldName, inspectorToDataType(field.getFieldObjectInspector),
-        nullable = true)
-    }.toArray)
-  }
-
   @transient
   private lazy val inputDataTypes: Array[DataType] = children.map(_.dataType).toArray
 
@@ -310,7 +303,36 @@ private[hive] case class HiveGenericUDTF(
   override def prettyName: String = name
 
   override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
-    copy(children = newChildren, resolvedElementSchema = Some(elementSchema))
+    copy(children = newChildren)
+}
+
+object HiveGenericUDTF extends HiveInspectors {
+  def apply(
+      name: String,
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression]): HiveGenericUDTF = {
+    HiveGenericUDTF(name, funcWrapper, children, inferElementSchema(funcWrapper, children))
+  }
+
+  def inferElementSchema(
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression]): StructType = {
+    val function: GenericUDTF = funcWrapper.createFunction()
+    function.setCollector(new Collector {
+      override def collect(input: java.lang.Object): Unit = {}
+    })
+    val inspectors = children.map(toInspector)
+    val fields = inspectors.indices.map(index => s"_col$index").asJava
+    val inputInspector =
+      ObjectInspectorFactory.getStandardStructObjectInspector(fields, inspectors.asJava)
+    val outputInspector = function.initialize(inputInspector)
+    StructType(outputInspector.getAllStructFieldRefs.asScala.map { field =>
+      StructField(
+        field.getFieldName,
+        inspectorToDataType(field.getFieldObjectInspector),
+        nullable = true)
+    }.toArray)
+  }
 }
 
 /**
@@ -355,10 +377,11 @@ private[hive] case class HiveUDAFFunction(
     name: String,
     funcWrapper: HiveFunctionWrapper,
     children: Seq[Expression],
-    isUDAFBridgeRequired: Boolean = false,
-    mutableAggBufferOffset: Int = 0,
-    inputAggBufferOffset: Int = 0,
-    resolvedDataTypes: Option[(DataType, DataType)] = None)
+    isUDAFBridgeRequired: Boolean,
+    mutableAggBufferOffset: Int,
+    inputAggBufferOffset: Int,
+    partialResultDataType: DataType,
+    override val dataType: DataType)
   extends TypedImperativeAggregate[HiveUDAFBuffer]
   with HiveInspectors
   with UserDefinedExpression {
@@ -366,14 +389,10 @@ private[hive] case class HiveUDAFFunction(
   final override val nodePatterns: Seq[TreePattern] = Seq(USER_DEFINED_AGGREGATION)
 
   override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
-    copy(
-      mutableAggBufferOffset = newMutableAggBufferOffset,
-      resolvedDataTypes = Some(catalystDataTypes))
+    copy(mutableAggBufferOffset = newMutableAggBufferOffset)
 
   override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate =
-    copy(
-      inputAggBufferOffset = newInputAggBufferOffset,
-      resolvedDataTypes = Some(catalystDataTypes))
+    copy(inputAggBufferOffset = newInputAggBufferOffset)
 
   // Hive `ObjectInspector`s for all child expressions (input parameters of the function).
   @transient
@@ -416,20 +435,9 @@ private[hive] case class HiveUDAFFunction(
       evaluator.init(GenericUDAFEvaluator.Mode.FINAL, Array(partial1HiveEvaluator.objectInspector)))
   }
 
-  // Resolve both Catalyst types together during analysis so the partial type is serialized with
-  // the expression instead of being rebuilt from the executor's SQLConf.
-  private lazy val catalystDataTypes = resolvedDataTypes.getOrElse((
-    inspectorToDataType(partial1HiveEvaluator.objectInspector),
-    inspectorToDataType(finalHiveEvaluator.objectInspector)))
-
-  private def partialResultDataType: DataType = catalystDataTypes._1
-
-  // Wrapper functions used to wrap Spark SQL input arguments into Hive specific format.
   @transient
   private lazy val inputWrappers = children.map(x => wrapperFor(toInspector(x), x.dataType)).toArray
 
-  // Unwrapper function used to unwrap final aggregation result objects returned by Hive UDAFs into
-  // Spark SQL specific format.
   @transient
   private lazy val resultUnwrapper = unwrapperFor(finalHiveEvaluator.objectInspector, dataType)
 
@@ -441,8 +449,6 @@ private[hive] case class HiveUDAFFunction(
 
   override def nullable: Boolean = true
 
-  override lazy val dataType: DataType = catalystDataTypes._2
-
   override def prettyName: String = name
 
   override def sql(isDistinct: Boolean): String = {
@@ -584,7 +590,56 @@ private[hive] case class HiveUDAFFunction(
   }
 
   override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
-    copy(children = newChildren, resolvedDataTypes = Some(catalystDataTypes))
+    copy(children = newChildren)
+}
+
+object HiveUDAFFunction extends HiveInspectors {
+  def apply(
+      name: String,
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression]): HiveUDAFFunction = {
+    apply(name, funcWrapper, children, isUDAFBridgeRequired = false)
+  }
+
+  def apply(
+      name: String,
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression],
+      isUDAFBridgeRequired: Boolean): HiveUDAFFunction = {
+    val (partialType, resultType) =
+      inferResolvedTypes(funcWrapper, children, isUDAFBridgeRequired)
+    HiveUDAFFunction(
+      name,
+      funcWrapper,
+      children,
+      isUDAFBridgeRequired,
+      mutableAggBufferOffset = 0,
+      inputAggBufferOffset = 0,
+      partialType,
+      resultType)
+  }
+
+  def inferResolvedTypes(
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression],
+      isUDAFBridgeRequired: Boolean): (DataType, DataType) = {
+    val inputInspectors = children.map(toInspector).toArray
+    def newEvaluator(): GenericUDAFEvaluator = {
+      val resolver = if (isUDAFBridgeRequired) {
+        new SparkGenericUDAFBridge(funcWrapper.createFunction[UDAF]())
+      } else {
+        funcWrapper.createFunction[AbstractGenericUDAFResolver]()
+      }
+      val parameterInfo = new SimpleGenericUDAFParameterInfo(inputInspectors, false, false, false)
+      resolver.getEvaluator(parameterInfo)
+    }
+    val partial1 = newEvaluator()
+    val partialInspector = partial1.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputInspectors)
+    val finalEvaluator = newEvaluator()
+    val finalInspector =
+      finalEvaluator.init(GenericUDAFEvaluator.Mode.FINAL, Array(partialInspector))
+    (inspectorToDataType(partialInspector), inspectorToDataType(finalInspector))
+  }
 }
 
 case class HiveUDAFBuffer(buf: AggregationBuffer, canDoMerge: Boolean)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala
index c90d31faa52a8..b777bacfe4fee 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveScalaReflectionSuite.scala
@@ -29,7 +29,7 @@ class HiveScalaReflectionSuite extends SparkFunSuite {
 
   test("SPARK-38510: ScalaReflection.getConstructorParameterNames should work for classes with " +
     "cyclic annotation references") {
-    assert(Seq("name", "funcWrapper", "children", "resolvedDataType") ===
+    assert(Seq("name", "funcWrapper", "children", "dataType") ===
       ScalaReflection.getConstructorParameterNames(classOf[HiveGenericUDF]))
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
index 58241fd27dd08..5dde1a09094df 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -394,6 +394,30 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
     }
   }
 
+  test("SPARK-59277: TRANSFORM CHAR overflow with Hive SerDe raises EXCEED_LIMIT_LENGTH") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val exception = intercept[Exception] {
+        sql(
+          """
+            |SELECT TRANSFORM('abcdef')
+            |USING 'cat'
+            |AS (c CHAR(4))
+            |FROM VALUES (1) input(dummy)
+            |""".stripMargin).collect()
+      }
+      val runtimeException = exception match {
+        case s: org.apache.spark.SparkRuntimeException => s
+        case other =>
+          other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
+      }
+      checkError(
+        exception = runtimeException,
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "4"))
+    }
+  }
+
   test("SPARK-32400: TRANSFORM doesn't support CalendarIntervalType/UserDefinedType (hive serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
     withTempView("v") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 7b617e1f20ff3..00c60d333b401 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -1030,10 +1030,9 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
         "return_char",
         HiveFunctionWrapper(classOf[ReturnCharGenericUDF].getName),
         Seq(Literal("ab")))
-      assert(expression.resolvedDataType.isEmpty)
       assert(expression.dataType === CharType(5))
       val copied = expression.withNewChildren(Seq(Literal("cd"))).asInstanceOf[HiveGenericUDF]
-      assert(copied.resolvedDataType.contains(CharType(5)))
+      assert(copied.dataType === CharType(5))
       serialize(expression)
     }
 
@@ -1041,7 +1040,6 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
         SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "false",
         SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> "false") {
       val expression = deserialize(inferredBytes)
-      assert(expression.resolvedDataType.isEmpty)
       assert(expression.dataType === CharType(5))
       assert(expression.eval(InternalRow.empty) === UTF8String.fromString("ab   "))
     }
@@ -1054,7 +1052,7 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
           "return_string",
           HiveFunctionWrapper(classOf[ReturnStringGenericUDF].getName),
           Seq(Literal(value)),
-          Some(dataType))
+          dataType)
         assert(expression.dataType === dataType)
         serialize(expression)
       }

From 49396dd16731309554786eba9f40ba032bb986f3 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Thu, 17 Sep 2026 00:01:32 +0000
Subject: [PATCH 11/22] fix: [SPARK-59277] limit SerDe rewrite and check Hive
 runtime types

---
 .../BaseScriptTransformationSuite.scala       |  33 ++++-
 .../spark/sql/hive/HiveInspectors.scala       |  67 ++++++++--
 .../HiveScriptTransformationExec.scala        |  24 ++--
 .../spark/sql/hive/hiveUDFEvaluators.scala    |   7 +-
 .../org/apache/spark/sql/hive/hiveUDFs.scala  | 118 +++++++++++-------
 .../spark/sql/hive/HiveInspectorSuite.scala   |  16 ++-
 .../HiveScriptTransformationSuite.scala       |  89 +++++++++++++
 .../sql/hive/execution/HiveUDAFSuite.scala    |  18 +++
 .../sql/hive/execution/HiveUDFSuite.scala     |  41 +++++-
 9 files changed, 340 insertions(+), 73 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index f84dd2dda0c40..49e20535c8dda 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -104,17 +104,15 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
 
-  test("SPARK-59277: TRANSFORM CHAR/VARCHAR overflow without SerDe raises EXCEED_LIMIT_LENGTH") {
+  test("SPARK-59277: TRANSFORM CHAR overflow without SerDe raises EXCEED_LIMIT_LENGTH") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
     withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val input = Seq(("abcdef", "abcdefgh")).toDF("c", "v")
+      val input = Seq("abcdef").toDF("c")
       val exception = intercept[Exception] {
         QueryTest.executePlan(
           createScriptTransformationExec(
             script = "cat",
-            output = Seq(
-              AttributeReference("c", CharType(4))(),
-              AttributeReference("v", VarcharType(5))()),
+            output = Seq(AttributeReference("c", CharType(4))()),
             child = input.queryExecution.sparkPlan,
             ioschema = defaultIOSchema),
           spark.sqlContext)
@@ -131,6 +129,31 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     }
   }
 
+  test("SPARK-59277: TRANSFORM VARCHAR overflow without SerDe raises EXCEED_LIMIT_LENGTH") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val input = Seq("abcdefgh").toDF("v")
+      val exception = intercept[Exception] {
+        QueryTest.executePlan(
+          createScriptTransformationExec(
+            script = "cat",
+            output = Seq(AttributeReference("v", VarcharType(5))()),
+            child = input.queryExecution.sparkPlan,
+            ioschema = defaultIOSchema),
+          spark.sqlContext)
+      }
+      val runtimeException = exception match {
+        case s: org.apache.spark.SparkRuntimeException => s
+        case other =>
+          other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
+      }
+      checkError(
+        exception = runtimeException,
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "5"))
+    }
+  }
+
   test("script transformation should not swallow errors from upstream operators (no serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 3d2cbe3775a32..e65d650b1f211 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -1171,25 +1171,39 @@ private[hive] trait HiveInspectors {
     case _ => e.children.forall(canEarlyEval)
   }
 
-  def inspectorToDataType(inspector: ObjectInspector): DataType = inspector match {
+  def inspectorToDataType(inspector: ObjectInspector): DataType =
+    inspectorToDataType(inspector, SQLConf.get.charVarcharFirstClassTypes)
+
+  /**
+   * Maps a Hive inspector to a Catalyst type. When `preserveCharVarchar` is true, CHAR/VARCHAR
+   * inspectors stay CHAR/VARCHAR even if first-class types are disabled. Runtime compatibility
+   * checks use that mode so analysis snapshots are not compared against a conf-dependent STRING
+   * rewrite.
+   */
+  def inspectorToDataType(
+      inspector: ObjectInspector,
+      preserveCharVarchar: Boolean): DataType = inspector match {
     case s: StructObjectInspector =>
       StructType(s.getAllStructFieldRefs.asScala.map(f =>
         types.StructField(
-          f.getFieldName, inspectorToDataType(f.getFieldObjectInspector), nullable = true)
+          f.getFieldName,
+          inspectorToDataType(f.getFieldObjectInspector, preserveCharVarchar),
+          nullable = true)
       ).toArray)
-    case l: ListObjectInspector => ArrayType(inspectorToDataType(l.getListElementObjectInspector))
+    case l: ListObjectInspector =>
+      ArrayType(inspectorToDataType(l.getListElementObjectInspector, preserveCharVarchar))
     case m: MapObjectInspector =>
       MapType(
-        inspectorToDataType(m.getMapKeyObjectInspector),
-        inspectorToDataType(m.getMapValueObjectInspector))
+        inspectorToDataType(m.getMapKeyObjectInspector, preserveCharVarchar),
+        inspectorToDataType(m.getMapValueObjectInspector, preserveCharVarchar))
     case _: WritableStringObjectInspector => StringType
     case _: JavaStringObjectInspector => StringType
     // Hive object inspectors cannot represent collations, so Hive function results use the
     // default collation while preserving CHAR/VARCHAR length under first-class semantics.
-    case hvoi: HiveVarcharObjectInspector if SQLConf.get.charVarcharFirstClassTypes =>
+    case hvoi: HiveVarcharObjectInspector if preserveCharVarchar =>
       VarcharType(hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength)
     case _: HiveVarcharObjectInspector => StringType
-    case hcoi: HiveCharObjectInspector if SQLConf.get.charVarcharFirstClassTypes =>
+    case hcoi: HiveCharObjectInspector if preserveCharVarchar =>
       CharType(hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength)
     case _: HiveCharObjectInspector => StringType
     case _: WritableIntObjectInspector => IntegerType
@@ -1222,6 +1236,45 @@ private[hive] trait HiveInspectors {
     case _: JavaVoidObjectInspector => NullType
   }
 
+  /**
+   * Analysis snapshots the Catalyst return type, but runtime inspectors are rebuilt from the
+   * current children (including foldability). Accept string-family drift so CHAR/VARCHAR
+   * conversion can still apply when Hive surfaces STRING, and require other types to match.
+   */
+  def checkCompatibleHiveReturnType(
+      inspector: ObjectInspector,
+      expectedType: DataType): Unit = {
+    checkCompatibleHiveReturnType(
+      inspectorToDataType(inspector, preserveCharVarchar = true),
+      expectedType)
+  }
+
+  def checkCompatibleHiveReturnType(
+      runtimeType: DataType,
+      expectedType: DataType): Unit = {
+    if (!compatibleHiveReturnType(runtimeType, expectedType)) {
+      throw SparkException.internalError(
+        s"Hive function runtime type ${runtimeType.catalogString} is incompatible " +
+          s"with analysis type ${expectedType.catalogString}.")
+    }
+  }
+
+  private def compatibleHiveReturnType(
+      runtimeType: DataType,
+      expectedType: DataType): Boolean = {
+    (runtimeType, expectedType) match {
+      case (_: StringType, _: StringType) => true
+      case (ArrayType(rt, _), ArrayType(et, _)) => compatibleHiveReturnType(rt, et)
+      case (MapType(rk, rv, _), MapType(ek, ev, _)) =>
+        compatibleHiveReturnType(rk, ek) && compatibleHiveReturnType(rv, ev)
+      case (rt: StructType, et: StructType) if rt.length == et.length =>
+        rt.fields.zip(et.fields).forall { case (rf, ef) =>
+          compatibleHiveReturnType(rf.dataType, ef.dataType)
+        }
+      case (rt, et) => rt.sameType(et)
+    }
+  }
+
   private def decimalTypeInfoToCatalyst(inspector: PrimitiveObjectInspector): DecimalType = {
     val info = inspector.getTypeInfo.asInstanceOf[DecimalTypeInfo]
     DecimalType(info.precision(), info.scale())
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
index 2037f2a50458b..9e4596cb78fd0 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.hive.ql.exec.{RecordReader, RecordWriter}
 import org.apache.hadoop.hive.serde.serdeConstants
 import org.apache.hadoop.hive.serde2.AbstractSerDe
+import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
 import org.apache.hadoop.hive.serde2.objectinspector._
 import org.apache.hadoop.io.Writable
 
@@ -258,10 +259,7 @@ object HiveScriptIOSchema extends HiveInspectors {
       output: Seq[Attribute]): Option[(AbstractSerDe, StructObjectInspector)] = {
     ioschema.outputSerdeClass.map { serdeClass =>
       val (columns, columnTypes) = parseAttrs(output)
-      // Hive CHAR/VARCHAR SerDe truncates silently. Always deserialize as STRING
-      // (even under first-class CHAR/VARCHAR) so unwrapperFor can raise
-      // EXCEED_LIMIT_LENGTH instead of returning a truncated value.
-      val serdeTypes = columnTypes.map(toHiveSerdePhysicalType)
+      val serdeTypes = outputTypesForSerDe(serdeClass, columnTypes)
       val serde = initSerDe(serdeClass, columns, serdeTypes, ioschema.outputSerdeProps)
       val structObjectInspector = serde.getObjectInspector().asInstanceOf[StructObjectInspector]
       (serde, structObjectInspector)
@@ -275,9 +273,10 @@ object HiveScriptIOSchema extends HiveInspectors {
   }
 
   /**
-   * Hive SerDe CHAR/VARCHAR types truncate on deserialize. Map them to STRING so Spark
-   * applies first-class length checks. Unlike `replaceCharVarcharWithString`, this must
-   * run even when first-class CHAR/VARCHAR is enabled.
+   * Hive LazySimpleSerDe CHAR/VARCHAR types truncate on deserialize. Map them to STRING so
+   * Spark applies first-class length checks. Unlike `replaceCharVarcharWithString`, this must
+   * run even when first-class CHAR/VARCHAR is enabled. Only LazySimpleSerDe and subclasses
+   * get this rewrite; other SerDes keep the declared CHAR/VARCHAR schema.
    */
   private def toHiveSerdePhysicalType(dt: DataType): DataType = dt match {
     case ArrayType(et, n) => ArrayType(toHiveSerdePhysicalType(et), n)
@@ -289,6 +288,17 @@ object HiveScriptIOSchema extends HiveInspectors {
     case other => other
   }
 
+  private def outputTypesForSerDe(
+      serdeClassName: String,
+      columnTypes: Seq[DataType]): Seq[DataType] = {
+    val serdeClass = Utils.classForName[AbstractSerDe](serdeClassName)
+    if (classOf[LazySimpleSerDe].isAssignableFrom(serdeClass)) {
+      columnTypes.map(toHiveSerdePhysicalType)
+    } else {
+      columnTypes
+    }
+  }
+
   def initSerDe(
       serdeClassName: String,
       columns: Seq[String],
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
index 1dead6c3bf2d0..5ab96ca7683e3 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
@@ -178,8 +178,11 @@ private[hive] class HiveGenericUDFEvaluator(
   private lazy val argumentInspectors = children.map(toInspector).toArray
 
   @transient
-  lazy val returnInspector =
-    HiveGenericUDFEvaluator.initialize(function, argumentInspectors)
+  lazy val returnInspector = {
+    val inspector = HiveGenericUDFEvaluator.initialize(function, argumentInspectors)
+    checkCompatibleHiveReturnType(inspector, catalystReturnType)
+    inspector
+  }
 
   @transient
   private lazy val deferredObjects: Array[DeferredObject] = argumentInspectors.zip(children).map {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 935647d90dac3..bee28ce49be8f 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -25,7 +25,9 @@ import scala.jdk.CollectionConverters._
 import org.apache.hadoop.hive.ql.exec._
 import org.apache.hadoop.hive.ql.udf.generic._
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer
-import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, ObjectInspector, ObjectInspectorFactory}
+import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, ObjectInspector}
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
@@ -233,27 +235,20 @@ private[hive] case class HiveGenericUDTF(
   extends Generator with HiveInspectors with CodegenFallback with UserDefinedExpression {
 
   @transient
-  protected lazy val function: GenericUDTF = {
-    val fun: GenericUDTF = funcWrapper.createFunction()
-    fun.setCollector(collector)
-    fun
-  }
+  protected lazy val collector = new UDTFCollector
 
   @transient
-  protected lazy val inputInspector = {
-    val inspectors = children.map(toInspector)
-    val fields = inspectors.indices.map(index => s"_col$index").asJava
-    ObjectInspectorFactory.getStandardStructObjectInspector(fields, inspectors.asJava)
-  }
+  private lazy val initialized = HiveGenericUDTF.initialize(
+    funcWrapper, children, collector, Some(elementSchema))
 
   @transient
-  protected lazy val outputInspector = function.initialize(inputInspector)
+  protected lazy val function: GenericUDTF = initialized.function
 
   @transient
-  protected lazy val udtInput = new Array[AnyRef](children.length)
+  protected lazy val outputInspector = initialized.outputInspector
 
   @transient
-  protected lazy val collector = new UDTFCollector
+  protected lazy val udtInput = new Array[AnyRef](children.length)
 
   @transient
   private lazy val inputDataTypes: Array[DataType] = children.map(_.dataType).toArray
@@ -307,6 +302,11 @@ private[hive] case class HiveGenericUDTF(
 }
 
 object HiveGenericUDTF extends HiveInspectors {
+  private[hive] case class InitializedUDTF(
+      function: GenericUDTF,
+      inputInspector: StructObjectInspector,
+      outputInspector: StructObjectInspector)
+
   def apply(
       name: String,
       funcWrapper: HiveFunctionWrapper,
@@ -314,19 +314,32 @@ object HiveGenericUDTF extends HiveInspectors {
     HiveGenericUDTF(name, funcWrapper, children, inferElementSchema(funcWrapper, children))
   }
 
-  def inferElementSchema(
+  private[hive] def initialize(
       funcWrapper: HiveFunctionWrapper,
-      children: Seq[Expression]): StructType = {
+      children: Seq[Expression],
+      collector: Collector,
+      expectedSchema: Option[StructType] = None): InitializedUDTF = {
     val function: GenericUDTF = funcWrapper.createFunction()
-    function.setCollector(new Collector {
-      override def collect(input: java.lang.Object): Unit = {}
-    })
+    function.setCollector(collector)
     val inspectors = children.map(toInspector)
     val fields = inspectors.indices.map(index => s"_col$index").asJava
     val inputInspector =
       ObjectInspectorFactory.getStandardStructObjectInspector(fields, inspectors.asJava)
     val outputInspector = function.initialize(inputInspector)
-    StructType(outputInspector.getAllStructFieldRefs.asScala.map { field =>
+    expectedSchema.foreach(checkCompatibleHiveReturnType(outputInspector, _))
+    InitializedUDTF(function, inputInspector, outputInspector)
+  }
+
+  def inferElementSchema(
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression]): StructType = {
+    val initialized = initialize(
+      funcWrapper,
+      children,
+      new Collector {
+        override def collect(input: java.lang.Object): Unit = {}
+      })
+    StructType(initialized.outputInspector.getAllStructFieldRefs.asScala.map { field =>
       StructField(
         field.getFieldName,
         inspectorToDataType(field.getFieldObjectInspector),
@@ -394,25 +407,18 @@ private[hive] case class HiveUDAFFunction(
   override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate =
     copy(inputAggBufferOffset = newInputAggBufferOffset)
 
-  // Hive `ObjectInspector`s for all child expressions (input parameters of the function).
   @transient
-  private lazy val inputInspectors = children.map(toInspector).toArray
+  private lazy val initialized = HiveUDAFFunction.initializeEvaluators(
+    funcWrapper,
+    children,
+    isUDAFBridgeRequired,
+    Some(partialResultDataType),
+    Some(dataType))
 
   // Spark SQL data types of input parameters.
   @transient
   private lazy val inputDataTypes: Array[DataType] = children.map(_.dataType).toArray
 
-  private def newEvaluator(): GenericUDAFEvaluator = {
-    val resolver = if (isUDAFBridgeRequired) {
-      new SparkGenericUDAFBridge(funcWrapper.createFunction[UDAF]())
-    } else {
-      funcWrapper.createFunction[AbstractGenericUDAFResolver]()
-    }
-
-    val parameterInfo = new SimpleGenericUDAFParameterInfo(inputInspectors, false, false, false)
-    resolver.getEvaluator(parameterInfo)
-  }
-
   private case class HiveEvaluator(
       evaluator: GenericUDAFEvaluator,
       objectInspector: ObjectInspector)
@@ -420,20 +426,14 @@ private[hive] case class HiveUDAFFunction(
   // The UDAF evaluator used to consume raw input rows and produce partial aggregation results.
   // Hive `ObjectInspector` used to inspect partial aggregation results.
   @transient
-  private lazy val partial1HiveEvaluator = {
-    val evaluator = newEvaluator()
-    HiveEvaluator(evaluator, evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, inputInspectors))
-  }
+  private lazy val partial1HiveEvaluator = HiveEvaluator(
+    initialized.partialEvaluator, initialized.partialInspector)
 
   // The UDAF evaluator used to consume partial aggregation results and produce final results.
   // Hive `ObjectInspector` used to inspect final results.
   @transient
-  private lazy val finalHiveEvaluator = {
-    val evaluator = newEvaluator()
-    HiveEvaluator(
-      evaluator,
-      evaluator.init(GenericUDAFEvaluator.Mode.FINAL, Array(partial1HiveEvaluator.objectInspector)))
-  }
+  private lazy val finalHiveEvaluator = HiveEvaluator(
+    initialized.finalEvaluator, initialized.finalInspector)
 
   @transient
   private lazy val inputWrappers = children.map(x => wrapperFor(toInspector(x), x.dataType)).toArray
@@ -594,6 +594,12 @@ private[hive] case class HiveUDAFFunction(
 }
 
 object HiveUDAFFunction extends HiveInspectors {
+  private[hive] case class InitializedEvaluators(
+      partialEvaluator: GenericUDAFEvaluator,
+      partialInspector: ObjectInspector,
+      finalEvaluator: GenericUDAFEvaluator,
+      finalInspector: ObjectInspector)
+
   def apply(
       name: String,
       funcWrapper: HiveFunctionWrapper,
@@ -619,10 +625,12 @@ object HiveUDAFFunction extends HiveInspectors {
       resultType)
   }
 
-  def inferResolvedTypes(
+  private[hive] def initializeEvaluators(
       funcWrapper: HiveFunctionWrapper,
       children: Seq[Expression],
-      isUDAFBridgeRequired: Boolean): (DataType, DataType) = {
+      isUDAFBridgeRequired: Boolean,
+      expectedPartialType: Option[DataType] = None,
+      expectedResultType: Option[DataType] = None): InitializedEvaluators = {
     val inputInspectors = children.map(toInspector).toArray
     def newEvaluator(): GenericUDAFEvaluator = {
       val resolver = if (isUDAFBridgeRequired) {
@@ -630,7 +638,8 @@ object HiveUDAFFunction extends HiveInspectors {
       } else {
         funcWrapper.createFunction[AbstractGenericUDAFResolver]()
       }
-      val parameterInfo = new SimpleGenericUDAFParameterInfo(inputInspectors, false, false, false)
+      val parameterInfo = new SimpleGenericUDAFParameterInfo(
+        inputInspectors, false, false, false)
       resolver.getEvaluator(parameterInfo)
     }
     val partial1 = newEvaluator()
@@ -638,7 +647,22 @@ object HiveUDAFFunction extends HiveInspectors {
     val finalEvaluator = newEvaluator()
     val finalInspector =
       finalEvaluator.init(GenericUDAFEvaluator.Mode.FINAL, Array(partialInspector))
-    (inspectorToDataType(partialInspector), inspectorToDataType(finalInspector))
+    expectedPartialType.foreach(checkCompatibleHiveReturnType(partialInspector, _))
+    expectedResultType.foreach(checkCompatibleHiveReturnType(finalInspector, _))
+    InitializedEvaluators(
+      partial1,
+      partialInspector,
+      finalEvaluator,
+      finalInspector)
+  }
+
+  def inferResolvedTypes(
+      funcWrapper: HiveFunctionWrapper,
+      children: Seq[Expression],
+      isUDAFBridgeRequired: Boolean): (DataType, DataType) = {
+    val initialized = initializeEvaluators(funcWrapper, children, isUDAFBridgeRequired)
+    (inspectorToDataType(initialized.partialInspector),
+      inspectorToDataType(initialized.finalInspector))
   }
 }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index d32f1c612a50e..195148265f328 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
 import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, DecimalTypeInfo, VarcharTypeInfo}
 import org.apache.hadoop.io.LongWritable
 
-import org.apache.spark.{SparkFunSuite, SparkRuntimeException}
+import org.apache.spark.{SparkException, SparkFunSuite, SparkRuntimeException}
 import org.apache.spark.sql.{AnalysisException, Row, TestUserClassUDT}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Literal, SpecificInternalRow}
@@ -453,11 +453,23 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
   test("SPARK-59277: Hive CHAR/VARCHAR inspectors remain STRING under legacy semantics") {
     withFirstClassCharVarchar(enabled = false) {
       Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
-        assert(inspectorToDataType(toInspector(dataType)) === StringType)
+        val inspector = toInspector(dataType)
+        assert(inspectorToDataType(inspector) === StringType)
+        assert(inspectorToDataType(inspector, preserveCharVarchar = true) === dataType)
       }
     }
   }
 
+  test("SPARK-59277: Hive return types stay compatible across string-family drift") {
+    checkCompatibleHiveReturnType(StringType, CharType(5))
+    checkCompatibleHiveReturnType(CharType(5), StringType)
+    checkCompatibleHiveReturnType(VarcharType(3), CharType(5))
+    checkCompatibleHiveReturnType(ArrayType(StringType), ArrayType(VarcharType(7)))
+    intercept[SparkException] {
+      checkCompatibleHiveReturnType(IntegerType, CharType(5))
+    }
+  }
+
   test("SPARK-59277: Hive CHAR/VARCHAR boundaries enforce Spark length semantics") {
     withFirstClassCharVarchar(enabled = true) {
       val varchar = VarcharType(3)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
index 5dde1a09094df..40e83f5ad140c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -20,8 +20,15 @@ package org.apache.spark.sql.hive.execution
 import java.sql.Timestamp
 import java.time.{Duration, Period}
 import java.time.temporal.ChronoUnit
+import java.util.{Arrays, Properties}
 
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hive.serde.serdeConstants
+import org.apache.hadoop.hive.serde2.{AbstractSerDe, SerDeStats}
 import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
+import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectInspectorFactory}
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
+import org.apache.hadoop.io.{Text, Writable}
 import org.scalatest.exceptions.TestFailedException
 
 import org.apache.spark.{SparkException, TestUtils}
@@ -418,6 +425,62 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
     }
   }
 
+  test("SPARK-59277: TRANSFORM VARCHAR overflow with Hive SerDe raises EXCEED_LIMIT_LENGTH") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val exception = intercept[Exception] {
+        sql(
+          """
+            |SELECT TRANSFORM('abcdefgh')
+            |USING 'cat'
+            |AS (v VARCHAR(5))
+            |FROM VALUES (1) input(dummy)
+            |""".stripMargin).collect()
+      }
+      val runtimeException = exception match {
+        case s: org.apache.spark.SparkRuntimeException => s
+        case other =>
+          other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
+      }
+      checkError(
+        exception = runtimeException,
+        condition = "EXCEED_LIMIT_LENGTH",
+        parameters = Map("limit" -> "5"))
+    }
+  }
+
+  test("SPARK-59277: output SerDe CHAR/VARCHAR rewrite is LazySimpleSerDe-only") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val output = Seq(
+        AttributeReference("c", CharType(4))(),
+        AttributeReference("v", VarcharType(5))(),
+        AttributeReference("nested", ArrayType(CharType(4)))())
+
+      val (_, lazySoi) = HiveScriptIOSchema.initOutputSerDe(hiveIOSchema, output).get
+      assert(lazySoi.getAllStructFieldRefs.get(0).getFieldObjectInspector.getTypeName ===
+        "string")
+      assert(lazySoi.getAllStructFieldRefs.get(1).getFieldObjectInspector.getTypeName ===
+        "string")
+      assert(lazySoi.getAllStructFieldRefs.get(2).getFieldObjectInspector.getTypeName ===
+        "array<string>")
+
+      val subclassSchema = hiveIOSchema.copy(
+        outputSerdeClass = Some(classOf[TestLazySimpleSerDe].getCanonicalName))
+      val (_, subclassSoi) = HiveScriptIOSchema.initOutputSerDe(subclassSchema, output).get
+      assert(subclassSoi.getAllStructFieldRefs.get(0).getFieldObjectInspector.getTypeName ===
+        "string")
+
+      SchemaCapturingSerDe.lastColumnTypes = null
+      val customSchema = defaultIOSchema.copy(
+        outputSerdeClass = Some(classOf[SchemaCapturingSerDe].getCanonicalName))
+      HiveScriptIOSchema.initOutputSerDe(customSchema, output)
+      val captured = SchemaCapturingSerDe.lastColumnTypes
+      assert(captured.contains("char(4)"))
+      assert(captured.contains("varchar(5)"))
+      assert(captured.contains("array<char(4)>"))
+    }
+  }
+
   test("SPARK-32400: TRANSFORM doesn't support CalendarIntervalType/UserDefinedType (hive serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
     withTempView("v") {
@@ -714,3 +777,29 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
     }
   }
 }
+
+class TestLazySimpleSerDe extends LazySimpleSerDe
+
+class SchemaCapturingSerDe extends AbstractSerDe {
+  override def initialize(conf: Configuration, tbl: Properties): Unit = {
+    SchemaCapturingSerDe.lastColumnTypes =
+      tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES)
+  }
+
+  override def getObjectInspector: ObjectInspector =
+    ObjectInspectorFactory.getStandardStructObjectInspector(
+      Arrays.asList("col"),
+      Arrays.asList(PrimitiveObjectInspectorFactory.javaStringObjectInspector))
+
+  override def getSerializedClass: Class[_ <: Writable] = classOf[Text]
+
+  override def getSerDeStats: SerDeStats = null
+
+  override def serialize(obj: Any, inspector: ObjectInspector): Writable = null
+
+  override def deserialize(blob: Writable): AnyRef = null
+}
+
+object SchemaCapturingSerDe {
+  @volatile var lastColumnTypes: String = _
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
index 2a5d072039fd3..509cb6ae687ab 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
@@ -31,14 +31,18 @@ import test.org.apache.spark.sql.MyDoubleAvg
 
 import org.apache.spark.SPARK_DOC_ROOT
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Literal}
 import org.apache.spark.sql.catalyst.expressions.Cast._
 import org.apache.spark.sql.catalyst.expressions.aggregate.Complete
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.ObjectHashAggregateExec
+import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
+import org.apache.spark.sql.hive.HiveUDAFFunction
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{CharType, VarcharType}
 import org.apache.spark.tags.SlowHiveTest
+import org.apache.spark.unsafe.types.UTF8String
 
 @SlowHiveTest
 class HiveUDAFSuite extends QueryTest
@@ -221,6 +225,20 @@ class HiveUDAFSuite extends QueryTest
     }
   }
 
+  test("SPARK-59277: HiveUDAFFunction keeps analysis type after child constantness changes") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val attr = AttributeReference("value", VarcharType(7))()
+      val original = HiveUDAFFunction(
+        "hive_max",
+        HiveFunctionWrapper(classOf[GenericUDAFMax].getName),
+        Seq(Literal.create(UTF8String.fromString("abc"), VarcharType(7))))
+      assert(original.dataType === VarcharType(7))
+      val copied = original.withNewChildren(Seq(attr)).asInstanceOf[HiveUDAFFunction]
+      assert(copied.dataType === original.dataType)
+      copied.serialize(null)
+    }
+  }
+
   test("SPARK-59277: Hive UDAF partial buffer type can differ from the final result") {
     withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
       withUserDefinedFunction("char_max" -> true) {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 00c60d333b401..07cfad84ad3c6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -42,14 +42,14 @@ import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BindReferences, CodegenObjectFactoryMode, Literal}
 import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.execution.WholeStageCodegenExec
 import org.apache.spark.sql.functions.{call_function, max}
-import org.apache.spark.sql.hive.HiveGenericUDF
+import org.apache.spark.sql.hive.{HiveGenericUDF, HiveGenericUDTF}
 import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
 import org.apache.spark.sql.hive.test.{TestHiveSingleton, TestUDTFJar}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{CharType, DataType, StringType, TimestampType, TimeType, VarcharType}
+import org.apache.spark.sql.types._
 import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
@@ -1080,6 +1080,41 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
     }
   }
 
+  test("SPARK-59277: HiveGenericUDF keeps analysis type after child constantness changes") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val attr = AttributeReference("value", CharType(5))()
+      val original = HiveGenericUDF(
+        "hive_upper",
+        HiveFunctionWrapper(classOf[GenericUDFUpper].getName),
+        Seq(Literal.create(UTF8String.fromString("Ab"), CharType(5))))
+      assert(original.dataType === CharType(5))
+      val copied = original.withNewChildren(Seq(attr)).asInstanceOf[HiveGenericUDF]
+      assert(copied.dataType === CharType(5))
+      val bound = BindReferences.bindReference(copied, Seq(attr))
+      assert(bound.eval(InternalRow(UTF8String.fromString("Ab   "))) ===
+        UTF8String.fromString("AB   "))
+    }
+  }
+
+  test("SPARK-59277: HiveGenericUDTF keeps analysis schema after child constantness changes") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val arrayType = ArrayType(VarcharType(7))
+      val values = new GenericArrayData(Array[Any](UTF8String.fromString("abc")))
+      val original = HiveGenericUDTF(
+        "hive_explode",
+        HiveFunctionWrapper(classOf[GenericUDTFExplode].getName),
+        Seq(Literal(values, arrayType)))
+      assert(original.elementSchema.head.dataType === VarcharType(7))
+      val attr = AttributeReference("values", arrayType)()
+      val copied = original.withNewChildren(Seq(attr)).asInstanceOf[HiveGenericUDTF]
+      assert(copied.elementSchema === original.elementSchema)
+      val bound = BindReferences.bindReference(copied, Seq(attr))
+        .asInstanceOf[HiveGenericUDTF]
+      val rows = bound.eval(InternalRow(values)).iterator.toSeq
+      assert(rows.map(_.get(0, VarcharType(7))) === Seq(UTF8String.fromString("abc")))
+    }
+  }
+
   test("SPARK-58792: copied HiveGenericUDF nodes must not share a mutable GenericUDF") {
     val tsAttr = AttributeReference("ts", TimestampType, nullable = false)()
     val constTs = Literal(

From 9ceec6e5e236fc2e70d162b4cf6cb1e29bc33119 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Thu, 17 Sep 2026 02:44:27 +0000
Subject: [PATCH 12/22] fix: [SPARK-59277] enforce nested TRANSFORM CHAR
 semantics

---
 .../BaseScriptTransformationExec.scala        | 17 ++++-
 .../BaseScriptTransformationSuite.scala       | 55 ++++++++++++++++
 .../spark/sql/hive/HiveInspectors.scala       | 64 +++++++++++--------
 .../org/apache/spark/sql/hive/hiveUDFs.scala  |  3 +-
 4 files changed, 108 insertions(+), 31 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index 5e29f174a293f..0968669d8ca49 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Cast, Expression, GenericInternalRow, JsonToStructs, Literal, StructsToJson, UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical.ScriptInputOutputSchema
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils}
+import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, DateTimeUtils, IntervalUtils}
 import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -249,6 +249,21 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
         data => IntervalUtils.microsToDuration(
           IntervalUtils.castStringToDTInterval(UTF8String.fromString(data), start, end)),
         converter)
+      case dt @ (_: ArrayType | _: MapType | _: StructType)
+          if CharVarcharUtils.hasCharVarchar(dt) =>
+        val physicalType = CharVarcharUtils.replaceCharVarcharWithString(dt)
+        val complexTypeFactory = JsonToStructs(
+          physicalType,
+          ioschema.outputSerdeProps.toMap,
+          Literal(null),
+          Some(conf.sessionLocalTimeZone))
+        val toScala = CatalystTypeConverters.createToScalaConverter(physicalType)
+        (data: String) =>
+          if (data == ioschema.outputRowFormatMap("TOK_TABLEROWFORMATNULL")) {
+            null
+          } else {
+            converter(toScala(complexTypeFactory.nullSafeEval(UTF8String.fromString(data))))
+          }
       case _: ArrayType | _: MapType | _: StructType =>
         val complexTypeFactory = JsonToStructs(attr.dataType,
           ioschema.outputSerdeProps.toMap, Literal(null), Some(conf.sessionLocalTimeZone))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index 49e20535c8dda..7cf349a3e7cee 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -154,6 +154,61 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     }
   }
 
+  test("SPARK-59277: TRANSFORM converts nested CHAR/VARCHAR without SerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      Seq(
+        ("""["ab"]""", ArrayType(CharType(4)), Row(Seq("ab  "))),
+        ("""["xy"]""", ArrayType(VarcharType(4)), Row(Seq("xy"))),
+        (
+          """{"value":"xy"}""",
+          StructType(Seq(StructField("value", CharType(5)))),
+          Row(Row("xy   ")))).foreach { case (json, dataType, expected) =>
+        val input = Seq(json).toDF("value")
+        checkAnswer(
+          input,
+          (child: SparkPlan) => createScriptTransformationExec(
+            script = "cat",
+            output = Seq(AttributeReference("value", dataType)()),
+            child = child,
+            ioschema = defaultIOSchema),
+          Seq(expected))
+      }
+    }
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("SPARK-59277: TRANSFORM nested CHAR/VARCHAR overflow without SerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      Seq(
+        (ArrayType(CharType(4)), """["abcdef"]"""),
+        (
+          StructType(Seq(StructField("value", VarcharType(4)))),
+          """{"value":"abcdef"}""")).foreach { case (dataType, json) =>
+        val input = Seq(json).toDF("value")
+        val exception = intercept[Exception] {
+          QueryTest.executePlan(
+            createScriptTransformationExec(
+              script = "cat",
+              output = Seq(AttributeReference("value", dataType)()),
+              child = input.queryExecution.sparkPlan,
+              ioschema = defaultIOSchema),
+            spark.sqlContext)
+        }
+        val runtimeException = exception match {
+          case s: org.apache.spark.SparkRuntimeException => s
+          case other =>
+            other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
+        }
+        checkError(
+          exception = runtimeException,
+          condition = "EXCEED_LIMIT_LENGTH",
+          parameters = Map("limit" -> "4"))
+      }
+    }
+  }
+
   test("script transformation should not swallow errors from upstream operators (no serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index e65d650b1f211..f204f929532c8 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -277,6 +277,30 @@ private[hive] trait HiveInspectors {
       (o: Any) =>
         x.getWritableConstantValue
     case x: PrimitiveObjectInspector => x match {
+      case hvoi: HiveVarcharObjectInspector =>
+        val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
+        charVarcharWrapper(
+          dataType match {
+            case v: VarcharType => Some(v.length)
+            case _ => None
+          },
+          length,
+          hvoi.preferWritable(),
+          CharVarcharCodegenUtils.varcharTypeWriteSideCheck,
+          (value, size) => new HiveVarchar(value, size),
+          (value, size) => new hiveIo.HiveVarcharWritable(new HiveVarchar(value, size)))
+      case hcoi: HiveCharObjectInspector =>
+        val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
+        charVarcharWrapper(
+          dataType match {
+            case c: CharType => Some(c.length)
+            case _ => None
+          },
+          length,
+          hcoi.preferWritable(),
+          CharVarcharCodegenUtils.charTypeWriteSideCheck,
+          (value, size) => new HiveChar(value, size),
+          (value, size) => new hiveIo.HiveCharWritable(new HiveChar(value, size)))
       case _: StringObjectInspector if x.preferWritable() =>
         withNullSafe(o => getStringWritable(o))
       case _: StringObjectInspector =>
@@ -309,30 +333,6 @@ private[hive] trait HiveInspectors {
         withNullSafe(o => getByteWritable(o))
       case _: ByteObjectInspector =>
         withNullSafe(o => o.asInstanceOf[java.lang.Byte])
-      case hvoi: HiveVarcharObjectInspector =>
-        val length = hvoi.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength
-        charVarcharWrapper(
-          dataType match {
-            case v: VarcharType => Some(v.length)
-            case _ => None
-          },
-          length,
-          hvoi.preferWritable(),
-          CharVarcharCodegenUtils.varcharTypeWriteSideCheck,
-          (value, size) => new HiveVarchar(value, size),
-          (value, size) => new hiveIo.HiveVarcharWritable(new HiveVarchar(value, size)))
-      case hcoi: HiveCharObjectInspector =>
-        val length = hcoi.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
-        charVarcharWrapper(
-          dataType match {
-            case c: CharType => Some(c.length)
-            case _ => None
-          },
-          length,
-          hcoi.preferWritable(),
-          CharVarcharCodegenUtils.charTypeWriteSideCheck,
-          (value, size) => new HiveChar(value, size),
-          (value, size) => new hiveIo.HiveCharWritable(new HiveChar(value, size)))
       case _: JavaHiveDecimalObjectInspector =>
         withNullSafe(o =>
           HiveDecimal.create(o.asInstanceOf[Decimal].toJavaBigDecimal))
@@ -958,14 +958,22 @@ private[hive] trait HiveInspectors {
     }
 
   /**
-   * Builds an in-place unwrapper using the target Catalyst `dataType`. This preserves all
-   * target-aware conversion, including CHAR/VARCHAR checks and nanosecond timestamp precision.
+   * Builds an in-place unwrapper using the target Catalyst `dataType` only when target-aware
+   * conversion is required. Other types retain the primitive setter fast paths.
    */
   def unwrapperFor(
       field: HiveStructField,
       dataType: DataType): (Any, InternalRow, Int) => Unit = {
-    val unwrapper = unwrapperFor(field.getFieldObjectInspector, dataType)
-    (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
+    val requiresTypedConversion = dataType.existsRecursively {
+      case _: CharType | _: VarcharType | _: AnyTimestampNanoType => true
+      case _ => false
+    }
+    if (requiresTypedConversion) {
+      val unwrapper = unwrapperFor(field.getFieldObjectInspector, dataType)
+      (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
+    } else {
+      unwrapperFor(field)
+    }
   }
 
   def wrap(a: Any, oi: ObjectInspector, dataType: DataType): AnyRef = {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index bee28ce49be8f..7472341eabb21 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -304,7 +304,6 @@ private[hive] case class HiveGenericUDTF(
 object HiveGenericUDTF extends HiveInspectors {
   private[hive] case class InitializedUDTF(
       function: GenericUDTF,
-      inputInspector: StructObjectInspector,
       outputInspector: StructObjectInspector)
 
   def apply(
@@ -327,7 +326,7 @@ object HiveGenericUDTF extends HiveInspectors {
       ObjectInspectorFactory.getStandardStructObjectInspector(fields, inspectors.asJava)
     val outputInspector = function.initialize(inputInspector)
     expectedSchema.foreach(checkCompatibleHiveReturnType(outputInspector, _))
-    InitializedUDTF(function, inputInspector, outputInspector)
+    InitializedUDTF(function, outputInspector)
   }
 
   def inferElementSchema(

From c825068e0b041dff3c0f365bb4989b1feeeffd2d Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Thu, 17 Sep 2026 04:37:27 +0000
Subject: [PATCH 13/22] fix: [SPARK-59277] physicalize nested TRANSFORM CHAR
 without SerDe

Share the unbounded STRING rewrite so first-class CHAR/VARCHAR overflow
checks still apply, keep malformed nested JSON as null, and tighten Hive
return-type compatibility to STRING boundary drift only.
---
 .../BaseScriptTransformationExec.scala        | 48 +++++++++++++++----
 .../BaseScriptTransformationSuite.scala       | 22 +++++++++
 .../spark/sql/hive/HiveInspectors.scala       | 12 +++--
 .../HiveScriptTransformationExec.scala        | 17 ++-----
 .../spark/sql/hive/HiveInspectorSuite.scala   | 16 +++++--
 .../HiveScriptTransformationSuite.scala       | 32 +++++++++++++
 .../sql/hive/execution/HiveUDFSuite.scala     | 27 ++++++++++-
 7 files changed, 143 insertions(+), 31 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index 0968669d8ca49..bf3fa7da0934a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -31,7 +31,17 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.LogKeys._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Cast, Expression, GenericInternalRow, JsonToStructs, Literal, StructsToJson, UnsafeProjection}
+import org.apache.spark.sql.catalyst.expressions.{
+  Attribute,
+  AttributeSet,
+  BoundReference,
+  Cast,
+  Expression,
+  GenericInternalRow,
+  JsonToStructs,
+  Literal,
+  StructsToJson,
+  UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical.ScriptInputOutputSchema
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, DateTimeUtils, IntervalUtils}
@@ -251,19 +261,32 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
         converter)
       case dt @ (_: ArrayType | _: MapType | _: StructType)
           if CharVarcharUtils.hasCharVarchar(dt) =>
-        val physicalType = CharVarcharUtils.replaceCharVarcharWithString(dt)
+        val physicalType = ScriptTransformationIOSchema.toUnboundedStringType(dt)
+        // JSON object keys are strings. Cast them to the declared map key type after parsing.
+        val jsonType = physicalType.transformRecursively {
+          case map: MapType if !map.keyType.isInstanceOf[StringType] =>
+            map.copy(keyType = StringType)
+        }
         val complexTypeFactory = JsonToStructs(
-          physicalType,
+          jsonType,
           ioschema.outputSerdeProps.toMap,
           Literal(null),
           Some(conf.sessionLocalTimeZone))
+        val parsedToPhysical = if (jsonType.sameType(physicalType)) {
+          identity[Any] _
+        } else {
+          val cast = Cast(
+            BoundReference(0, jsonType, nullable = true),
+            physicalType,
+            Some(conf.sessionLocalTimeZone))
+          value: Any => cast.eval(InternalRow(value))
+        }
         val toScala = CatalystTypeConverters.createToScalaConverter(physicalType)
-        (data: String) =>
-          if (data == ioschema.outputRowFormatMap("TOK_TABLEROWFORMATNULL")) {
-            null
-          } else {
-            converter(toScala(complexTypeFactory.nullSafeEval(UTF8String.fromString(data))))
-          }
+        val parser = wrapperConvertException(
+          data => parsedToPhysical(
+            complexTypeFactory.nullSafeEval(UTF8String.fromString(data))),
+          identity)
+        data => converter(toScala(parser(data)))
       case _: ArrayType | _: MapType | _: StructType =>
         val complexTypeFactory = JsonToStructs(attr.dataType,
           ioschema.outputSerdeProps.toMap, Literal(null), Some(conf.sessionLocalTimeZone))
@@ -399,6 +422,13 @@ case class ScriptTransformationIOSchema(
 }
 
 object ScriptTransformationIOSchema {
+  private[sql] def toUnboundedStringType(dataType: DataType): DataType = {
+    dataType.transformRecursively {
+      case c: CharType => c.toStringType
+      case v: VarcharType => v.toStringType
+    }
+  }
+
   val defaultFormat = Map(
     ("TOK_TABLEROWFORMATFIELD", "\u0001"),
     ("TOK_TABLEROWFORMATLINES", "\n"),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index 7cf349a3e7cee..d6f96de9c6d28 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -160,6 +160,10 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
       Seq(
         ("""["ab"]""", ArrayType(CharType(4)), Row(Seq("ab  "))),
         ("""["xy"]""", ArrayType(VarcharType(4)), Row(Seq("xy"))),
+        (
+          """{"1":"ab"}""",
+          MapType(IntegerType, CharType(4)),
+          Row(Map(1 -> "ab  "))),
         (
           """{"value":"xy"}""",
           StructType(Seq(StructField("value", CharType(5)))),
@@ -183,6 +187,7 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
       Seq(
         (ArrayType(CharType(4)), """["abcdef"]"""),
+        (MapType(IntegerType, CharType(4)), """{"1":"abcdef"}"""),
         (
           StructType(Seq(StructField("value", VarcharType(4)))),
           """{"value":"abcdef"}""")).foreach { case (dataType, json) =>
@@ -209,6 +214,23 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     }
   }
 
+  test("SPARK-59277: malformed nested CHAR JSON without SerDe returns null") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val input = Seq("""{"1":""").toDF("value")
+      checkAnswer(
+        input,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(
+            AttributeReference("value", MapType(IntegerType, CharType(4)))()),
+          child = child,
+          ioschema = defaultIOSchema),
+        Seq(Row(null)))
+    }
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
   test("script transformation should not swallow errors from upstream operators (no serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index f204f929532c8..372ac96fa45d1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -1246,8 +1246,10 @@ private[hive] trait HiveInspectors {
 
   /**
    * Analysis snapshots the Catalyst return type, but runtime inspectors are rebuilt from the
-   * current children (including foldability). Accept string-family drift so CHAR/VARCHAR
-   * conversion can still apply when Hive surfaces STRING, and require other types to match.
+   * current children (including foldability and session CHAR/VARCHAR settings). STRING may drift
+   * to or from a bounded string type across that boundary. Two bounded types must match exactly:
+   * accepting a different kind or length would apply the snapshotted conversion to an incompatible
+   * runtime value.
    */
   def checkCompatibleHiveReturnType(
       inspector: ObjectInspector,
@@ -1271,7 +1273,11 @@ private[hive] trait HiveInspectors {
       runtimeType: DataType,
       expectedType: DataType): Boolean = {
     (runtimeType, expectedType) match {
-      case (_: StringType, _: StringType) => true
+      case (rt: CharType, et: CharType) => rt == et
+      case (rt: VarcharType, et: VarcharType) => rt == et
+      case (_: CharType | _: VarcharType, _: CharType | _: VarcharType) => false
+      case (_: StringType, _: CharType | _: VarcharType) => true
+      case (_: CharType | _: VarcharType, _: StringType) => true
       case (ArrayType(rt, _), ArrayType(et, _)) => compatibleHiveReturnType(rt, et)
       case (MapType(rk, rv, _), MapType(ek, ev, _)) =>
         compatibleHiveReturnType(rk, ek) && compatibleHiveReturnType(rv, ev)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
index 9e4596cb78fd0..107530f8ffbce 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
@@ -274,26 +274,15 @@ object HiveScriptIOSchema extends HiveInspectors {
 
   /**
    * Hive LazySimpleSerDe CHAR/VARCHAR types truncate on deserialize. Map them to STRING so
-   * Spark applies first-class length checks. Unlike `replaceCharVarcharWithString`, this must
-   * run even when first-class CHAR/VARCHAR is enabled. Only LazySimpleSerDe and subclasses
-   * get this rewrite; other SerDes keep the declared CHAR/VARCHAR schema.
+   * Spark applies first-class length checks. Only LazySimpleSerDe and subclasses get this
+   * rewrite; other SerDes keep the declared CHAR/VARCHAR schema.
    */
-  private def toHiveSerdePhysicalType(dt: DataType): DataType = dt match {
-    case ArrayType(et, n) => ArrayType(toHiveSerdePhysicalType(et), n)
-    case MapType(kt, vt, n) =>
-      MapType(toHiveSerdePhysicalType(kt), toHiveSerdePhysicalType(vt), n)
-    case StructType(fields) =>
-      StructType(fields.map(f => f.copy(dataType = toHiveSerdePhysicalType(f.dataType))))
-    case _: CharType | _: VarcharType => StringType
-    case other => other
-  }
-
   private def outputTypesForSerDe(
       serdeClassName: String,
       columnTypes: Seq[DataType]): Seq[DataType] = {
     val serdeClass = Utils.classForName[AbstractSerDe](serdeClassName)
     if (classOf[LazySimpleSerDe].isAssignableFrom(serdeClass)) {
-      columnTypes.map(toHiveSerdePhysicalType)
+      columnTypes.map(ScriptTransformationIOSchema.toUnboundedStringType)
     } else {
       columnTypes
     }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 195148265f328..0650a540f30b5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -460,13 +460,21 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
     }
   }
 
-  test("SPARK-59277: Hive return types stay compatible across string-family drift") {
+  test("SPARK-59277: Hive return type compatibility allows only STRING boundary drift") {
     checkCompatibleHiveReturnType(StringType, CharType(5))
     checkCompatibleHiveReturnType(CharType(5), StringType)
-    checkCompatibleHiveReturnType(VarcharType(3), CharType(5))
     checkCompatibleHiveReturnType(ArrayType(StringType), ArrayType(VarcharType(7)))
-    intercept[SparkException] {
-      checkCompatibleHiveReturnType(IntegerType, CharType(5))
+    checkCompatibleHiveReturnType(CharType(5), CharType(5))
+    checkCompatibleHiveReturnType(VarcharType(3), VarcharType(3))
+
+    Seq[(DataType, DataType)](
+      VarcharType(3) -> CharType(5),
+      CharType(4) -> CharType(5),
+      VarcharType(4) -> VarcharType(5),
+      IntegerType -> CharType(5)).foreach { case (runtimeType, expectedType) =>
+      intercept[SparkException] {
+        checkCompatibleHiveReturnType(runtimeType, expectedType)
+      }
     }
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
index 40e83f5ad140c..5772784795fd1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -449,6 +449,38 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
     }
   }
 
+  test("SPARK-59277: nested CHAR/VARCHAR overflow with Hive SerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      Seq(
+        """
+          |SELECT TRANSFORM(array('abcdef'))
+          |USING 'cat'
+          |AS (value ARRAY<CHAR(4)>)
+          |FROM VALUES (1) input(dummy)
+          |""".stripMargin,
+        """
+          |SELECT TRANSFORM(named_struct('value', 'abcdef'))
+          |USING 'cat'
+          |AS (value STRUCT<value: VARCHAR(4)>)
+          |FROM VALUES (1) input(dummy)
+          |""".stripMargin).foreach { query =>
+        val exception = intercept[Exception] {
+          sql(query).collect()
+        }
+        val runtimeException = exception match {
+          case s: org.apache.spark.SparkRuntimeException => s
+          case other =>
+            other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
+        }
+        checkError(
+          exception = runtimeException,
+          condition = "EXCEED_LIMIT_LENGTH",
+          parameters = Map("limit" -> "4"))
+      }
+    }
+  }
+
   test("SPARK-59277: output SerDe CHAR/VARCHAR rewrite is LazySimpleSerDe-only") {
     withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
       val output = Seq(
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 07cfad84ad3c6..719ea2b891c49 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -33,7 +33,9 @@ import org.apache.hadoop.hive.ql.udf.generic._
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject
 import org.apache.hadoop.hive.serde2.{AbstractSerDe, SerDeStats}
 import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectInspectorFactory}
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.{
+  HiveCharObjectInspector,
+  PrimitiveObjectInspectorFactory}
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory
 import org.apache.hadoop.io.{LongWritable, Writable}
 
@@ -900,10 +902,13 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
     withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
       withUserDefinedFunction(
           "hive_simple_concat" -> true,
+          "hive_char_padded" -> true,
           "hive_upper" -> true,
           "hive_explode" -> true) {
         sql(s"CREATE TEMPORARY FUNCTION hive_simple_concat AS " +
           s"'${classOf[UDFStringString].getName}'")
+        sql(s"CREATE TEMPORARY FUNCTION hive_char_padded AS " +
+          s"'${classOf[InspectCharGenericUDF].getName}'")
         sql(s"CREATE TEMPORARY FUNCTION hive_upper AS '${classOf[GenericUDFUpper].getName}'")
         sql(s"CREATE TEMPORARY FUNCTION hive_explode AS '${classOf[GenericUDTFExplode].getName}'")
 
@@ -912,8 +917,14 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
             |  CAST('A' AS CHAR(3)),
             |  CAST('b' AS VARCHAR(2))) AS value""".stripMargin)
         assert(simple.schema.head.dataType === StringType)
+        // Hive's simple-UDF conversion from CHAR to a Java String strips trailing spaces.
         checkAnswer(simple, Row("A b"))
 
+        // A GenericUDF reading the HiveChar directly observes the padded CHAR value.
+        checkAnswer(
+          sql("SELECT hive_char_padded(CAST('A' AS CHAR(3)))"),
+          Row("A  "))
+
         val scalar = sql(
           "SELECT hive_upper(CAST('Ab' AS CHAR(5) COLLATE UTF8_LCASE)) AS value")
         assert(scalar.schema.head.dataType === CharType(5))
@@ -1269,6 +1280,20 @@ class ReturnStringGenericUDF extends GenericUDF {
   override def getDisplayString(children: Array[String]): String = "return_string"
 }
 
+class InspectCharGenericUDF extends GenericUDF {
+  private var inspector: HiveCharObjectInspector = _
+
+  override def initialize(arguments: Array[ObjectInspector]): ObjectInspector = {
+    inspector = arguments(0).asInstanceOf[HiveCharObjectInspector]
+    PrimitiveObjectInspectorFactory.javaStringObjectInspector
+  }
+
+  override def evaluate(arguments: Array[DeferredObject]): AnyRef =
+    inspector.getPrimitiveJavaObject(arguments(0).get).getPaddedValue
+
+  override def getDisplayString(children: Array[String]): String = "inspect_char"
+}
+
 @UDFType(stateful = true)
 class StatefulUDF extends UDF {
   private val result = new LongWritable(0)

From 808127c513cffd3eeb128e83a951b7ac38f8608e Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Thu, 17 Sep 2026 05:04:20 +0000
Subject: [PATCH 14/22] fix: [SPARK-59277] rewrite nested JSON map keys
 recursively

transformRecursively stops at the first matching MapType, so nested
integer keys stayed INT while Jackson produced strings.
---
 .../BaseScriptTransformationExec.scala          | 17 +++++++++++++----
 .../BaseScriptTransformationSuite.scala         | 15 +++++++++++++++
 .../spark/sql/hive/hiveUDFEvaluators.scala      |  4 ++--
 3 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index bf3fa7da0934a..02adaddc5df35 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -263,10 +263,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
           if CharVarcharUtils.hasCharVarchar(dt) =>
         val physicalType = ScriptTransformationIOSchema.toUnboundedStringType(dt)
         // JSON object keys are strings. Cast them to the declared map key type after parsing.
-        val jsonType = physicalType.transformRecursively {
-          case map: MapType if !map.keyType.isInstanceOf[StringType] =>
-            map.copy(keyType = StringType)
-        }
+        val jsonType = ScriptTransformationIOSchema.toJsonMapKeyType(physicalType)
         val complexTypeFactory = JsonToStructs(
           jsonType,
           ioschema.outputSerdeProps.toMap,
@@ -429,6 +426,18 @@ object ScriptTransformationIOSchema {
     }
   }
 
+  // JSON object keys are always strings. Rewrite every map key, including nested maps.
+  // `transformRecursively` would stop at the first matching MapType and skip children.
+  private[sql] def toJsonMapKeyType(dataType: DataType): DataType = dataType match {
+    case ArrayType(et, n) => ArrayType(toJsonMapKeyType(et), n)
+    case MapType(kt, vt, n) =>
+      val jsonKey = if (kt.isInstanceOf[StringType]) kt else StringType
+      MapType(jsonKey, toJsonMapKeyType(vt), n)
+    case StructType(fields) =>
+      StructType(fields.map(f => f.copy(dataType = toJsonMapKeyType(f.dataType))))
+    case other => other
+  }
+
   val defaultFormat = Map(
     ("TOK_TABLEROWFORMATFIELD", "\u0001"),
     ("TOK_TABLEROWFORMATLINES", "\n"),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index d6f96de9c6d28..0c7937ca3fac5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -164,6 +164,18 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
           """{"1":"ab"}""",
           MapType(IntegerType, CharType(4)),
           Row(Map(1 -> "ab  "))),
+        (
+          """{"1":{"2":"ab"}}""",
+          MapType(IntegerType, MapType(IntegerType, CharType(4))),
+          Row(Map(1 -> Map(2 -> "ab  ")))),
+        (
+          """[{"1":"ab"}]""",
+          ArrayType(MapType(IntegerType, CharType(4))),
+          Row(Seq(Map(1 -> "ab  ")))),
+        (
+          """{"m":{"1":"ab"}}""",
+          StructType(Seq(StructField("m", MapType(IntegerType, CharType(4))))),
+          Row(Row(Map(1 -> "ab  ")))),
         (
           """{"value":"xy"}""",
           StructType(Seq(StructField("value", CharType(5)))),
@@ -188,6 +200,9 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
       Seq(
         (ArrayType(CharType(4)), """["abcdef"]"""),
         (MapType(IntegerType, CharType(4)), """{"1":"abcdef"}"""),
+        (
+          MapType(IntegerType, MapType(IntegerType, CharType(4))),
+          """{"1":{"2":"abcdef"}}"""),
         (
           StructType(Seq(StructField("value", VarcharType(4)))),
           """{"value":"abcdef"}""")).foreach { case (dataType, json) =>
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
index 5ab96ca7683e3..9e5cdca1e934c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
@@ -114,8 +114,8 @@ class HiveSimpleUDFEvaluator(
 private[hive] object HiveGenericUDFEvaluator extends HiveInspectors {
 
   /**
-   * Driver-side Hive initialize for `SELECT hive_udf(...)`. Stores the Catalyst type on
-   * `HiveGenericUDF.dataType` (for example CHAR(5) from a CHAR inspector).
+   * Driver-side Hive initialize for `SELECT hive_udf(...)`. Returns the Catalyst type
+   * (for example CHAR(5) from a CHAR inspector). `HiveGenericUDF.apply` stores it.
    */
   def inferReturnType(
       funcWrapper: HiveFunctionWrapper,

From 0cc7cdb2bc9c7e1917cc57a0f000d1775a740fdb Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Sat, 19 Sep 2026 01:44:35 +0000
Subject: [PATCH 15/22] fix: [SPARK-59277] address Hive review feedback

---
 .../BaseScriptTransformationExec.scala        | 75 ++++++++++++++++++-
 .../BaseScriptTransformationSuite.scala       | 30 ++++++++
 .../spark/sql/hive/hiveUDFEvaluators.scala    |  2 +-
 .../spark/sql/hive/HiveInspectorSuite.scala   | 21 ++++--
 4 files changed, 119 insertions(+), 9 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index 02adaddc5df35..d65af41f6fe1d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -32,14 +32,24 @@ import org.apache.spark.internal.LogKeys._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.expressions.{
+  ArrayTransform,
   Attribute,
   AttributeSet,
   BoundReference,
   Cast,
+  CreateNamedStruct,
   Expression,
   GenericInternalRow,
+  GetStructField,
+  If,
+  IsNull,
   JsonToStructs,
+  LambdaFunction,
   Literal,
+  MapFromArrays,
+  MapKeys,
+  MapValues,
+  NamedLambdaVariable,
   StructsToJson,
   UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical.ScriptInputOutputSchema
@@ -50,6 +60,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.{CircularBuffer, RedirectThread, SerializableConfiguration, Utils}
+import org.apache.spark.util.ArrayImplicits._
 
 trait BaseScriptTransformationExec extends UnaryExecNode {
   def script: String
@@ -212,7 +223,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
     val converter = CatalystTypeConverters.createToCatalystConverter(attr.dataType)
     attr.dataType match {
       case _: CharType | _: VarcharType =>
-        // First-class CHAR/VARCHAR must not use Hive LazySimpleSerde's null-on-error path.
+        // First-class CHAR/VARCHAR must not use Hive LazySimpleSerDe's null-on-error path.
         (data: String) =>
           if (data == ioschema.outputRowFormatMap("TOK_TABLEROWFORMATNULL")) {
             null
@@ -272,11 +283,11 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
         val parsedToPhysical = if (jsonType.sameType(physicalType)) {
           identity[Any] _
         } else {
-          val cast = Cast(
+          val restoreMapKeys = ScriptTransformationIOSchema.restoreJsonMapKeys(
             BoundReference(0, jsonType, nullable = true),
             physicalType,
             Some(conf.sessionLocalTimeZone))
-          value: Any => cast.eval(InternalRow(value))
+          value: Any => restoreMapKeys.eval(InternalRow(value))
         }
         val toScala = CatalystTypeConverters.createToScalaConverter(physicalType)
         val parser = wrapperConvertException(
@@ -438,6 +449,64 @@ object ScriptTransformationIOSchema {
     case other => other
   }
 
+  private[sql] def restoreJsonMapKeys(
+      expression: Expression,
+      targetType: DataType,
+      timeZoneId: Option[String]): Expression = {
+    def restore(
+        expression: Expression,
+        jsonType: DataType,
+        targetType: DataType): Expression = (jsonType, targetType) match {
+      case (ArrayType(jsonElementType, containsNull), ArrayType(targetElementType, _)) =>
+        val element = NamedLambdaVariable("element", jsonElementType, containsNull)
+        val restoredElement = restore(element, jsonElementType, targetElementType)
+        if (restoredElement.fastEquals(element)) {
+          expression
+        } else {
+          ArrayTransform(expression, LambdaFunction(restoredElement, Seq(element)))
+        }
+      case (
+          MapType(jsonKeyType, jsonValueType, valueContainsNull),
+          MapType(targetKeyType, targetValueType, _)) =>
+        val keys = MapKeys(expression)
+        val key = NamedLambdaVariable("key", jsonKeyType, nullable = false)
+        val restoredKey = if (jsonKeyType.sameType(targetKeyType)) {
+          key
+        } else {
+          Cast(key, targetKeyType, timeZoneId)
+        }
+        val restoredKeys = ArrayTransform(keys, LambdaFunction(restoredKey, Seq(key)))
+
+        val values = MapValues(expression)
+        val value = NamedLambdaVariable("value", jsonValueType, valueContainsNull)
+        val restoredValue = restore(value, jsonValueType, targetValueType)
+        val restoredValues = if (restoredValue.fastEquals(value)) {
+          values
+        } else {
+          ArrayTransform(values, LambdaFunction(restoredValue, Seq(value)))
+        }
+        MapFromArrays(restoredKeys, restoredValues)
+      case (jsonStruct: StructType, targetStruct: StructType) =>
+        val fields = targetStruct.fields.zipWithIndex.flatMap { case (field, index) =>
+          Seq(
+            Literal(field.name),
+            restore(
+              GetStructField(expression, index, Some(field.name)),
+              jsonStruct(index).dataType,
+              field.dataType))
+        }
+        val restoredStruct = CreateNamedStruct(fields.toImmutableArraySeq)
+        if (expression.nullable) {
+          If(IsNull(expression), Literal(null, restoredStruct.dataType), restoredStruct)
+        } else {
+          restoredStruct
+        }
+      case _ => expression
+    }
+
+    restore(expression, expression.dataType, targetType)
+  }
+
   val defaultFormat = Map(
     ("TOK_TABLEROWFORMATFIELD", "\u0001"),
     ("TOK_TABLEROWFORMATLINES", "\n"),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index 0c7937ca3fac5..d9a93ae04a703 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -246,6 +246,36 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
 
+  test("SPARK-59277: TRANSFORM validates restored JSON map keys without SerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val mapType = MapType(IntegerType, CharType(4))
+      Seq(
+        ("""{"1":"ab"}""", mapType, Row(Map(1 -> "ab  "))),
+        ("""{"not-an-int":"ab"}""", mapType, Row(null)),
+        ("""{"1":"a","01":"b"}""", mapType, Row(null)),
+        (
+          """[{"not-an-int":"ab"}]""",
+          ArrayType(mapType),
+          Row(null)),
+        (
+          """{"m":{"1":"a","01":"b"}}""",
+          StructType(Seq(StructField("m", mapType))),
+          Row(null))).foreach { case (json, dataType, expected) =>
+        val input = Seq(json).toDF("value")
+        checkAnswer(
+          input,
+          (child: SparkPlan) => createScriptTransformationExec(
+            script = "cat",
+            output = Seq(AttributeReference("value", dataType)()),
+            child = child,
+            ioschema = defaultIOSchema),
+          Seq(expected))
+      }
+    }
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
   test("script transformation should not swallow errors from upstream operators (no serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
index 9e5cdca1e934c..a24767701376d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala
@@ -114,7 +114,7 @@ class HiveSimpleUDFEvaluator(
 private[hive] object HiveGenericUDFEvaluator extends HiveInspectors {
 
   /**
-   * Driver-side Hive initialize for `SELECT hive_udf(...)`. Returns the Catalyst type
+   * Driver-side Hive initialization for `SELECT hive_udf(...)`. Returns the Catalyst type
    * (for example CHAR(5) from a CHAR inspector). `HiveGenericUDF.apply` stores it.
    */
   def inferReturnType(
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 0650a540f30b5..8b49a289d1e03 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -466,15 +466,25 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
     checkCompatibleHiveReturnType(ArrayType(StringType), ArrayType(VarcharType(7)))
     checkCompatibleHiveReturnType(CharType(5), CharType(5))
     checkCompatibleHiveReturnType(VarcharType(3), VarcharType(3))
+    checkCompatibleHiveReturnType(
+      MapType(StringType, VarcharType(3)),
+      MapType(CharType(5), StringType))
+    checkCompatibleHiveReturnType(
+      StructType.fromDDL("c STRING, v VARCHAR(3)"),
+      StructType.fromDDL("c CHAR(5), v STRING"))
 
     Seq[(DataType, DataType)](
       VarcharType(3) -> CharType(5),
       CharType(4) -> CharType(5),
       VarcharType(4) -> VarcharType(5),
-      IntegerType -> CharType(5)).foreach { case (runtimeType, expectedType) =>
-      intercept[SparkException] {
-        checkCompatibleHiveReturnType(runtimeType, expectedType)
-      }
+      IntegerType -> CharType(5),
+      MapType(CharType(4), StringType) -> MapType(CharType(5), StringType),
+      StructType.fromDDL("c STRING") -> StructType.fromDDL("c STRING, v STRING"),
+      StructType.fromDDL("c CHAR(4)") -> StructType.fromDDL("c CHAR(5)")).foreach {
+      case (runtimeType, expectedType) =>
+        intercept[SparkException] {
+          checkCompatibleHiveReturnType(runtimeType, expectedType)
+        }
     }
   }
 
@@ -521,7 +531,8 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
 
   test("SPARK-59277: Hive object inspectors reject unsupported CHAR/VARCHAR lengths") {
     withFirstClassCharVarchar(enabled = true) {
-      Seq[DataType](CharType(0), CharType(256), VarcharType(65536)).foreach { dataType =>
+      Seq[DataType](
+        CharType(0), CharType(256), VarcharType(0), VarcharType(65536)).foreach { dataType =>
         val expectedParams = Map("typeName" -> s"\"${dataType.sql}\"")
         checkError(
           exception = intercept[AnalysisException](toInspector(dataType)),

From 05da5e1d2568ab2b4ee12da5f6ab6edb886b2107 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Sat, 19 Sep 2026 17:35:58 +0000
Subject: [PATCH 16/22] fix: [SPARK-59277] fresh map builder per row in
 no-SerDe key restoration

---
 .../BaseScriptTransformationExec.scala        | 141 ++++++++++--------
 .../BaseScriptTransformationSuite.scala       |  21 +++
 2 files changed, 96 insertions(+), 66 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index d65af41f6fe1d..068a495de2d12 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -32,35 +32,31 @@ import org.apache.spark.internal.LogKeys._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.expressions.{
-  ArrayTransform,
   Attribute,
   AttributeSet,
   BoundReference,
   Cast,
-  CreateNamedStruct,
   Expression,
   GenericInternalRow,
-  GetStructField,
-  If,
-  IsNull,
   JsonToStructs,
-  LambdaFunction,
   Literal,
-  MapFromArrays,
-  MapKeys,
-  MapValues,
-  NamedLambdaVariable,
   StructsToJson,
   UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical.ScriptInputOutputSchema
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
-import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, DateTimeUtils, IntervalUtils}
+import org.apache.spark.sql.catalyst.util.{
+  ArrayBasedMapBuilder,
+  ArrayData,
+  CharVarcharUtils,
+  DateTimeUtils,
+  GenericArrayData,
+  IntervalUtils,
+  MapData}
 import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.{CircularBuffer, RedirectThread, SerializableConfiguration, Utils}
-import org.apache.spark.util.ArrayImplicits._
 
 trait BaseScriptTransformationExec extends UnaryExecNode {
   def script: String
@@ -283,11 +279,9 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
         val parsedToPhysical = if (jsonType.sameType(physicalType)) {
           identity[Any] _
         } else {
-          val restoreMapKeys = ScriptTransformationIOSchema.restoreJsonMapKeys(
-            BoundReference(0, jsonType, nullable = true),
-            physicalType,
-            Some(conf.sessionLocalTimeZone))
-          value: Any => restoreMapKeys.eval(InternalRow(value))
+          val restoreMapKeys = ScriptTransformationIOSchema.makeJsonMapKeyRestorer(
+            jsonType, physicalType, Some(conf.sessionLocalTimeZone))
+          value: Any => restoreMapKeys(value)
         }
         val toScala = CatalystTypeConverters.createToScalaConverter(physicalType)
         val parser = wrapperConvertException(
@@ -307,7 +301,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
     }
   }
 
-  // Keep consistent with Hive `LazySimpleSerde`, when there is a type case error, return null
+  // Keep consistent with Hive `LazySimpleSerDe`, when there is a type case error, return null
   private val wrapperConvertException: (String => Any, Any => Any) => String => Any =
     (f: String => Any, converter: Any => Any) =>
       (data: String) => converter {
@@ -449,62 +443,77 @@ object ScriptTransformationIOSchema {
     case other => other
   }
 
-  private[sql] def restoreJsonMapKeys(
-      expression: Expression,
+  /**
+   * Build a per-call map-key restorer that converts parsed JSON string keys
+   * back to the declared physical key type and validates the result through a
+   * fresh [[ArrayBasedMapBuilder]] on every invocation, so a failed or
+   * duplicate key cannot leave shared state dirty for the next row.
+   */
+  private[sql] def makeJsonMapKeyRestorer(
+      jsonType: DataType,
       targetType: DataType,
-      timeZoneId: Option[String]): Expression = {
-    def restore(
-        expression: Expression,
-        jsonType: DataType,
-        targetType: DataType): Expression = (jsonType, targetType) match {
-      case (ArrayType(jsonElementType, containsNull), ArrayType(targetElementType, _)) =>
-        val element = NamedLambdaVariable("element", jsonElementType, containsNull)
-        val restoredElement = restore(element, jsonElementType, targetElementType)
-        if (restoredElement.fastEquals(element)) {
-          expression
-        } else {
-          ArrayTransform(expression, LambdaFunction(restoredElement, Seq(element)))
-        }
-      case (
-          MapType(jsonKeyType, jsonValueType, valueContainsNull),
-          MapType(targetKeyType, targetValueType, _)) =>
-        val keys = MapKeys(expression)
-        val key = NamedLambdaVariable("key", jsonKeyType, nullable = false)
-        val restoredKey = if (jsonKeyType.sameType(targetKeyType)) {
-          key
-        } else {
-          Cast(key, targetKeyType, timeZoneId)
+      timeZoneId: Option[String]): Any => Any = {
+
+    def make(jt: DataType, tt: DataType): Any => Any = (jt, tt) match {
+      case (ArrayType(jet, _), ArrayType(tet, _)) =>
+        val elem = make(jet, tet)
+        (input: Any) => {
+          val arr = input.asInstanceOf[ArrayData]
+          val n = arr.numElements()
+          val out = new Array[Any](n)
+          var i = 0
+          while (i < n) {
+            out(i) = if (arr.isNullAt(i)) null
+              else elem(arr.get(i, jet))
+            i += 1
+          }
+          new GenericArrayData(out)
         }
-        val restoredKeys = ArrayTransform(keys, LambdaFunction(restoredKey, Seq(key)))
 
-        val values = MapValues(expression)
-        val value = NamedLambdaVariable("value", jsonValueType, valueContainsNull)
-        val restoredValue = restore(value, jsonValueType, targetValueType)
-        val restoredValues = if (restoredValue.fastEquals(value)) {
-          values
-        } else {
-          ArrayTransform(values, LambdaFunction(restoredValue, Seq(value)))
+      case (MapType(jkt, jvt, _), MapType(tkt, tvt, _)) =>
+        val keyCast: Any => Any = if (jkt.sameType(tkt)) identity
+          else {
+            val c = Cast(BoundReference(0, jkt, nullable = false),
+              tkt, timeZoneId)
+            (k: Any) => c.eval(InternalRow(k))
+          }
+        val valRestore = make(jvt, tvt)
+        (input: Any) => {
+          val map = input.asInstanceOf[MapData]
+          val n = map.numElements()
+          val builder = new ArrayBasedMapBuilder(tkt, tvt)
+          var i = 0
+          while (i < n) {
+            val k = keyCast(map.keyArray().get(i, jkt))
+            val v = if (map.valueArray().isNullAt(i)) null
+              else valRestore(map.valueArray().get(i, jvt))
+            builder.put(k, v)
+            i += 1
+          }
+          builder.build()
         }
-        MapFromArrays(restoredKeys, restoredValues)
-      case (jsonStruct: StructType, targetStruct: StructType) =>
-        val fields = targetStruct.fields.zipWithIndex.flatMap { case (field, index) =>
-          Seq(
-            Literal(field.name),
-            restore(
-              GetStructField(expression, index, Some(field.name)),
-              jsonStruct(index).dataType,
-              field.dataType))
+
+      case (js: StructType, ts: StructType) =>
+        val restorers = js.fields.zip(ts.fields).map {
+          case (jf, tf) => make(jf.dataType, tf.dataType)
         }
-        val restoredStruct = CreateNamedStruct(fields.toImmutableArraySeq)
-        if (expression.nullable) {
-          If(IsNull(expression), Literal(null, restoredStruct.dataType), restoredStruct)
-        } else {
-          restoredStruct
+        (input: Any) => {
+          val row = input.asInstanceOf[InternalRow]
+          val out = new GenericInternalRow(ts.length)
+          var i = 0
+          while (i < ts.length) {
+            if (row.isNullAt(i)) out.setNullAt(i)
+            else out.update(i,
+              restorers(i)(row.get(i, js(i).dataType)))
+            i += 1
+          }
+          out
         }
-      case _ => expression
+
+      case _ => identity
     }
 
-    restore(expression, expression.dataType, targetType)
+    make(jsonType, targetType)
   }
 
   val defaultFormat = Map(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index d9a93ae04a703..bd6b844df86e0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -276,6 +276,27 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
 
+  test("SPARK-59277: colliding map key followed by valid row without SerDe") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      val mapType = MapType(IntegerType, CharType(4))
+      // Row 1 has duplicate converted keys (1 and 01 both cast to 1).
+      // Row 2 is valid. Both rows are in the same partition.
+      val input = Seq(
+        """{"1":"a","01":"b"}""",
+        """{"2":"cd"}""").toDF("value")
+      checkAnswer(
+        input,
+        (child: SparkPlan) => createScriptTransformationExec(
+          script = "cat",
+          output = Seq(AttributeReference("value", mapType)()),
+          child = child,
+          ioschema = defaultIOSchema),
+        Seq(Row(null), Row(Map(2 -> "cd  "))))
+    }
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
   test("script transformation should not swallow errors from upstream operators (no serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
 

From d9044026aa2d54a3ced03181a23d7942a6033b12 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Sat, 19 Sep 2026 17:52:03 +0000
Subject: [PATCH 17/22] fix: [SPARK-59277] tighten restorer API and force
 single-partition test

---
 .../sql/execution/BaseScriptTransformationExec.scala   | 10 +++++-----
 .../sql/execution/BaseScriptTransformationSuite.scala  |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index 068a495de2d12..fc47a2e21b944 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -280,7 +280,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
           identity[Any] _
         } else {
           val restoreMapKeys = ScriptTransformationIOSchema.makeJsonMapKeyRestorer(
-            jsonType, physicalType, Some(conf.sessionLocalTimeZone))
+            physicalType, Some(conf.sessionLocalTimeZone))
           value: Any => restoreMapKeys(value)
         }
         val toScala = CatalystTypeConverters.createToScalaConverter(physicalType)
@@ -301,7 +301,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
     }
   }
 
-  // Keep consistent with Hive `LazySimpleSerDe`, when there is a type case error, return null
+  // Match Hive `LazySimpleSerDe`: return null when a type cast fails.
   private val wrapperConvertException: (String => Any, Any => Any) => String => Any =
     (f: String => Any, converter: Any => Any) =>
       (data: String) => converter {
@@ -446,13 +446,13 @@ object ScriptTransformationIOSchema {
   /**
    * Build a per-call map-key restorer that converts parsed JSON string keys
    * back to the declared physical key type and validates the result through a
-   * fresh [[ArrayBasedMapBuilder]] on every invocation, so a failed or
-   * duplicate key cannot leave shared state dirty for the next row.
+   * fresh [[ArrayBasedMapBuilder]] on every invocation, so a failed key
+   * conversion or duplicate key cannot leave shared state dirty for the next row.
    */
   private[sql] def makeJsonMapKeyRestorer(
-      jsonType: DataType,
       targetType: DataType,
       timeZoneId: Option[String]): Any => Any = {
+    val jsonType = toJsonMapKeyType(targetType)
 
     def make(jt: DataType, tt: DataType): Any => Any = (jt, tt) match {
       case (ArrayType(jet, _), ArrayType(tet, _)) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index bd6b844df86e0..d9f2902472087 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -284,7 +284,7 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
       // Row 2 is valid. Both rows are in the same partition.
       val input = Seq(
         """{"1":"a","01":"b"}""",
-        """{"2":"cd"}""").toDF("value")
+        """{"2":"cd"}""").toDF("value").coalesce(1)
       checkAnswer(
         input,
         (child: SparkPlan) => createScriptTransformationExec(

From f1230d5662e5b70f8412debfa8e6e70ae3c18985 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Mon, 21 Sep 2026 05:14:32 +0000
Subject: [PATCH 18/22] refactor: [SPARK-59277] move script TRANSFORM
 CHAR/VARCHAR support to follow-up PR

Script TRANSFORM support (BaseScriptTransformationExec, HiveScriptTransformationExec,
and their tests) is moved to a separate JIRA subtask as suggested in review. This PR
now covers only Hive inspector conversion and UDF/UDAF/UDTF support.
---
 .../BaseScriptTransformationExec.scala        | 148 +-----------
 .../BaseScriptTransformationSuite.scala       | 211 ------------------
 .../HiveScriptTransformationExec.scala        |  26 +--
 .../HiveScriptTransformationSuite.scala       | 167 --------------
 4 files changed, 7 insertions(+), 545 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index fc47a2e21b944..a313e2c671bec 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -31,27 +31,10 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.LogKeys._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{
-  Attribute,
-  AttributeSet,
-  BoundReference,
-  Cast,
-  Expression,
-  GenericInternalRow,
-  JsonToStructs,
-  Literal,
-  StructsToJson,
-  UnsafeProjection}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Cast, Expression, GenericInternalRow, JsonToStructs, Literal, StructsToJson, UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical.ScriptInputOutputSchema
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
-import org.apache.spark.sql.catalyst.util.{
-  ArrayBasedMapBuilder,
-  ArrayData,
-  CharVarcharUtils,
-  DateTimeUtils,
-  GenericArrayData,
-  IntervalUtils,
-  MapData}
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils}
 import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -218,15 +201,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
   private lazy val outputFieldWriters: Seq[String => Any] = output.map { attr =>
     val converter = CatalystTypeConverters.createToCatalystConverter(attr.dataType)
     attr.dataType match {
-      case _: CharType | _: VarcharType =>
-        // First-class CHAR/VARCHAR must not use Hive LazySimpleSerDe's null-on-error path.
-        (data: String) =>
-          if (data == ioschema.outputRowFormatMap("TOK_TABLEROWFORMATNULL")) {
-            null
-          } else {
-            converter(data)
-          }
-      case _: StringType => wrapperConvertException(data => data, converter)
+      case StringType => wrapperConvertException(data => data, converter)
       case BooleanType => wrapperConvertException(data => data.toBoolean, converter)
       case ByteType => wrapperConvertException(data => data.toByte, converter)
       case BinaryType =>
@@ -266,29 +241,6 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
         data => IntervalUtils.microsToDuration(
           IntervalUtils.castStringToDTInterval(UTF8String.fromString(data), start, end)),
         converter)
-      case dt @ (_: ArrayType | _: MapType | _: StructType)
-          if CharVarcharUtils.hasCharVarchar(dt) =>
-        val physicalType = ScriptTransformationIOSchema.toUnboundedStringType(dt)
-        // JSON object keys are strings. Cast them to the declared map key type after parsing.
-        val jsonType = ScriptTransformationIOSchema.toJsonMapKeyType(physicalType)
-        val complexTypeFactory = JsonToStructs(
-          jsonType,
-          ioschema.outputSerdeProps.toMap,
-          Literal(null),
-          Some(conf.sessionLocalTimeZone))
-        val parsedToPhysical = if (jsonType.sameType(physicalType)) {
-          identity[Any] _
-        } else {
-          val restoreMapKeys = ScriptTransformationIOSchema.makeJsonMapKeyRestorer(
-            physicalType, Some(conf.sessionLocalTimeZone))
-          value: Any => restoreMapKeys(value)
-        }
-        val toScala = CatalystTypeConverters.createToScalaConverter(physicalType)
-        val parser = wrapperConvertException(
-          data => parsedToPhysical(
-            complexTypeFactory.nullSafeEval(UTF8String.fromString(data))),
-          identity)
-        data => converter(toScala(parser(data)))
       case _: ArrayType | _: MapType | _: StructType =>
         val complexTypeFactory = JsonToStructs(attr.dataType,
           ioschema.outputSerdeProps.toMap, Literal(null), Some(conf.sessionLocalTimeZone))
@@ -301,7 +253,7 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
     }
   }
 
-  // Match Hive `LazySimpleSerDe`: return null when a type cast fails.
+  // Keep consistent with Hive `LazySimpleSerde`, when there is a type case error, return null
   private val wrapperConvertException: (String => Any, Any => Any) => String => Any =
     (f: String => Any, converter: Any => Any) =>
       (data: String) => converter {
@@ -424,98 +376,6 @@ case class ScriptTransformationIOSchema(
 }
 
 object ScriptTransformationIOSchema {
-  private[sql] def toUnboundedStringType(dataType: DataType): DataType = {
-    dataType.transformRecursively {
-      case c: CharType => c.toStringType
-      case v: VarcharType => v.toStringType
-    }
-  }
-
-  // JSON object keys are always strings. Rewrite every map key, including nested maps.
-  // `transformRecursively` would stop at the first matching MapType and skip children.
-  private[sql] def toJsonMapKeyType(dataType: DataType): DataType = dataType match {
-    case ArrayType(et, n) => ArrayType(toJsonMapKeyType(et), n)
-    case MapType(kt, vt, n) =>
-      val jsonKey = if (kt.isInstanceOf[StringType]) kt else StringType
-      MapType(jsonKey, toJsonMapKeyType(vt), n)
-    case StructType(fields) =>
-      StructType(fields.map(f => f.copy(dataType = toJsonMapKeyType(f.dataType))))
-    case other => other
-  }
-
-  /**
-   * Build a per-call map-key restorer that converts parsed JSON string keys
-   * back to the declared physical key type and validates the result through a
-   * fresh [[ArrayBasedMapBuilder]] on every invocation, so a failed key
-   * conversion or duplicate key cannot leave shared state dirty for the next row.
-   */
-  private[sql] def makeJsonMapKeyRestorer(
-      targetType: DataType,
-      timeZoneId: Option[String]): Any => Any = {
-    val jsonType = toJsonMapKeyType(targetType)
-
-    def make(jt: DataType, tt: DataType): Any => Any = (jt, tt) match {
-      case (ArrayType(jet, _), ArrayType(tet, _)) =>
-        val elem = make(jet, tet)
-        (input: Any) => {
-          val arr = input.asInstanceOf[ArrayData]
-          val n = arr.numElements()
-          val out = new Array[Any](n)
-          var i = 0
-          while (i < n) {
-            out(i) = if (arr.isNullAt(i)) null
-              else elem(arr.get(i, jet))
-            i += 1
-          }
-          new GenericArrayData(out)
-        }
-
-      case (MapType(jkt, jvt, _), MapType(tkt, tvt, _)) =>
-        val keyCast: Any => Any = if (jkt.sameType(tkt)) identity
-          else {
-            val c = Cast(BoundReference(0, jkt, nullable = false),
-              tkt, timeZoneId)
-            (k: Any) => c.eval(InternalRow(k))
-          }
-        val valRestore = make(jvt, tvt)
-        (input: Any) => {
-          val map = input.asInstanceOf[MapData]
-          val n = map.numElements()
-          val builder = new ArrayBasedMapBuilder(tkt, tvt)
-          var i = 0
-          while (i < n) {
-            val k = keyCast(map.keyArray().get(i, jkt))
-            val v = if (map.valueArray().isNullAt(i)) null
-              else valRestore(map.valueArray().get(i, jvt))
-            builder.put(k, v)
-            i += 1
-          }
-          builder.build()
-        }
-
-      case (js: StructType, ts: StructType) =>
-        val restorers = js.fields.zip(ts.fields).map {
-          case (jf, tf) => make(jf.dataType, tf.dataType)
-        }
-        (input: Any) => {
-          val row = input.asInstanceOf[InternalRow]
-          val out = new GenericInternalRow(ts.length)
-          var i = 0
-          while (i < ts.length) {
-            if (row.isNullAt(i)) out.setNullAt(i)
-            else out.update(i,
-              restorers(i)(row.get(i, js(i).dataType)))
-            i += 1
-          }
-          out
-        }
-
-      case _ => identity
-    }
-
-    make(jsonType, targetType)
-  }
-
   val defaultFormat = Map(
     ("TOK_TABLEROWFORMATFIELD", "\u0001"),
     ("TOK_TABLEROWFORMATLINES", "\n"),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
index d9f2902472087..62c5f5631776b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala
@@ -86,217 +86,6 @@ abstract class BaseScriptTransformationSuite extends QueryTest {
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
 
-  test("SPARK-59277: TRANSFORM output supports first-class CHAR/VARCHAR without SerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val input = Seq(("ab", "xyz")).toDF("c", "v")
-      checkAnswer(
-        input,
-        (child: SparkPlan) => createScriptTransformationExec(
-          script = "cat",
-          output = Seq(
-            AttributeReference("c", CharType(4, "UTF8_LCASE"))(),
-            AttributeReference("v", VarcharType(5, "UNICODE_CI"))()),
-          child = child,
-          ioschema = defaultIOSchema),
-        Seq(Row("ab  ", "xyz")))
-    }
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("SPARK-59277: TRANSFORM CHAR overflow without SerDe raises EXCEED_LIMIT_LENGTH") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val input = Seq("abcdef").toDF("c")
-      val exception = intercept[Exception] {
-        QueryTest.executePlan(
-          createScriptTransformationExec(
-            script = "cat",
-            output = Seq(AttributeReference("c", CharType(4))()),
-            child = input.queryExecution.sparkPlan,
-            ioschema = defaultIOSchema),
-          spark.sqlContext)
-      }
-      val runtimeException = exception match {
-        case s: org.apache.spark.SparkRuntimeException => s
-        case other =>
-          other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
-      }
-      checkError(
-        exception = runtimeException,
-        condition = "EXCEED_LIMIT_LENGTH",
-        parameters = Map("limit" -> "4"))
-    }
-  }
-
-  test("SPARK-59277: TRANSFORM VARCHAR overflow without SerDe raises EXCEED_LIMIT_LENGTH") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val input = Seq("abcdefgh").toDF("v")
-      val exception = intercept[Exception] {
-        QueryTest.executePlan(
-          createScriptTransformationExec(
-            script = "cat",
-            output = Seq(AttributeReference("v", VarcharType(5))()),
-            child = input.queryExecution.sparkPlan,
-            ioschema = defaultIOSchema),
-          spark.sqlContext)
-      }
-      val runtimeException = exception match {
-        case s: org.apache.spark.SparkRuntimeException => s
-        case other =>
-          other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
-      }
-      checkError(
-        exception = runtimeException,
-        condition = "EXCEED_LIMIT_LENGTH",
-        parameters = Map("limit" -> "5"))
-    }
-  }
-
-  test("SPARK-59277: TRANSFORM converts nested CHAR/VARCHAR without SerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      Seq(
-        ("""["ab"]""", ArrayType(CharType(4)), Row(Seq("ab  "))),
-        ("""["xy"]""", ArrayType(VarcharType(4)), Row(Seq("xy"))),
-        (
-          """{"1":"ab"}""",
-          MapType(IntegerType, CharType(4)),
-          Row(Map(1 -> "ab  "))),
-        (
-          """{"1":{"2":"ab"}}""",
-          MapType(IntegerType, MapType(IntegerType, CharType(4))),
-          Row(Map(1 -> Map(2 -> "ab  ")))),
-        (
-          """[{"1":"ab"}]""",
-          ArrayType(MapType(IntegerType, CharType(4))),
-          Row(Seq(Map(1 -> "ab  ")))),
-        (
-          """{"m":{"1":"ab"}}""",
-          StructType(Seq(StructField("m", MapType(IntegerType, CharType(4))))),
-          Row(Row(Map(1 -> "ab  ")))),
-        (
-          """{"value":"xy"}""",
-          StructType(Seq(StructField("value", CharType(5)))),
-          Row(Row("xy   ")))).foreach { case (json, dataType, expected) =>
-        val input = Seq(json).toDF("value")
-        checkAnswer(
-          input,
-          (child: SparkPlan) => createScriptTransformationExec(
-            script = "cat",
-            output = Seq(AttributeReference("value", dataType)()),
-            child = child,
-            ioschema = defaultIOSchema),
-          Seq(expected))
-      }
-    }
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("SPARK-59277: TRANSFORM nested CHAR/VARCHAR overflow without SerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      Seq(
-        (ArrayType(CharType(4)), """["abcdef"]"""),
-        (MapType(IntegerType, CharType(4)), """{"1":"abcdef"}"""),
-        (
-          MapType(IntegerType, MapType(IntegerType, CharType(4))),
-          """{"1":{"2":"abcdef"}}"""),
-        (
-          StructType(Seq(StructField("value", VarcharType(4)))),
-          """{"value":"abcdef"}""")).foreach { case (dataType, json) =>
-        val input = Seq(json).toDF("value")
-        val exception = intercept[Exception] {
-          QueryTest.executePlan(
-            createScriptTransformationExec(
-              script = "cat",
-              output = Seq(AttributeReference("value", dataType)()),
-              child = input.queryExecution.sparkPlan,
-              ioschema = defaultIOSchema),
-            spark.sqlContext)
-        }
-        val runtimeException = exception match {
-          case s: org.apache.spark.SparkRuntimeException => s
-          case other =>
-            other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
-        }
-        checkError(
-          exception = runtimeException,
-          condition = "EXCEED_LIMIT_LENGTH",
-          parameters = Map("limit" -> "4"))
-      }
-    }
-  }
-
-  test("SPARK-59277: malformed nested CHAR JSON without SerDe returns null") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val input = Seq("""{"1":""").toDF("value")
-      checkAnswer(
-        input,
-        (child: SparkPlan) => createScriptTransformationExec(
-          script = "cat",
-          output = Seq(
-            AttributeReference("value", MapType(IntegerType, CharType(4)))()),
-          child = child,
-          ioschema = defaultIOSchema),
-        Seq(Row(null)))
-    }
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("SPARK-59277: TRANSFORM validates restored JSON map keys without SerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val mapType = MapType(IntegerType, CharType(4))
-      Seq(
-        ("""{"1":"ab"}""", mapType, Row(Map(1 -> "ab  "))),
-        ("""{"not-an-int":"ab"}""", mapType, Row(null)),
-        ("""{"1":"a","01":"b"}""", mapType, Row(null)),
-        (
-          """[{"not-an-int":"ab"}]""",
-          ArrayType(mapType),
-          Row(null)),
-        (
-          """{"m":{"1":"a","01":"b"}}""",
-          StructType(Seq(StructField("m", mapType))),
-          Row(null))).foreach { case (json, dataType, expected) =>
-        val input = Seq(json).toDF("value")
-        checkAnswer(
-          input,
-          (child: SparkPlan) => createScriptTransformationExec(
-            script = "cat",
-            output = Seq(AttributeReference("value", dataType)()),
-            child = child,
-            ioschema = defaultIOSchema),
-          Seq(expected))
-      }
-    }
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
-  test("SPARK-59277: colliding map key followed by valid row without SerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val mapType = MapType(IntegerType, CharType(4))
-      // Row 1 has duplicate converted keys (1 and 01 both cast to 1).
-      // Row 2 is valid. Both rows are in the same partition.
-      val input = Seq(
-        """{"1":"a","01":"b"}""",
-        """{"2":"cd"}""").toDF("value").coalesce(1)
-      checkAnswer(
-        input,
-        (child: SparkPlan) => createScriptTransformationExec(
-          script = "cat",
-          output = Seq(AttributeReference("value", mapType)()),
-          child = child,
-          ioschema = defaultIOSchema),
-        Seq(Row(null), Row(Map(2 -> "cd  "))))
-    }
-    assert(uncaughtExceptionHandler.exception.isEmpty)
-  }
-
   test("script transformation should not swallow errors from upstream operators (no serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
index 107530f8ffbce..de2d15415837a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.hive.ql.exec.{RecordReader, RecordWriter}
 import org.apache.hadoop.hive.serde.serdeConstants
 import org.apache.hadoop.hive.serde2.AbstractSerDe
-import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
 import org.apache.hadoop.hive.serde2.objectinspector._
 import org.apache.hadoop.io.Writable
 
@@ -37,7 +36,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.hive.HiveInspectors
 import org.apache.spark.sql.hive.HiveShim._
-import org.apache.spark.sql.types._
+import org.apache.spark.sql.types.DataType
 import org.apache.spark.util.{CircularBuffer, Utils}
 
 /**
@@ -76,9 +75,7 @@ private[hive] case class HiveScriptTransformationExec(
       val mutableRow = new SpecificInternalRow(output.map(_.dataType))
 
       @transient
-      lazy val unwrappers = outputSoi.getAllStructFieldRefs.asScala.zip(output).map {
-        case (field, attr) => unwrapperFor(field, attr.dataType)
-      }
+      lazy val unwrappers = outputSoi.getAllStructFieldRefs.asScala.map(unwrapperFor)
 
       override def hasNext: Boolean = {
         if (completed) {
@@ -259,8 +256,7 @@ object HiveScriptIOSchema extends HiveInspectors {
       output: Seq[Attribute]): Option[(AbstractSerDe, StructObjectInspector)] = {
     ioschema.outputSerdeClass.map { serdeClass =>
       val (columns, columnTypes) = parseAttrs(output)
-      val serdeTypes = outputTypesForSerDe(serdeClass, columnTypes)
-      val serde = initSerDe(serdeClass, columns, serdeTypes, ioschema.outputSerdeProps)
+      val serde = initSerDe(serdeClass, columns, columnTypes, ioschema.outputSerdeProps)
       val structObjectInspector = serde.getObjectInspector().asInstanceOf[StructObjectInspector]
       (serde, structObjectInspector)
     }
@@ -272,22 +268,6 @@ object HiveScriptIOSchema extends HiveInspectors {
     (columns, columnTypes)
   }
 
-  /**
-   * Hive LazySimpleSerDe CHAR/VARCHAR types truncate on deserialize. Map them to STRING so
-   * Spark applies first-class length checks. Only LazySimpleSerDe and subclasses get this
-   * rewrite; other SerDes keep the declared CHAR/VARCHAR schema.
-   */
-  private def outputTypesForSerDe(
-      serdeClassName: String,
-      columnTypes: Seq[DataType]): Seq[DataType] = {
-    val serdeClass = Utils.classForName[AbstractSerDe](serdeClassName)
-    if (classOf[LazySimpleSerDe].isAssignableFrom(serdeClass)) {
-      columnTypes.map(ScriptTransformationIOSchema.toUnboundedStringType)
-    } else {
-      columnTypes
-    }
-  }
-
   def initSerDe(
       serdeClassName: String,
       columns: Seq[String],
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
index 5772784795fd1..b1ff05b8c1b06 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -20,15 +20,8 @@ package org.apache.spark.sql.hive.execution
 import java.sql.Timestamp
 import java.time.{Duration, Period}
 import java.time.temporal.ChronoUnit
-import java.util.{Arrays, Properties}
 
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hive.serde.serdeConstants
-import org.apache.hadoop.hive.serde2.{AbstractSerDe, SerDeStats}
 import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
-import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectInspectorFactory}
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
-import org.apache.hadoop.io.{Text, Writable}
 import org.scalatest.exceptions.TestFailedException
 
 import org.apache.spark.{SparkException, TestUtils}
@@ -38,7 +31,6 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.types.DayTimeIntervalType._
 import org.apache.spark.sql.types.YearMonthIntervalType._
@@ -380,139 +372,6 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
     }
   }
 
-  test("SPARK-59277: TRANSFORM supports nested collated CHAR/VARCHAR with Hive SerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val query = sql(
-        """
-          |SELECT TRANSFORM(
-          |  array(CAST('ab' AS CHAR(4) COLLATE UTF8_LCASE)),
-          |  named_struct('value', CAST('xyz' AS VARCHAR(6) COLLATE UNICODE_CI)))
-          |USING 'cat'
-          |AS (
-          |  chars ARRAY<CHAR(4) COLLATE UTF8_LCASE>,
-          |  nested STRUCT<value: VARCHAR(6) COLLATE UNICODE_CI>)
-          |FROM VALUES (1) input(dummy)
-          |""".stripMargin)
-      assert(query.schema.map(_.dataType) === Seq(
-        ArrayType(CharType(4, "UTF8_LCASE")),
-        StructType(Seq(StructField("value", VarcharType(6, "UNICODE_CI"))))))
-      checkAnswer(query, Row(Seq("ab  "), Row("xyz")))
-    }
-  }
-
-  test("SPARK-59277: TRANSFORM CHAR overflow with Hive SerDe raises EXCEED_LIMIT_LENGTH") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val exception = intercept[Exception] {
-        sql(
-          """
-            |SELECT TRANSFORM('abcdef')
-            |USING 'cat'
-            |AS (c CHAR(4))
-            |FROM VALUES (1) input(dummy)
-            |""".stripMargin).collect()
-      }
-      val runtimeException = exception match {
-        case s: org.apache.spark.SparkRuntimeException => s
-        case other =>
-          other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
-      }
-      checkError(
-        exception = runtimeException,
-        condition = "EXCEED_LIMIT_LENGTH",
-        parameters = Map("limit" -> "4"))
-    }
-  }
-
-  test("SPARK-59277: TRANSFORM VARCHAR overflow with Hive SerDe raises EXCEED_LIMIT_LENGTH") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val exception = intercept[Exception] {
-        sql(
-          """
-            |SELECT TRANSFORM('abcdefgh')
-            |USING 'cat'
-            |AS (v VARCHAR(5))
-            |FROM VALUES (1) input(dummy)
-            |""".stripMargin).collect()
-      }
-      val runtimeException = exception match {
-        case s: org.apache.spark.SparkRuntimeException => s
-        case other =>
-          other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
-      }
-      checkError(
-        exception = runtimeException,
-        condition = "EXCEED_LIMIT_LENGTH",
-        parameters = Map("limit" -> "5"))
-    }
-  }
-
-  test("SPARK-59277: nested CHAR/VARCHAR overflow with Hive SerDe") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      Seq(
-        """
-          |SELECT TRANSFORM(array('abcdef'))
-          |USING 'cat'
-          |AS (value ARRAY<CHAR(4)>)
-          |FROM VALUES (1) input(dummy)
-          |""".stripMargin,
-        """
-          |SELECT TRANSFORM(named_struct('value', 'abcdef'))
-          |USING 'cat'
-          |AS (value STRUCT<value: VARCHAR(4)>)
-          |FROM VALUES (1) input(dummy)
-          |""".stripMargin).foreach { query =>
-        val exception = intercept[Exception] {
-          sql(query).collect()
-        }
-        val runtimeException = exception match {
-          case s: org.apache.spark.SparkRuntimeException => s
-          case other =>
-            other.getCause.asInstanceOf[org.apache.spark.SparkRuntimeException]
-        }
-        checkError(
-          exception = runtimeException,
-          condition = "EXCEED_LIMIT_LENGTH",
-          parameters = Map("limit" -> "4"))
-      }
-    }
-  }
-
-  test("SPARK-59277: output SerDe CHAR/VARCHAR rewrite is LazySimpleSerDe-only") {
-    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
-      val output = Seq(
-        AttributeReference("c", CharType(4))(),
-        AttributeReference("v", VarcharType(5))(),
-        AttributeReference("nested", ArrayType(CharType(4)))())
-
-      val (_, lazySoi) = HiveScriptIOSchema.initOutputSerDe(hiveIOSchema, output).get
-      assert(lazySoi.getAllStructFieldRefs.get(0).getFieldObjectInspector.getTypeName ===
-        "string")
-      assert(lazySoi.getAllStructFieldRefs.get(1).getFieldObjectInspector.getTypeName ===
-        "string")
-      assert(lazySoi.getAllStructFieldRefs.get(2).getFieldObjectInspector.getTypeName ===
-        "array<string>")
-
-      val subclassSchema = hiveIOSchema.copy(
-        outputSerdeClass = Some(classOf[TestLazySimpleSerDe].getCanonicalName))
-      val (_, subclassSoi) = HiveScriptIOSchema.initOutputSerDe(subclassSchema, output).get
-      assert(subclassSoi.getAllStructFieldRefs.get(0).getFieldObjectInspector.getTypeName ===
-        "string")
-
-      SchemaCapturingSerDe.lastColumnTypes = null
-      val customSchema = defaultIOSchema.copy(
-        outputSerdeClass = Some(classOf[SchemaCapturingSerDe].getCanonicalName))
-      HiveScriptIOSchema.initOutputSerDe(customSchema, output)
-      val captured = SchemaCapturingSerDe.lastColumnTypes
-      assert(captured.contains("char(4)"))
-      assert(captured.contains("varchar(5)"))
-      assert(captured.contains("array<char(4)>"))
-    }
-  }
-
   test("SPARK-32400: TRANSFORM doesn't support CalendarIntervalType/UserDefinedType (hive serde)") {
     assume(TestUtils.testCommandAvailable("/bin/bash"))
     withTempView("v") {
@@ -809,29 +668,3 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
     }
   }
 }
-
-class TestLazySimpleSerDe extends LazySimpleSerDe
-
-class SchemaCapturingSerDe extends AbstractSerDe {
-  override def initialize(conf: Configuration, tbl: Properties): Unit = {
-    SchemaCapturingSerDe.lastColumnTypes =
-      tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES)
-  }
-
-  override def getObjectInspector: ObjectInspector =
-    ObjectInspectorFactory.getStandardStructObjectInspector(
-      Arrays.asList("col"),
-      Arrays.asList(PrimitiveObjectInspectorFactory.javaStringObjectInspector))
-
-  override def getSerializedClass: Class[_ <: Writable] = classOf[Text]
-
-  override def getSerDeStats: SerDeStats = null
-
-  override def serialize(obj: Any, inspector: ObjectInspector): Writable = null
-
-  override def deserialize(blob: Writable): AnyRef = null
-}
-
-object SchemaCapturingSerDe {
-  @volatile var lastColumnTypes: String = _
-}

From a0fb4390ae35a668e64ebb4d14ed03326ef12e76 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Mon, 21 Sep 2026 15:34:49 +0000
Subject: [PATCH 19/22] fix: [SPARK-59277] validate Hive map keys after CHAR
 conversion and test runtime-inspector guards

---
 .../spark/sql/hive/HiveInspectors.scala       | 16 +++++++-
 .../sql/hive/execution/HiveUDAFSuite.scala    | 39 ++++++++++++++++++-
 .../sql/hive/execution/HiveUDFSuite.scala     | 31 +++++++++++++++
 3 files changed, 83 insertions(+), 3 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 372ac96fa45d1..4d05a429a89fe 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -903,10 +903,24 @@ private[hive] trait HiveInspectors {
       case (mi: MapObjectInspector, MapType(keyType, valueType, _)) =>
         val keyUnwrapper = unwrapperFor(mi.getMapKeyObjectInspector, keyType)
         val valueUnwrapper = unwrapperFor(mi.getMapValueObjectInspector, valueType)
+        val needsKeyValidation = CharVarcharUtils.hasCharVarchar(keyType)
         data: Any => {
           if (data != null) {
             val map = mi.getMap(data)
-            if (map == null) null else ArrayBasedMapData(map, keyUnwrapper, valueUnwrapper)
+            if (map == null) {
+              null
+            } else if (needsKeyValidation) {
+              val builder = new ArrayBasedMapBuilder(keyType, valueType)
+              val it = map.entrySet().iterator().asInstanceOf[
+                java.util.Iterator[java.util.Map.Entry[Any, Any]]]
+              while (it.hasNext) {
+                val e = it.next()
+                builder.put(keyUnwrapper(e.getKey), valueUnwrapper(e.getValue))
+              }
+              builder.build()
+            } else {
+              ArrayBasedMapData(map, keyUnwrapper, valueUnwrapper)
+            }
           } else {
             null
           }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
index 509cb6ae687ab..b266a5feba384 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
@@ -29,7 +29,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.{PrimitiveObjectI
 import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, TypeInfo}
 import test.org.apache.spark.sql.MyDoubleAvg
 
-import org.apache.spark.SPARK_DOC_ROOT
+import org.apache.spark.{SPARK_DOC_ROOT, SparkException}
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Literal}
 import org.apache.spark.sql.catalyst.expressions.Cast._
@@ -40,7 +40,7 @@ import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
 import org.apache.spark.sql.hive.HiveUDAFFunction
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{CharType, VarcharType}
+import org.apache.spark.sql.types.{CharType, LongType, StringType, VarcharType}
 import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -255,6 +255,41 @@ class HiveUDAFSuite extends QueryTest
     }
   }
 
+  test("SPARK-59277: incompatible UDAF partial inspector triggers mismatch error") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      // MockPartialStringFinalCharUDAF exposes STRING partial / CHAR(5) final.
+      // Feed LongType as the expected partial type to trigger the mismatch.
+      val udaf = HiveUDAFFunction(
+        "char_max",
+        HiveFunctionWrapper(classOf[MockPartialStringFinalCharUDAF].getName),
+        Seq(Literal("x")),
+        isUDAFBridgeRequired = false,
+        mutableAggBufferOffset = 0,
+        inputAggBufferOffset = 0,
+        partialResultDataType = LongType,
+        dataType = CharType(5))
+      intercept[SparkException] { udaf.serialize(null) }
+    }
+  }
+
+  test("SPARK-59277: incompatible UDAF final inspector triggers mismatch error") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      // MockPartialStringFinalCharUDAF exposes CHAR(5) final.
+      // Feed VarcharType(5) as the expected final type: CHAR(5) vs VARCHAR(5)
+      // is incompatible (different bounded-string kind).
+      val udaf = HiveUDAFFunction(
+        "char_max",
+        HiveFunctionWrapper(classOf[MockPartialStringFinalCharUDAF].getName),
+        Seq(Literal("x")),
+        isUDAFBridgeRequired = false,
+        mutableAggBufferOffset = 0,
+        inputAggBufferOffset = 0,
+        partialResultDataType = StringType,
+        dataType = VarcharType(5))
+      intercept[SparkException] { udaf.serialize(null) }
+    }
+  }
+
   test("non-deterministic children expressions of UDAF") {
     withTempView("view1") {
       spark.range(1).selectExpr("id as x", "id as y").createTempView("view1")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 719ea2b891c49..efba2248819a2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -1126,6 +1126,37 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton {
     }
   }
 
+  test("SPARK-59277: incompatible UDF runtime inspector triggers mismatch error") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      // ReturnCharGenericUDF always returns CHAR(5), but we force VarcharType(5) as
+      // the captured analysis type. The runtime check must detect the mismatch.
+      val udf = HiveGenericUDF(
+        "return_char",
+        HiveFunctionWrapper(classOf[ReturnCharGenericUDF].getName),
+        Seq(Literal("x")),
+        VarcharType(5))
+      intercept[SparkException] { udf.eval(InternalRow.empty) }
+    }
+  }
+
+  test("SPARK-59277: incompatible UDTF runtime inspector triggers mismatch error") {
+    withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+      // GenericUDTFExplode returns the element type of its input. Feeding it
+      // ARRAY<CHAR(5)> makes its runtime inspector return CHAR(5). We supply a
+      // captured schema claiming VARCHAR(5), so the check must fire.
+      val arrayType = ArrayType(CharType(5))
+      val values = new GenericArrayData(
+        Array[Any](UTF8String.fromString("abc  ")))
+      val schema = StructType(Seq(StructField("col", VarcharType(5))))
+      val udtf = HiveGenericUDTF(
+        "hive_explode",
+        HiveFunctionWrapper(classOf[GenericUDTFExplode].getName),
+        Seq(Literal(values, arrayType)),
+        schema)
+      intercept[SparkException] { udtf.eval(InternalRow.empty) }
+    }
+  }
+
   test("SPARK-58792: copied HiveGenericUDF nodes must not share a mutable GenericUDF") {
     val tsAttr = AttributeReference("ts", TimestampType, nullable = false)()
     val constTs = Literal(

From cec2eff61be2eda7a8ad948bc694cb813a1c1d8a Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Mon, 21 Sep 2026 17:30:01 +0000
Subject: [PATCH 20/22] fix: [SPARK-59277] check struct field names, remove
 orphaned in-place unwrapper, add map-key collision test

---
 .../spark/sql/hive/HiveInspectors.scala       | 21 +------
 .../spark/sql/hive/HiveInspectorSuite.scala   | 59 ++++++++-----------
 2 files changed, 27 insertions(+), 53 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 4d05a429a89fe..b8758407877f0 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -971,25 +971,6 @@ private[hive] trait HiveInspectors {
         (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
     }
 
-  /**
-   * Builds an in-place unwrapper using the target Catalyst `dataType` only when target-aware
-   * conversion is required. Other types retain the primitive setter fast paths.
-   */
-  def unwrapperFor(
-      field: HiveStructField,
-      dataType: DataType): (Any, InternalRow, Int) => Unit = {
-    val requiresTypedConversion = dataType.existsRecursively {
-      case _: CharType | _: VarcharType | _: AnyTimestampNanoType => true
-      case _ => false
-    }
-    if (requiresTypedConversion) {
-      val unwrapper = unwrapperFor(field.getFieldObjectInspector, dataType)
-      (value: Any, row: InternalRow, ordinal: Int) => row(ordinal) = unwrapper(value)
-    } else {
-      unwrapperFor(field)
-    }
-  }
-
   def wrap(a: Any, oi: ObjectInspector, dataType: DataType): AnyRef = {
     wrapperFor(oi, dataType)(a).asInstanceOf[AnyRef]
   }
@@ -1297,7 +1278,7 @@ private[hive] trait HiveInspectors {
         compatibleHiveReturnType(rk, ek) && compatibleHiveReturnType(rv, ev)
       case (rt: StructType, et: StructType) if rt.length == et.length =>
         rt.fields.zip(et.fields).forall { case (rf, ef) =>
-          compatibleHiveReturnType(rf.dataType, ef.dataType)
+          rf.name == ef.name && compatibleHiveReturnType(rf.dataType, ef.dataType)
         }
       case (rt, et) => rt.sameType(et)
     }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 8b49a289d1e03..114d2379585b0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -17,12 +17,11 @@
 
 package org.apache.spark.sql.hive
 
-import java.sql.Timestamp
 import java.util
 
 import org.apache.hadoop.hive.ql.udf.UDAFPercentile
 import org.apache.hadoop.hive.serde2.io.{DoubleWritable, HiveCharWritable, HiveVarcharWritable}
-import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, ObjectInspector, ObjectInspectorFactory, PrimitiveObjectInspector, StructObjectInspector}
+import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, MapObjectInspector, ObjectInspector, ObjectInspectorFactory, PrimitiveObjectInspector, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
 import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, DecimalTypeInfo, VarcharTypeInfo}
@@ -31,8 +30,8 @@ import org.apache.hadoop.io.LongWritable
 import org.apache.spark.{SparkException, SparkFunSuite, SparkRuntimeException}
 import org.apache.spark.sql.{AnalysisException, Row, TestUserClassUDT}
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Literal, SpecificInternalRow}
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeUtils, GenericArrayData, MapData}
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -396,34 +395,6 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
         wrap(input, inspector, dataType)).asInstanceOf[InternalRow]
       assert(result.getArray(0).getUTF8String(0) === UTF8String.fromString("a   "))
       assert(result.getMap(1).valueArray().getUTF8String(0) === UTF8String.fromString("value"))
-
-      val outerType = StructType(Seq(StructField("nested", dataType)))
-      val outerInspector = toInspector(outerType).asInstanceOf[StructObjectInspector]
-      val field = outerInspector.getAllStructFieldRefs.get(0)
-      val targetRow = new SpecificInternalRow(Seq(dataType))
-      unwrapperFor(field, dataType)(wrap(input, inspector, dataType), targetRow, 0)
-      val nestedResult = targetRow.getStruct(0, dataType.length)
-      assert(nestedResult.getArray(0).getUTF8String(0) === UTF8String.fromString("a   "))
-      assert(
-        nestedResult.getMap(1).valueArray().getUTF8String(0) === UTF8String.fromString("value"))
-    }
-  }
-
-  test("SPARK-59277: typed field unwrappers preserve nanosecond timestamps") {
-    val value = Timestamp.valueOf("2026-09-16 12:34:56.123456789")
-    Seq(
-      TimestampNTZNanosType(9) ->
-        DateTimeUtils.localDateTimeToTimestampNanos(value.toLocalDateTime, 9),
-      TimestampLTZNanosType(9) ->
-        DateTimeUtils.instantToTimestampNanos(value.toInstant, 9)).foreach {
-      case (dataType, expected) =>
-        val inspector = ObjectInspectorFactory.getStandardStructObjectInspector(
-          util.Arrays.asList("value"),
-          util.Arrays.asList(PrimitiveObjectInspectorFactory.javaTimestampObjectInspector))
-        val field = inspector.getAllStructFieldRefs.get(0)
-        val targetRow = new SpecificInternalRow(Seq(dataType))
-        unwrapperFor(field, dataType)(value, targetRow, 0)
-        assert(targetRow.get(0, dataType) === expected)
     }
   }
 
@@ -480,7 +451,8 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
       IntegerType -> CharType(5),
       MapType(CharType(4), StringType) -> MapType(CharType(5), StringType),
       StructType.fromDDL("c STRING") -> StructType.fromDDL("c STRING, v STRING"),
-      StructType.fromDDL("c CHAR(4)") -> StructType.fromDDL("c CHAR(5)")).foreach {
+      StructType.fromDDL("c CHAR(4)") -> StructType.fromDDL("c CHAR(5)"),
+      StructType.fromDDL("a INT, b INT") -> StructType.fromDDL("b INT, a INT")).foreach {
       case (runtimeType, expectedType) =>
         intercept[SparkException] {
           checkCompatibleHiveReturnType(runtimeType, expectedType)
@@ -529,6 +501,27 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
     }
   }
 
+  test("SPARK-59277: Hive map unwrapper validates CHAR-padded key collisions") {
+    withFirstClassCharVarchar(enabled = true) {
+      // Two distinct STRING keys that both pad to the same CHAR(2) value.
+      val mapType = MapType(CharType(2), StringType)
+      val inspector = toInspector(mapType).asInstanceOf[MapObjectInspector]
+      val javaMap = new java.util.HashMap[Any, Any]()
+      javaMap.put("a", "v1")
+      javaMap.put("a ", "v2")
+      val hiveMap = wrap(
+        ArrayBasedMapData(
+          Array[Any](UTF8String.fromString("a"), UTF8String.fromString("a ")),
+          Array[Any](UTF8String.fromString("v1"), UTF8String.fromString("v2"))),
+        inspector, mapType)
+      val unwrapper = unwrapperFor(inspector, mapType)
+      val result = unwrapper(hiveMap).asInstanceOf[MapData]
+      // Both keys pad to "a ", so the builder deduplicates to one entry.
+      assert(result.numElements() === 1)
+      assert(result.keyArray().getUTF8String(0) === UTF8String.fromString("a "))
+    }
+  }
+
   test("SPARK-59277: Hive object inspectors reject unsupported CHAR/VARCHAR lengths") {
     withFirstClassCharVarchar(enabled = true) {
       Seq[DataType](

From ae383a15232bfa7f94070e1763f4d8efabd6e4de Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Mon, 21 Sep 2026 18:08:25 +0000
Subject: [PATCH 21/22] fix: [SPARK-59277] test CHAR map-key collision with raw
 Java inspectors and both dedup policies

---
 .../spark/sql/hive/HiveInspectorSuite.scala   | 46 +++++++++++++------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 114d2379585b0..d20e7e5cd8d6c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -21,7 +21,7 @@ import java.util
 
 import org.apache.hadoop.hive.ql.udf.UDAFPercentile
 import org.apache.hadoop.hive.serde2.io.{DoubleWritable, HiveCharWritable, HiveVarcharWritable}
-import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, MapObjectInspector, ObjectInspector, ObjectInspectorFactory, PrimitiveObjectInspector, StructObjectInspector}
+import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, ObjectInspector, ObjectInspectorFactory, PrimitiveObjectInspector, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
 import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, DecimalTypeInfo, VarcharTypeInfo}
@@ -503,22 +503,38 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
 
   test("SPARK-59277: Hive map unwrapper validates CHAR-padded key collisions") {
     withFirstClassCharVarchar(enabled = true) {
-      // Two distinct STRING keys that both pad to the same CHAR(2) value.
-      val mapType = MapType(CharType(2), StringType)
-      val inspector = toInspector(mapType).asInstanceOf[MapObjectInspector]
-      val javaMap = new java.util.HashMap[Any, Any]()
+      // Two distinct Java String keys that both pad to the same CHAR(2).
+      // Use a plain-string key inspector so the Java HashMap preserves both
+      // entries (HiveChar equality would deduplicate them before our code runs).
+      val keyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector
+      val valueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector
+      val mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI)
+      val javaMap = new java.util.LinkedHashMap[Any, Any]()
       javaMap.put("a", "v1")
       javaMap.put("a ", "v2")
-      val hiveMap = wrap(
-        ArrayBasedMapData(
-          Array[Any](UTF8String.fromString("a"), UTF8String.fromString("a ")),
-          Array[Any](UTF8String.fromString("v1"), UTF8String.fromString("v2"))),
-        inspector, mapType)
-      val unwrapper = unwrapperFor(inspector, mapType)
-      val result = unwrapper(hiveMap).asInstanceOf[MapData]
-      // Both keys pad to "a ", so the builder deduplicates to one entry.
-      assert(result.numElements() === 1)
-      assert(result.keyArray().getUTF8String(0) === UTF8String.fromString("a "))
+
+      val targetType = MapType(CharType(2), StringType)
+
+      // Default EXCEPTION policy: duplicate padded keys must throw.
+      val exceptionConf = new SQLConf
+      exceptionConf.setConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS, true)
+      exceptionConf.setConfString(SQLConf.MAP_KEY_DEDUP_POLICY.key, "EXCEPTION")
+      SQLConf.withExistingConf(exceptionConf) {
+        val unwrapper = unwrapperFor(mapOI, targetType)
+        intercept[SparkRuntimeException] { unwrapper(javaMap) }
+      }
+
+      // LAST_WIN policy: duplicate padded keys deduplicate to one entry.
+      val lastWinConf = new SQLConf
+      lastWinConf.setConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS, true)
+      lastWinConf.setConfString(SQLConf.MAP_KEY_DEDUP_POLICY.key, "LAST_WIN")
+      SQLConf.withExistingConf(lastWinConf) {
+        val unwrapper = unwrapperFor(mapOI, targetType)
+        val result = unwrapper(javaMap).asInstanceOf[MapData]
+        assert(result.numElements() === 1)
+        assert(result.keyArray().getUTF8String(0) ===
+          UTF8String.fromString("a "))
+      }
     }
   }
 

From bf11663218a5be5ec40658bdd743f1f068e49d83 Mon Sep 17 00:00:00 2001
From: srielau <serge@rielau.com>
Date: Mon, 21 Sep 2026 18:22:33 +0000
Subject: [PATCH 22/22] fix: [SPARK-59277] cover writable inspector and assert
 LAST_WIN value in map-key collision test

---
 .../spark/sql/hive/HiveInspectorSuite.scala   | 76 ++++++++++++-------
 1 file changed, 49 insertions(+), 27 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index d20e7e5cd8d6c..9f135bf6bc731 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.{ConstantObjectInspector, O
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory
 import org.apache.hadoop.hive.serde2.typeinfo.{CharTypeInfo, DecimalTypeInfo, VarcharTypeInfo}
-import org.apache.hadoop.io.LongWritable
+import org.apache.hadoop.io.{LongWritable, Text}
 
 import org.apache.spark.{SparkException, SparkFunSuite, SparkRuntimeException}
 import org.apache.spark.sql.{AnalysisException, Row, TestUserClassUDT}
@@ -503,37 +503,59 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
 
   test("SPARK-59277: Hive map unwrapper validates CHAR-padded key collisions") {
     withFirstClassCharVarchar(enabled = true) {
-      // Two distinct Java String keys that both pad to the same CHAR(2).
-      // Use a plain-string key inspector so the Java HashMap preserves both
-      // entries (HiveChar equality would deduplicate them before our code runs).
-      val keyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector
-      val valueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector
-      val mapOI = ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI)
+      val targetType = MapType(CharType(2), StringType)
+
+      // Two distinct keys that both pad to the same CHAR(2). Use plain-string
+      // inspectors so the Java HashMap preserves both entries (HiveChar
+      // equality would deduplicate them before our code runs).
+      val javaKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector
+      val javaValueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector
+      val javaMapOI = ObjectInspectorFactory
+        .getStandardMapObjectInspector(javaKeyOI, javaValueOI)
       val javaMap = new java.util.LinkedHashMap[Any, Any]()
       javaMap.put("a", "v1")
       javaMap.put("a ", "v2")
 
-      val targetType = MapType(CharType(2), StringType)
-
-      // Default EXCEPTION policy: duplicate padded keys must throw.
-      val exceptionConf = new SQLConf
-      exceptionConf.setConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS, true)
-      exceptionConf.setConfString(SQLConf.MAP_KEY_DEDUP_POLICY.key, "EXCEPTION")
-      SQLConf.withExistingConf(exceptionConf) {
-        val unwrapper = unwrapperFor(mapOI, targetType)
-        intercept[SparkRuntimeException] { unwrapper(javaMap) }
-      }
+      val writableKeyOI =
+        PrimitiveObjectInspectorFactory.writableStringObjectInspector
+      val writableValueOI =
+        PrimitiveObjectInspectorFactory.writableStringObjectInspector
+      val writableMapOI = ObjectInspectorFactory
+        .getStandardMapObjectInspector(writableKeyOI, writableValueOI)
+      val writableMap = new java.util.LinkedHashMap[Any, Any]()
+      writableMap.put(new Text("a"), new Text("w1"))
+      writableMap.put(new Text("a "), new Text("w2"))
+
+      Seq(
+        ("java", javaMapOI, javaMap, "v2"),
+        ("writable", writableMapOI, writableMap, "w2")
+      ).foreach { case (label, mapOI, map, lastValue) =>
+        // EXCEPTION policy: duplicate padded keys must throw.
+        val exConf = new SQLConf
+        exConf.setConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS, true)
+        exConf.setConfString(
+          SQLConf.MAP_KEY_DEDUP_POLICY.key, "EXCEPTION")
+        SQLConf.withExistingConf(exConf) {
+          val unwrapper = unwrapperFor(mapOI, targetType)
+          intercept[SparkRuntimeException] { unwrapper(map) }
+        }
 
-      // LAST_WIN policy: duplicate padded keys deduplicate to one entry.
-      val lastWinConf = new SQLConf
-      lastWinConf.setConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS, true)
-      lastWinConf.setConfString(SQLConf.MAP_KEY_DEDUP_POLICY.key, "LAST_WIN")
-      SQLConf.withExistingConf(lastWinConf) {
-        val unwrapper = unwrapperFor(mapOI, targetType)
-        val result = unwrapper(javaMap).asInstanceOf[MapData]
-        assert(result.numElements() === 1)
-        assert(result.keyArray().getUTF8String(0) ===
-          UTF8String.fromString("a "))
+        // LAST_WIN policy: last inserted value wins.
+        val lwConf = new SQLConf
+        lwConf.setConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS, true)
+        lwConf.setConfString(
+          SQLConf.MAP_KEY_DEDUP_POLICY.key, "LAST_WIN")
+        SQLConf.withExistingConf(lwConf) {
+          val unwrapper = unwrapperFor(mapOI, targetType)
+          val result = unwrapper(map).asInstanceOf[MapData]
+          assert(result.numElements() === 1,
+            s"$label: expected 1 entry after dedup")
+          assert(result.keyArray().getUTF8String(0) ===
+            UTF8String.fromString("a "))
+          assert(result.valueArray().getUTF8String(0) ===
+            UTF8String.fromString(lastValue),
+            s"$label: last-win value")
+        }
       }
     }
   }