Fix nested type implicit schema evolution

apache · Dec 14, 2023 · e1423a8 · e1423a8
1 parent 74ab128
commit e1423a8
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/...atasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala b/...atasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
@@ -491,7 +491,6 @@ class TestAvroSchemaResolutionSupport extends HoodieClientTestBase with ScalaAss
  // NOTE: type promotion is not supported for the custom file format and the filegroup reader
  // HUDI-7045 and PR#10007 in progress to fix the issue
  .option(HoodieReaderConfig.FILE_GROUP_READER_ENABLED.key(), "false")
- .option("spark.sql.parquet.enableNestedColumnVectorizedReader", "false")
  .load(tempRecordPath)
  readDf.printSchema()
  readDf.show(false)

diff --git a/...apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala b/...apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala
@@ -120,7 +120,8 @@ class Spark33LegacyHoodieParquetFileFormat(private val shouldAppendPartitionValu
  val resultSchema = StructType(partitionSchema.fields ++ requiredSchema.fields)
  val sqlConf = sparkSession.sessionState.conf
  val enableOffHeapColumnVector = sqlConf.offHeapColumnVectorEnabled
- val enableVectorizedReader: Boolean = supportBatch(sparkSession, resultSchema)
+ val enableVectorizedReader: Boolean =
+ ParquetUtils.isBatchReadSupportedForSchema(sqlConf, resultSchema)
  val enableRecordFilter: Boolean = sqlConf.parquetRecordFilterEnabled
  val timestampConversion: Boolean = sqlConf.isParquetINT96TimestampConversion
  val capacity = sqlConf.parquetVectorizedReaderBatchSize