Fix hudi partition base path scheme to s3

apache · Jan 31, 2024 · febb22c · febb22c
1 parent c5573ab
commit febb22c
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -160,7 +160,7 @@ public void addPartitionsToTable(String tableName, List<String> partitionsToAdd)
  Table table = getTable(awsGlue, databaseName, tableName);
  StorageDescriptor sd = table.storageDescriptor();
  List<PartitionInput> partitionInputs = partitionsToAdd.stream().map(partition -> {
- String fullPartitionPath = FSUtils.getPartitionPath(getBasePath(), partition).toString();
+ String fullPartitionPath = FSUtils.getPartitionPath(s3aToS3(getBasePath()), partition).toString();
  List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partition);
  StorageDescriptor partitionSD = sd.copy(copySd -> copySd.location(fullPartitionPath));
  return PartitionInput.builder().values(partitionValues).storageDescriptor(partitionSD).build();
@@ -203,7 +203,7 @@ public void updatePartitionsToTable(String tableName, List<String> changedPartit
  Table table = getTable(awsGlue, databaseName, tableName);
  StorageDescriptor sd = table.storageDescriptor();
  List<BatchUpdatePartitionRequestEntry> updatePartitionEntries = changedPartitions.stream().map(partition -> {
- String fullPartitionPath = FSUtils.getPartitionPath(getBasePath(), partition).toString();
+ String fullPartitionPath = FSUtils.getPartitionPath(s3aToS3(getBasePath()), partition).toString();
  List<String> partitionValues = partitionValueExtractor.extractPartitionValuesInPath(partition);
  StorageDescriptor partitionSD = sd.copy(copySd -> copySd.location(fullPartitionPath));
  PartitionInput partitionInput = PartitionInput.builder().values(partitionValues).storageDescriptor(partitionSD).build();