Fixing getLatestFileSlice calls to getLatestMergedFileSlice in MDT wr…

…iter
apache · Oct 18, 2024 · 4f07da9 · 4f07da9
1 parent c4680e4
commit 4f07da9
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/...client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/...client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -609,7 +609,9 @@ private List<Pair<String, FileSlice>> getPartitionFileSlicePairs() throws IOExce
  List<String> partitions = metadata.getAllPartitionPaths();
  fsView.loadAllPartitions();
  List<Pair<String, FileSlice>> partitionFileSlicePairs = new ArrayList<>();
- partitions.forEach(partition -> fsView.getLatestFileSlices(partition).forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition, fs))));
+ partitions.forEach(partition -> fsView.getLatestMergedFileSlicesBeforeOrOn(partition,
+ dataMetaClient.getActiveTimeline().filterCompletedAndCompactionInstants().lastInstant().get().getTimestamp())
+ .forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition, fs))));
  return partitionFileSlicePairs;
  }
  }
@@ -645,8 +647,9 @@ private Pair<Integer, HoodieData<HoodieRecord>> initializeRecordIndexPartition()
  this.getClass().getSimpleName());
  } else {
  final List<Pair<String, FileSlice>> partitionFileSlicePairs = new ArrayList<>();
+ String latestCommit = dataMetaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().lastInstant().get().getTimestamp();
  for (String partition : partitions) {
- fsView.getLatestFileSlices(partition).forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition, fs)));
+ fsView.getLatestMergedFileSlicesBeforeOrOn(partition, latestCommit).forEach(fs -> partitionFileSlicePairs.add(Pair.of(partition, fs)));
  }
 
  LOG.info("Initializing record index from " + partitionFileSlicePairs.size() + " file slices in "