diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index c2a7f821dda0..b1575b155229 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -39,8 +39,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; @@ -446,6 +448,109 @@ public void testIdentifierAfterFullCompaction() throws Exception { containSameIdentifyEntryFile(fullCompacted, entryIdentifierExpected); } + @RepeatedTest(100) + public void testRandomFullCompaction() throws Exception { + List input = new ArrayList<>(); + Set manifestEntrySet = new HashSet<>(); + Set deleteManifestEntrySet = new HashSet<>(); + int inputSize = ThreadLocalRandom.current().nextInt(100) + 1; + int totalEntryNums = 0; + for (int i = 0; i < inputSize; i++) { + int entryNums = ThreadLocalRandom.current().nextInt(100) + 1; + input.add( + generateRandomData( + entryNums, totalEntryNums, manifestEntrySet, deleteManifestEntrySet)); + totalEntryNums += entryNums; + } + int suggerstSize = ThreadLocalRandom.current().nextInt(3000) + 1; + int sizeTrigger = ThreadLocalRandom.current().nextInt(40000) + 1; + List newMetas = new ArrayList<>(); + Optional> fullCompacted = + ManifestFileMerger.tryFullCompaction( + input, + newMetas, + manifestFile, + suggerstSize, + sizeTrigger, + getPartitionType(), + null); + + // *****verify result***** + List mustMergedFiles = + input.stream() + .filter( + manifest -> + manifest.fileSize() < suggerstSize + || manifest.numDeletedFiles() > 0) + .collect(Collectors.toList()); + long mustMergeSize = + mustMergedFiles.stream().map(ManifestFileMeta::fileSize).reduce(0L, Long::sum); + // manifest files which were not written after full compaction + List notMergedFiles = + input.stream() + .filter( + manifest -> + manifest.fileSize() >= suggerstSize + && manifest.numDeletedFiles() == 0) + .filter( + manifest -> + manifestFile.read(manifest.fileName(), manifest.fileSize()) + .stream() + .map(ManifestEntry::identifier) + .noneMatch(deleteManifestEntrySet::contains)) + .collect(Collectors.toList()); + + if (mustMergeSize < sizeTrigger) { + assertThat(fullCompacted).isEmpty(); + assertThat(newMetas).isEmpty(); + } else if (mustMergedFiles.size() <= 1) { + assertThat(fullCompacted).isEmpty(); + assertThat(newMetas).isEmpty(); + } else { + assertThat(fullCompacted.get().size()).isEqualTo(notMergedFiles.size() + 1); + assertThat(newMetas).size().isEqualTo(1); + } + } + + private ManifestFileMeta generateRandomData( + int entryNums, + int totalEntryNums, + Set manifestEntrySet, + Set deleteManifestEntrySet) { + List entries = new ArrayList<>(); + for (int i = 0; i < entryNums; i++) { + // 70% add, 30% delete + boolean isAdd = ThreadLocalRandom.current().nextInt(10) < 7; + if (manifestEntrySet.isEmpty() || isAdd) { + String fileName = String.format("file-%d", totalEntryNums + i); + Integer partition = ThreadLocalRandom.current().nextInt(10); + int level = ThreadLocalRandom.current().nextInt(6); + List extraFiles = Lists.newArrayList(String.format("index-%s", fileName)); + byte[] embeddedIndex = new byte[] {1, 2, 3}; + entries.add(makeEntry(true, fileName, partition, level, extraFiles, embeddedIndex)); + } else { + FileEntry.Identifier identifier = + manifestEntrySet.stream() + .skip(ThreadLocalRandom.current().nextInt(manifestEntrySet.size())) + .findFirst() + .get(); + entries.add( + makeEntry( + false, + identifier.fileName, + identifier.partition.getInt(0), + identifier.level, + identifier.extraFiles, + new byte[] {1, 2, 3})); + manifestEntrySet.remove(identifier); + deleteManifestEntrySet.add(identifier); + } + } + manifestEntrySet.addAll( + entries.stream().map(ManifestEntry::identifier).collect(Collectors.toSet())); + return makeManifest(entries.toArray(new ManifestEntry[0])); + } + private void createData( int numLastBits, List input, List expected) { // suggested size 500 and suggested count 3