[test] add random test case for manifest file full compaction

apache · Oct 15, 2024 · e9c6ff8 · e9c6ff8
1 parent 4045158
commit e9c6ff8
Showing 1 changed file with 105 additions and 0 deletions.
diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java
@@ -39,8 +39,10 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
@@ -446,6 +448,109 @@ public void testIdentifierAfterFullCompaction() throws Exception {
  containSameIdentifyEntryFile(fullCompacted, entryIdentifierExpected);
  }
 
+ @RepeatedTest(100)
+ public void testRandomFullCompaction() throws Exception {
+ List<ManifestFileMeta> input = new ArrayList<>();
+ Set<FileEntry.Identifier> manifestEntrySet = new HashSet<>();
+ Set<FileEntry.Identifier> deleteManifestEntrySet = new HashSet<>();
+ int inputSize = ThreadLocalRandom.current().nextInt(100) + 1;
+ int totalEntryNums = 0;
+ for (int i = 0; i < inputSize; i++) {
+ int entryNums = ThreadLocalRandom.current().nextInt(100) + 1;
+ input.add(
+ generateRandomData(
+ entryNums, totalEntryNums, manifestEntrySet, deleteManifestEntrySet));
+ totalEntryNums += entryNums;
+ }
+ int suggerstSize = ThreadLocalRandom.current().nextInt(3000) + 1;
+ int sizeTrigger = ThreadLocalRandom.current().nextInt(40000) + 1;
+ List<ManifestFileMeta> newMetas = new ArrayList<>();
+ Optional<List<ManifestFileMeta>> fullCompacted =
+ ManifestFileMerger.tryFullCompaction(
+ input,
+ newMetas,
+ manifestFile,
+ suggerstSize,
+ sizeTrigger,
+ getPartitionType(),
+ null);
+
+ // *****verify result*****
+ List<ManifestFileMeta> mustMergedFiles =
+ input.stream()
+ .filter(
+ manifest ->
+ manifest.fileSize() < suggerstSize
+ || manifest.numDeletedFiles() > 0)
+ .collect(Collectors.toList());
+ long mustMergeSize =
+ mustMergedFiles.stream().map(ManifestFileMeta::fileSize).reduce(0L, Long::sum);
+ // manifest files which were not written after full compaction
+ List<ManifestFileMeta> notMergedFiles =
+ input.stream()
+ .filter(
+ manifest ->
+ manifest.fileSize() >= suggerstSize
+ && manifest.numDeletedFiles() == 0)
+ .filter(
+ manifest ->
+ manifestFile.read(manifest.fileName(), manifest.fileSize())
+ .stream()
+ .map(ManifestEntry::identifier)
+ .noneMatch(deleteManifestEntrySet::contains))
+ .collect(Collectors.toList());
+
+ if (mustMergeSize < sizeTrigger) {
+ assertThat(fullCompacted).isEmpty();
+ assertThat(newMetas).isEmpty();
+ } else if (mustMergedFiles.size() <= 1) {
+ assertThat(fullCompacted).isEmpty();
+ assertThat(newMetas).isEmpty();
+ } else {
+ assertThat(fullCompacted.get().size()).isEqualTo(notMergedFiles.size() + 1);
+ assertThat(newMetas).size().isEqualTo(1);
+ }
+ }
+
+ private ManifestFileMeta generateRandomData(
+ int entryNums,
+ int totalEntryNums,
+ Set<FileEntry.Identifier> manifestEntrySet,
+ Set<FileEntry.Identifier> deleteManifestEntrySet) {
+ List<ManifestEntry> entries = new ArrayList<>();
+ for (int i = 0; i < entryNums; i++) {
+ // 70% add, 30% delete
+ boolean isAdd = ThreadLocalRandom.current().nextInt(10) < 7;
+ if (manifestEntrySet.isEmpty() || isAdd) {
+ String fileName = String.format("file-%d", totalEntryNums + i);
+ Integer partition = ThreadLocalRandom.current().nextInt(10);
+ int level = ThreadLocalRandom.current().nextInt(6);
+ List<String> extraFiles = Lists.newArrayList(String.format("index-%s", fileName));
+ byte[] embeddedIndex = new byte[] {1, 2, 3};
+ entries.add(makeEntry(true, fileName, partition, level, extraFiles, embeddedIndex));
+ } else {
+ FileEntry.Identifier identifier =
+ manifestEntrySet.stream()
+ .skip(ThreadLocalRandom.current().nextInt(manifestEntrySet.size()))
+ .findFirst()
+ .get();
+ entries.add(
+ makeEntry(
+ false,
+ identifier.fileName,
+ identifier.partition.getInt(0),
+ identifier.level,
+ identifier.extraFiles,
+ new byte[] {1, 2, 3}));
+ manifestEntrySet.remove(identifier);
+ deleteManifestEntrySet.add(identifier);
+ }
+ }
+ manifestEntrySet.addAll(
+ entries.stream().map(ManifestEntry::identifier).collect(Collectors.toSet()));
+ return makeManifest(entries.toArray(new ManifestEntry[0]));
+ }
+
  private void createData(
  int numLastBits, List<ManifestFileMeta> input, List<ManifestFileMeta> expected) {
  // suggested size 500 and suggested count 3