Skip to content

Commit

Permalink
fix test compile issues
Browse files Browse the repository at this point in the history
  • Loading branch information
the-other-tim-brown committed Oct 16, 2024
1 parent 81e3535 commit e4e0e1d
Show file tree
Hide file tree
Showing 5 changed files with 10 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ private void createNonpartitionedTable() throws IOException {
// Reload meta client and create fsView
metaClient = HoodieTableMetaClient.reload(metaClient);

nonpartitionedFsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline(), true);
nonpartitionedFsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context(), metaClient, metaClient.getActiveTimeline(), true);
}

private void createPartitionedTable() throws IOException {
Expand Down Expand Up @@ -159,7 +159,7 @@ private void createPartitionedTable() throws IOException {
// Reload meta client and create fsView
metaClient = HoodieTableMetaClient.reload(metaClient);

partitionedFsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline(), true);
partitionedFsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context(), metaClient, metaClient.getActiveTimeline(), true);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -781,7 +781,7 @@ public void testMetadataTableDeletePartition(HoodieTableType tableType) throws E

final Map<String, MetadataPartitionType> metadataEnabledPartitionTypes = new HashMap<>();
metadataWriter.getEnabledPartitionTypes().forEach(e -> metadataEnabledPartitionTypes.put(e.getPartitionPath(), e));
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metadataMetaClient, metadataMetaClient.getActiveTimeline());
HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(engineContext, metadataMetaClient, metadataMetaClient.getActiveTimeline());
metadataTablePartitions.forEach(partition -> {
List<FileSlice> latestSlices = fsView.getLatestFileSlices(partition).collect(Collectors.toList());
if (COLUMN_STATS.getPartitionPath().equals(partition)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ public CustomWriteConfig buildCustomWriteConfig(Schema avroSchema) {
}

public HoodieTableFileSystemView getFileSystemView() {
return new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline());
return HoodieTableFileSystemView.fileListingBasedFileSystemView(context(), metaClient, metaClient.getActiveTimeline());
}

public List<FileSlice> getLatestFileSlices(String partitionPath) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.FileSystemViewStorageType;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.table.view.SpillableMapBasedFileSystemView;
import org.apache.hudi.common.util.CleanerUtils;
import org.apache.hudi.common.util.ConfigUtils;
import org.apache.hudi.common.util.FileFormatUtils;
Expand Down Expand Up @@ -1650,10 +1649,10 @@ public HoodieMetadataValidationContext(
FileSystemViewStorageConfig viewConf = FileSystemViewStorageConfig.newBuilder().fromProperties(props).build();
ValidationUtils.checkArgument(viewConf.getStorageType().name().equals(viewStorageType), "View storage type not reflected");
HoodieCommonConfig commonConfig = HoodieCommonConfig.newBuilder().fromProperties(props).build();
this.fileSystemView = getFileSystemView(engineContext,
metaClient, metadataConfig, viewConf, commonConfig);
this.tableMetadata = HoodieTableMetadata.create(
engineContext, metaClient.getStorage(), metadataConfig, metaClient.getBasePath().toString());
this.fileSystemView = getFileSystemView(engineContext,
metaClient, metadataConfig, viewConf, commonConfig);
if (metaClient.getCommitsTimeline().filterCompletedInstants().countInstants() > 0) {
this.allColumnNameList = getAllColumnNames();
}
Expand All @@ -1665,18 +1664,17 @@ public HoodieMetadataValidationContext(
private HoodieTableFileSystemView getFileSystemView(HoodieEngineContext context,
HoodieTableMetaClient metaClient, HoodieMetadataConfig metadataConfig,
FileSystemViewStorageConfig viewConf, HoodieCommonConfig commonConfig) {
HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
switch (viewConf.getStorageType()) {
case SPILLABLE_DISK:
LOG.debug("Creating Spillable Disk based Table View");
return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf, commonConfig);
break;
case MEMORY:
LOG.debug("Creating in-memory based Table View");
return FileSystemViewManager.createInMemoryFileSystemView(context,
metaClient, metadataConfig);
break;
default:
throw new HoodieException("Unsupported storage type " + viewConf.getStorageType() + ", used with HoodieMetadataTableValidator");
}
return (HoodieTableFileSystemView) FileSystemViewManager.createViewManager(context, metadataConfig, viewConf, commonConfig, unused -> tableMetadata).getFileSystemView(metaClient);
}

public HoodieTableMetaClient getMetaClient() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ public void testAdditionalFilesinMetadata(Integer lastNFileSlices) throws IOExce
assertFalse(validator.hasValidationFailure());

// let's delete one of the log files from 1st commit and so FS based listing and MDT based listing diverges.
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(), false);
HoodieTableFileSystemView fsView = HoodieTableFileSystemView.fileListingBasedFileSystemView(context, metaClient, metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(), false);
HoodieFileGroup fileGroup = fsView.getAllFileGroups(StringUtils.EMPTY_STRING).collect(Collectors.toList()).get(0);
List<FileSlice> allFileSlices = fileGroup.getAllFileSlices().collect(Collectors.toList());
FileSlice earliestFileSlice = allFileSlices.get(allFileSlices.size() - 1);
Expand Down

0 comments on commit e4e0e1d

Please sign in to comment.