jsc.parallelize(partitionPaths, partitionPaths.size()) .flatMap((FlatMapFunction<String, CompactionOperation>) partitionPath -> fileSystemView .getLatestFileSlices(partitionPath) .filter(slice -> !fileIdsWithPendingCompactions.contains(slice.getFileId())) .map(
jsc.parallelize(partitionPaths, partitionPaths.size()) .flatMap((FlatMapFunction<String, CompactionOperation>) partitionPath -> fileSystemView .getLatestFileSlices(partitionPath) .filter(slice -> !fileIdsWithPendingCompactions.contains(slice.getFileId())) .map(
this.partitionPath = record.getPartitionPath(); Optional<FileSlice> fileSlice = fileSystemView.getLatestFileSlices(partitionPath) .filter(fileSlice1 -> fileSlice1.getFileId().equals(fileId)).findFirst();
List<FileSlice> fileSlices = rtView.getLatestFileSlices(partitionPath).collect(Collectors.toList()); String compactionRequestedTime = "4"; String compactDataFileName = FSUtils.makeDataFileName(compactionRequestedTime, 1, fileId); fileSliceList = rtView.getLatestFileSlices(partitionPath).collect(Collectors.toList()); dataFiles = fileSliceList.stream().map(FileSlice::getDataFile) .filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); FileSlice slice = table.getRTFileSystemView().getLatestFileSlices(DEFAULT_FIRST_PARTITION_PATH) .filter(fs -> fs.getFileId().equals(fileId)).findFirst().get(); List<FileSlice> slices = new ArrayList<>();
List<FileSlice> fileSliceList = rtView.getLatestFileSlices(partitionPath).collect(Collectors.toList()); assertEquals(1, fileSliceList.size()); FileSlice fileSlice = fileSliceList.get(0);
hoodieIndex.isGlobal() ? this.getRTFileSystemView().getLatestFileSlices(partitionPath) .collect(Collectors.toMap(FileSlice::getFileId, FileSlice::getBaseInstantTime)) : null; commitMetadata.getPartitionToWriteStats().get(partitionPath).stream()
hoodieIndex.isGlobal() ? this.getRTFileSystemView().getLatestFileSlices(partitionPath) .collect(Collectors.toMap(FileSlice::getFileId, FileSlice::getBaseInstantTime)) : null; commitMetadata.getPartitionToWriteStats().get(partitionPath).stream()
table = HoodieTable.getHoodieTable(metaClient, config, jsc); for (String partitionPath : dataGen.getPartitionPaths()) { List<FileSlice> groupedLogFiles = table.getRTFileSystemView().getLatestFileSlices(partitionPath) .collect(Collectors.toList()); for (FileSlice fileSlice : groupedLogFiles) { List<FileSlice> groupedLogFiles = table.getRTFileSystemView().getLatestFileSlices(partitionPath) .collect(Collectors.toList()); for (FileSlice slice : groupedLogFiles) {
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getDataFile().isPresent()).count() == 0); Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count() == 0); numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count();
table = HoodieTable.getHoodieTable(metaClient, config, jsc); for (String partitionPath : dataGen.getPartitionPaths()) { List<FileSlice> groupedLogFiles = table.getRTFileSystemView().getLatestFileSlices(partitionPath) .collect(Collectors.toList()); for (FileSlice fileSlice : groupedLogFiles) {
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getDataFile().isPresent()).count() == 0); Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count() > 0); numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count(); Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getDataFile().isPresent()).count() == 0); Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count() > 0);
this.partitionPath = record.getPartitionPath(); Optional<FileSlice> fileSlice = fileSystemView.getLatestFileSlices(partitionPath) .filter(fileSlice1 -> fileSlice1.getFileId().equals(fileId)).findFirst();
Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getDataFile().isPresent()).count() == 0); Assert.assertTrue(tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count() > 0); numLogFiles += tableRTFileSystemView.getLatestFileSlices(partitionPath).filter(fileSlice -> fileSlice.getLogFiles().count() > 0).count();