@Test public void testComputeWorkUnitsWithNoPrexistentCheckpointsMultiplePartitions() throws IOException { // No prexisting checkpoints for the workunit calculator this.fileSystem.mkdirs(new Path(this.dataPath, PARTITION_2)); this.fileSystem.mkdirs(new Path(this.dataPath, PARTITION_3)); this.partitionManager = new HDFSPartitionManager(JOB_NAME, this.metadataPath, this.dataPath, this.fileSystem); this.metadataManager = new HDFSMetadataManager(this.fileSystem, new Path(this.metadataPath, JOB_NAME).toString(), new AtomicBoolean(true)); Assert.assertFalse(this.partitionManager.isSinglePartition()); final Optional<StringValue> latestCheckpoint = getLatestCheckpoint(this.metadataManager); Assert.assertFalse(latestCheckpoint.isPresent()); virtuallyProcessPartition(this.partitionManager, this.metadataManager, Optional.absent(), PARTITION_2); final HDFSPartitionManager partitionManager2 = new HDFSPartitionManager(JOB_NAME, this.metadataPath, this.dataPath, this.fileSystem); final HDFSMetadataManager metadataManager2 = new HDFSMetadataManager(this.fileSystem, new Path(this.metadataPath, JOB_NAME).toString(), new AtomicBoolean(true)); final Optional<StringValue> latestCheckpoint2 = getLatestCheckpoint(metadataManager2); Assert.assertTrue(latestCheckpoint2.isPresent()); virtuallyProcessPartition(partitionManager2, metadataManager2, Optional.of(new StringValue(PARTITION_2)), PARTITION_3); }
private void virtuallyProcessPartition(@NonNull final HDFSPartitionManager partitionManager, @NonNull final HDFSMetadataManager metadataManager, @NotEmpty final Optional<StringValue> expectedLatestCheckpoint, @NotEmpty final String expectedNextPartition) throws IOException { Assert.assertEquals(expectedLatestCheckpoint, getLatestCheckpoint(metadataManager)); final ParquetWorkUnitCalculator calculator = new ParquetWorkUnitCalculator(this.hiveConfig, this.fileSystem); calculator.initPreviousRunState(metadataManager); final ParquetWorkUnitCalculatorResult result = calculator.computeWorkUnits(); final List<String> workUnits = result.getWorkUnits(); Assert.assertEquals(1, workUnits.size()); Assert.assertEquals(expectedNextPartition, workUnits.get(0)); Assert.assertTrue(result.getNextRunState().getPartition().isPresent()); Assert.assertEquals(expectedNextPartition, result.getNextRunState().getPartition().get()); calculator.saveNextRunState(metadataManager, result.getNextRunState()); Assert.assertEquals(expectedNextPartition, metadataManager.get(MetadataConstants.CHECKPOINT_KEY).get().getValue()); metadataManager.saveChanges(); }
Assert.assertFalse(getLatestCheckpoint(this.metadataManager).isPresent()); virtuallyProcessPartition(this.partitionManager, this.metadataManager, Optional.absent(), this.dataPath); new AtomicBoolean(true)); final Optional<StringValue> latestCheckpoint2 = getLatestCheckpoint(metadataManager2); Assert.assertTrue(latestCheckpoint2.isPresent()); Assert.assertEquals(this.dataPath, latestCheckpoint2.get().getValue());