Preconditions.checkArgument(workUnit.contains(ComplianceConfigurationKeys.PARTITION_NAME), "Older WorkUnit doesn't contain property partition name."); int executionAttempts = workUnit.getPropAsInt(ComplianceConfigurationKeys.EXECUTION_ATTEMPTS, ComplianceConfigurationKeys.DEFAULT_EXECUTION_ATTEMPTS); if (executionAttempts < this.maxWorkUnitExecutionAttempts) {
@Test(dependsOnMethods = "testSerializeState") public void testDeserializeState() throws IOException { WorkUnit workUnit1 = WorkUnit.createEmpty(); WorkUnit workUnit2 = WorkUnit.createEmpty(); SerializationUtils.deserializeState(this.fs, new Path(this.outputPath, "wu1"), workUnit1); SerializationUtils.deserializeState(this.fs, new Path(this.outputPath, "wu2"), workUnit2); Assert.assertEquals(workUnit1.getPropertyNames().size(), 2); Assert.assertEquals(workUnit1.getProp("foo"), "bar"); Assert.assertEquals(workUnit1.getPropAsInt("a"), 10); Assert.assertEquals(workUnit2.getPropertyNames().size(), 2); Assert.assertEquals(workUnit2.getProp("foo"), "baz"); Assert.assertEquals(workUnit2.getPropAsInt("b"), 20); }
@Test(dependsOnMethods = "testSerializeToFile") public void testDeserializeFromFile() throws IOException { WorkUnit workUnit1 = WorkUnit.createEmpty(); WorkUnit workUnit2 = WorkUnit.createEmpty(); try (ParallelRunner parallelRunner = new ParallelRunner(2, this.fs)) { parallelRunner.deserializeFromFile(workUnit1, new Path(this.outputPath, "wu1")); parallelRunner.deserializeFromFile(workUnit2, new Path(this.outputPath, "wu2")); } Assert.assertEquals(workUnit1.getPropertyNames().size(), 2); Assert.assertEquals(workUnit1.getProp("foo"), "bar"); Assert.assertEquals(workUnit1.getPropAsInt("a"), 10); Assert.assertEquals(workUnit2.getPropertyNames().size(), 2); Assert.assertEquals(workUnit2.getProp("foo"), "baz"); Assert.assertEquals(workUnit2.getPropAsInt("b"), 20); }
this.filesToPull = Lists.newArrayList(workUnitState.getPropAsList(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, "")); this.statusCount = this.workUnit.getPropAsInt(ConfigurationKeys.FILEBASED_REPORT_STATUS_ON_COUNT, ConfigurationKeys.DEFAULT_FILEBASED_REPORT_STATUS_ON_COUNT); this.shouldSkipFirstRecord = this.workUnitState.getPropAsBoolean(ConfigurationKeys.SOURCE_SKIP_FIRST_RECORD, false);
@Test public void testSourceExtractor() throws DataRecordException, IOException { SourceState state = new SourceState(); state.setProp(HelloWorldSource.NUM_HELLOS_FULL_KEY, 10); HelloWorldSource source = new HelloWorldSource(); List<WorkUnit> wus = source.getWorkunits(state); Assert.assertEquals(wus.size(), 10); for (int i = 0; i < wus.size(); ++i) { WorkUnit wu = wus.get(i); Assert.assertEquals(wu.getPropAsInt(HelloWorldSource.HELLO_ID_FULL_KEY), i + 1); WorkUnitState wuState = new WorkUnitState(wu, state); Extractor<String, String> extr = source.getExtractor(wuState); Assert.assertEquals(extr.getExpectedRecordCount(), 1); Assert.assertEquals(extr.readRecord(null), "Hello world "+ (i+1) + " !"); } }
this.filesToPull = Lists.newArrayList(workUnitState.getPropAsList(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, "")); this.statusCount = this.workUnit.getPropAsInt(ConfigurationKeys.FILEBASED_REPORT_STATUS_ON_COUNT, ConfigurationKeys.DEFAULT_FILEBASED_REPORT_STATUS_ON_COUNT); this.shouldSkipFirstRecord = this.workUnitState.getPropAsBoolean(ConfigurationKeys.SOURCE_SKIP_FIRST_RECORD, false);