/** * Constructor. * * @param type {@link TableType} * @param namespace dot separated namespace path * @param table table name */ public Extract(TableType type, String namespace, String table) { this(new SourceState(), type, namespace, table); }
@BeforeClass public static void setUp() { WorkUnit workUnit = new WorkUnit(new SourceState(), new Extract(new SourceState(), Extract.TableType.SNAPSHOT_ONLY, "namespace", "dummy_table")); state = new WorkUnitState(workUnit); Type listType = new TypeToken<JsonObject>() { }.getType(); Gson gson = new Gson(); testData = gson.fromJson(new InputStreamReader( JsonElementConversionFactoryTest.class.getResourceAsStream("/converter/JsonElementConversionFactoryTest.json")), listType); }
private static SourceState getCombinedState(SourceState state, State tableSpecificState) { if (tableSpecificState == null) { return state; } SourceState combinedState = new SourceState(state, state.getPreviousDatasetStatesByUrns(), state.getPreviousWorkUnitStates()); combinedState.addAll(tableSpecificState); return combinedState; }
@BeforeClass public static void setUp() { Type listType = new TypeToken<JsonObject>() { }.getType(); Gson gson = new Gson(); JsonObject testData = gson.fromJson( new InputStreamReader(JsonIntermediateToParquetGroupConverter.class.getResourceAsStream(RESOURCE_PATH)), listType); testCases = testData.getAsJsonObject(); SourceState source = new SourceState(); workUnit = new WorkUnitState( source.createWorkUnit(source.createExtract(Extract.TableType.SNAPSHOT_ONLY, "test_namespace", "test_table"))); }
@Test(expectedExceptions = IllegalArgumentException.class) public void testConnectFailsWithS3URLWithoutAWSCredentials() throws FileBasedHelperException { Configuration conf = new Configuration(); // plain conf, no S3 credentials SourceState sourceState = new SourceState(); sourceState.setProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, "s3://support.elasticmapreduce/spark/install-spark/"); HadoopFsHelper fsHelper = new HadoopFsHelper(sourceState, conf); fsHelper.connect(); }
@Test public void testNoWhitelist() throws Exception { BackfillHiveSource backfillHiveSource = new BackfillHiveSource(); SourceState state = new SourceState(); backfillHiveSource.initBackfillHiveSource(state); Partition sourcePartition = Mockito.mock(Partition.class, Mockito.RETURNS_SMART_NULLS); Assert.assertTrue(backfillHiveSource.shouldCreateWorkunit(sourcePartition, new LongWatermark(0))); }
@Test void numberOfWorkUnits() throws IOException { SourceState sourceState = new SourceState(); DatePartitionedJsonFileSource source = new DatePartitionedJsonFileSource(); initState(sourceState); List<WorkUnit> workUnits = source.getWorkunits(sourceState); Assert.assertEquals(3, workUnits.size()); }
/** * Test the never-retry policy. */ @Test public void testGetPreviousWorkUnitStatesNeverRetry() { SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates); sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "never"); Assert.assertEquals(this.testSource.getPreviousWorkUnitStatesForRetry(sourceState), Collections.EMPTY_LIST); }
@Test(expectedExceptions = IllegalStateException.class) public void testPartitionBeginBegoreTableBegin() throws Exception { SourceState state = new SourceState(); PartitionLevelWatermarker watermarker = new PartitionLevelWatermarker(state); Table table = mockTable("test_dataset_urn"); Partition partition = mockPartition(table, ImmutableList.of("")); watermarker.onPartitionProcessBegin(partition, 0l, 0l); }
@Test public void testEquals() { SourceState sourceState = new SourceState(); sourceState.setProp("testKey", "true"); WorkUnitState workUnitState = new WorkUnitState(new WorkUnit(sourceState, null)); Assert.assertEquals(workUnitState, workUnitState); } }
/** * Test when work unit retry disabled. */ @Test public void testGetPreviousWorkUnitStatesDisabledRetry() { SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates); sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_ENABLED_KEY, Boolean.FALSE); Assert.assertEquals(this.testSource.getPreviousWorkUnitStatesForRetry(sourceState), Collections.EMPTY_LIST); }
/** * Test the always-retry policy, with WORK_UNIT_RETRY_ENABLED_KEY enabled. */ @Test public void testGetPreviousWorkUnitStatesEnabledRetry() { SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates); sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_ENABLED_KEY, Boolean.TRUE); List<WorkUnitState> returnedWorkUnitStates = this.testSource.getPreviousWorkUnitStatesForRetry(sourceState); Assert.assertEquals(returnedWorkUnitStates, this.expectedPreviousWorkUnitStates); }
/** * Test when work unit retry policy is on full, but the job commit policy is "partial". */ @Test public void testGetPreviousWorkUnitStatesOnFullRetryPartialCommit() { SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates); sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "onfull"); sourceState.setProp(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "partial"); Assert.assertEquals(this.testSource.getPreviousWorkUnitStatesForRetry(sourceState), Collections.EMPTY_LIST); }
/** * Test when work unit retry policy is on partial, but the job commit policy is "full". */ @Test public void testGetPreviousWorkUnitStatesOnPartialRetryFullCommit() { SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates); sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "onpartial"); sourceState.setProp(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "full"); Assert.assertEquals(this.testSource.getPreviousWorkUnitStatesForRetry(sourceState), Collections.EMPTY_LIST); }
/** * Test when work unit retry policy is on partial, and the job commit policy is "partial". */ @Test public void testGetPreviousWorkUnitStatesOnPartialRetryPartialCommit() { SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates); sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "onpartial"); sourceState.setProp(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "partial"); List<WorkUnitState> returnedWorkUnitStates = this.testSource.getPreviousWorkUnitStatesForRetry(sourceState); Assert.assertEquals(returnedWorkUnitStates, this.expectedPreviousWorkUnitStates); }
@Test public void testGetDefaultWriterFilePath() { String namespace = "gobblin.test"; String tableName = "test-table"; SourceState sourceState = new SourceState(); WorkUnit state = WorkUnit.create(new Extract(sourceState, TableType.APPEND_ONLY, namespace, tableName)); Assert.assertEquals(WriterUtils.getWriterFilePath(state, 0, 0), new Path(state.getExtract().getOutputFilePath())); Assert.assertEquals(WriterUtils.getWriterFilePath(state, 2, 0), new Path(state.getExtract().getOutputFilePath(), ConfigurationKeys.DEFAULT_FORK_BRANCH_NAME + "0")); }
/** * Test when work unit retry policy is on full, and the job commit policy is "full". */ @Test public void testGetPreviousWorkUnitStatesOnFullRetryFullCommit() { SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates); sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "onfull"); sourceState.setProp(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "full"); List<WorkUnitState> returnedWorkUnitStates = this.testSource.getPreviousWorkUnitStatesForRetry(sourceState); Assert.assertEquals(returnedWorkUnitStates, this.expectedPreviousWorkUnitStates); }
private static SourceState getTestState(String dbName) { SourceState testState = new SourceState(); testState.setProp("hive.dataset.database", dbName); testState.setProp("hive.dataset.table.pattern", "*"); testState.setProp(ConfigurationKeys.JOB_ID_KEY, "testJobId"); return testState; }
private SourceState createSourceState(String topic) { SourceState state = new SourceState(); state.setProp(ConfigurationKeys.KAFKA_BROKERS, "localhost:" + kafkaTestHelper.getKafkaServerPort()); state.setProp(KafkaSource.TOPIC_WHITELIST, topic); state.setProp(KafkaSource.GOBBLIN_KAFKA_CONSUMER_CLIENT_FACTORY_CLASS, Kafka09ConsumerClient.Factory.class.getName()); state.setProp(KafkaSource.BOOTSTRAP_WITH_OFFSET, "earliest"); return state; }
@Test public void testReadPreviousNullWatermarks() throws Exception { WorkUnitState previousWus = new WorkUnitState(); previousWus.setProp(ConfigurationKeys.DATASET_URN_KEY, "test_dataset_urn"); previousWus.setProp(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY, true); SourceState state = new SourceState(new State(), Lists.newArrayList(previousWus)); PartitionLevelWatermarker watermarker = new PartitionLevelWatermarker(state); Assert.assertEquals(watermarker.getPreviousWatermarks().size(), 0); }