MockAsyncHttpWriterBuilder(HttpClient client, MockRequestBuilder requestBuilder, MockResponseHandler responseHandler) { this.client = client; this.asyncRequestBuilder = requestBuilder; this.responseHandler = responseHandler; this.state = new WorkUnitState(); this.queueCapacity = 2; this.maxAttempts = 3; }
@Override public void readFields(DataInput in) throws IOException { int size = in.readInt(); for (int i = 0; i < size; i++) { WorkUnitState workUnitState = new WorkUnitState(); workUnitState.readFields(in); this.previousWorkUnitStates.add(new ImmutableWorkUnitState(workUnitState)); } super.readFields(in); }
@Test public void testConvertRecord() throws DataConversionException { TextToStringConverter textToStringConverter = (TextToStringConverter) new TextToStringConverter().init(new WorkUnitState()); Text text = new Text("test"); Iterator<String> iterator = textToStringConverter.convertRecord(null, text, new WorkUnitState()).iterator(); Assert.assertTrue(iterator.hasNext()); String textString = iterator.next(); Assert.assertEquals(textString, text.toString()); Assert.assertFalse(iterator.hasNext()); } }
@Test public void testConverter() throws DataConversionException, IOException { BytesToJsonConverter converter = new BytesToJsonConverter(); WorkUnitState state = new WorkUnitState(); JsonObject record = converter.convertRecord("dummySchema", IOUtils.toByteArray(this.getClass().getResourceAsStream("/converter/jsonToAvroRecord.json")), state).iterator().next(); Assert.assertEquals(record.get("longField").getAsLong(), 1234L); Assert.assertEquals(record.get("nestedRecords").getAsJsonObject().get("nestedField2").getAsString(), "test2"); } }
/** * Test for {@link StringFilterConverter#convertRecord(Class, String, WorkUnitState)} with a blank regex. */ @Test public void testConvertRecordWithNoRegex() throws DataConversionException { WorkUnitState workUnitState = new WorkUnitState(); StringFilterConverter converter = new StringFilterConverter(); converter.init(workUnitState); String test = "HelloWorld"; Iterator<String> itr = converter.convertRecord(String.class, test, workUnitState).iterator(); Assert.assertTrue(!itr.hasNext()); }
/** * Test that {@link StringSplitterConverter#init(WorkUnitState)} throws an {@link IllegalArgumentException} if the * parameter {@link ConfigurationKeys#CONVERTER_STRING_SPLITTER_DELIMITER} is not specified in the config. */ @Test(expectedExceptions = IllegalArgumentException.class) public void testInit() { WorkUnitState workUnitState = new WorkUnitState(); StringSplitterConverter converter = new StringSplitterConverter(); converter.init(workUnitState); }
@BeforeClass public void setUp() throws Exception { String avroSchemaString = IOUtils.toString(this.getClass().getResourceAsStream("/converter/jsonToAvroSchema.avsc"), StandardCharsets.UTF_8); this.jsonRecord = new JsonParser().parse(IOUtils.toString(this.getClass().getResourceAsStream( "/converter/jsonToAvroRecord.json"), StandardCharsets.UTF_8)).getAsJsonObject(); SourceState source = new SourceState(); this.state = new WorkUnitState( source.createWorkUnit(source.createExtract(TableType.SNAPSHOT_ONLY, "test_table", "test_namespace"))); this.state.setProp(ConfigurationKeys.CONVERTER_AVRO_SCHEMA_KEY, avroSchemaString); this.state.setProp(ConfigurationKeys.CONVERTER_IGNORE_FIELDS, "fieldToIgnore"); }
public void testBase(InstrumentedForkOperatorBase<String, String> fork) throws Exception { WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(true)); fork.init(state); fork.forkDataRecord(new WorkUnitState(), "in"); Map<String, Long> metrics = MetricsHelper.dumpMetrics(fork.getMetricContext()); Assert.assertEquals(metrics.get(MetricNames.ForkOperatorMetrics.RECORDS_IN_METER), Long.valueOf(1)); Assert.assertEquals(metrics.get(MetricNames.ForkOperatorMetrics.FORKS_OUT_METER), Long.valueOf(2)); Assert.assertEquals(metrics.get(MetricNames.ForkOperatorMetrics.FORK_TIMER), Long.valueOf(1)); Assert.assertEquals(MetricsHelper.dumpTags(fork.getMetricContext()).get("construct"), Constructs.FORK_OPERATOR.toString()); }
public void testEqualConversion() throws DataConversionException, IOException, JSONException { JsonParser parser = new JsonParser(); String expectedResourceKey = "/sobject/user/John"; String expectedJsonStr = "{ \"name\" : \"John\", \"favorite_number\" : 9, \"favorite_color\" : \"blue\", \"date_of_birth\" : 1462387756716, \"last_modified\" : 0, \"created\" : 1462387756716, \"address\" : {\"city\" : \"Mountain view\", \"street_number\" : 2029 } }"; RestEntry<JsonObject> expected = new RestEntry<JsonObject>(expectedResourceKey, parser.parse(expectedJsonStr).getAsJsonObject()); WorkUnitState workUnitState = new WorkUnitState(); workUnitState.setProp(AvroToRestJsonEntryConverter.CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "/sobject/user/${name}"); testConversion(expected, workUnitState); }
@Test(dataProvider = "fileDataProvider") public void testGz(final String filePath, final String expectedText) throws Exception { UnGzipConverter converter = new UnGzipConverter(); FileSystem fs = FileSystem.getLocal(new Configuration()); String fullPath = getClass().getClassLoader().getResource(filePath).getFile(); FileAwareInputStream fileAwareInputStream = FileAwareInputStream.builder() .file(CopyableFileUtils.getTestCopyableFile(filePath)).inputStream(fs.open(new Path(fullPath))).build(); Iterable<FileAwareInputStream> iterable = converter.convertRecord("outputSchema", fileAwareInputStream, new WorkUnitState()); String actual = readGzipStreamAsString(Iterables.getFirst(iterable, null).getInputStream()); Assert.assertEquals(actual.trim(), expectedText); }
public void testConversionWithJsonTemplate() throws DataConversionException, IOException, JSONException { JsonParser parser = new JsonParser(); String expectedResourceKey = "/sobject/user/John"; String expectedJsonStr = "{ \"name\" : \"John\", \"favoriteNumber\" : 9, \"city\" : \"Mountain view\" }"; RestEntry<JsonObject> expected = new RestEntry<JsonObject>(expectedResourceKey, parser.parse(expectedJsonStr).getAsJsonObject()); WorkUnitState workUnitState = new WorkUnitState(); workUnitState.setProp(AvroToRestJsonEntryConverter.CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "/sobject/user/${name}"); String template = "name=${name},favoriteNumber=${favorite_number},city=${address.city}"; workUnitState.setProp(AvroToRestJsonEntryConverter.CONVERTER_AVRO_REST_JSON_ENTRY_TEMPLATE, template); testConversion(expected, workUnitState); }
@BeforeClass public void setUp() throws Exception { WorkUnit workUnit = WorkUnit.createEmpty(); Properties properties = new Properties(); properties.load(new StringReader(TEST_JOB_CONFIG)); workUnit.addAll(properties); workUnit.setProp(ConfigurationKeys.JOB_ID_KEY, JobLauncherUtils.newJobId("GobblinTest1")); workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, JobLauncherUtils.newTaskId(workUnit.getProp(ConfigurationKeys.JOB_ID_KEY), 0)); this.taskContext = new TaskContext(new WorkUnitState(workUnit)); }
@Test public void testEquals() { SourceState sourceState = new SourceState(); sourceState.setProp("testKey", "true"); WorkUnitState workUnitState = new WorkUnitState(new WorkUnit(sourceState, null)); Assert.assertEquals(workUnitState, workUnitState); } }
@BeforeClass public static void setUp() { WorkUnit workUnit = new WorkUnit(new SourceState(), new Extract(new SourceState(), Extract.TableType.SNAPSHOT_ONLY, "namespace", "dummy_table")); state = new WorkUnitState(workUnit); Type listType = new TypeToken<JsonObject>() { }.getType(); Gson gson = new Gson(); testData = gson.fromJson(new InputStreamReader( JsonElementConversionFactoryTest.class.getResourceAsStream("/converter/JsonElementConversionFactoryTest.json")), listType); }
private TaskState getStreamingTaskState() { WorkUnitState workUnitState = new WorkUnitState(WorkUnit.create( new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName()))); workUnitState.setProp(ConfigurationKeys.TASK_KEY_KEY, "1234"); TaskState taskState = new TaskState(workUnitState); taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false)); taskState.setProp(TaskConfigurationKeys.TASK_EXECUTION_MODE, ExecutionModel.STREAMING.name()); taskState.setJobId("1234"); taskState.setTaskId("testContinuousTaskId"); return taskState; }
@Test public void testReadPreviousNullWatermarks() throws Exception { WorkUnitState previousWus = new WorkUnitState(); previousWus.setProp(ConfigurationKeys.DATASET_URN_KEY, "test_dataset_urn"); previousWus.setProp(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY, true); SourceState state = new SourceState(new State(), Lists.newArrayList(previousWus)); PartitionLevelWatermarker watermarker = new PartitionLevelWatermarker(state); Assert.assertEquals(watermarker.getPreviousWatermarks().size(), 0); }
/** * Create a {@link WorkUnitState} with a {@link org.apache.gobblin.broker.iface.SharedResourcesBroker} for running unit tests of * constructs. */ public static WorkUnitState createTestWorkUnitState() { return new WorkUnitState(new WorkUnit(), new State(), SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()). newSubscopedBuilder(new JobScopeInstance("jobName", "testJob"))); }
@BeforeTest public void setUp() throws SchemaConversionException { sampleRecord = TestUtils.generateRandomAvroRecord(); state = new WorkUnitState(); converter = new AvroToJsonStringConverter(); converter.convertSchema(sampleRecord.getSchema(), state); }
@Test public void testDecorated() throws DataRecordException, IOException { WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(true)); InstrumentedExtractorBase instrumentedExtractor = new InstrumentedExtractorDecorator(state, new TestInstrumentedExtractor(state) ); testBase(instrumentedExtractor); InstrumentedExtractorBase nonInstrumentedExtractor = new InstrumentedExtractorDecorator(state, new TestExtractor()); testBase(nonInstrumentedExtractor); }
@BeforeTest public void setUp() throws SchemaConversionException { sampleRecord = TestUtils.generateRandomAvroRecord(); state = new WorkUnitState(); converter = new AvroToJsonRecordWithMetadataConverter(); converter.convertSchema(sampleRecord.getSchema(), state); }