/** * Set the high watermark. * * @param value high watermark * @deprecated use {@link #setActualHighWatermark(Watermark)}. */ @Deprecated public void setHighWaterMark(long value) { setProp(ConfigurationKeys.WORK_UNIT_STATE_RUNTIME_HIGH_WATER_MARK, value); }
/** * Set the current runtime state of the {@link WorkUnit}. * * @param state {@link WorkingState} of the {@link WorkUnit} */ public void setWorkingState(WorkingState state) { setProp(ConfigurationKeys.WORK_UNIT_WORKING_STATE_KEY, state.toString()); }
/** * Backoff the actual high watermark to the low watermark returned by {@link WorkUnit#getLowWatermark()}. */ public void backoffActualHighWatermark() { JsonElement lowWatermark = this.workUnit.getLowWatermark(); if (lowWatermark == null) { return; } setProp(ConfigurationKeys.WORK_UNIT_STATE_ACTUAL_HIGH_WATER_MARK_KEY, lowWatermark.toString()); }
@Override public String getSchema() throws IOException { // Source is responsible to set SOURCE_SCHEMA ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); IOUtils.copyBytes(fs.open( new Path(workUnitState.getProp(ConfigurationKeys.SOURCE_SCHEMA))), outputStream, 4096, false); String schema = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); workUnitState.setProp((ConfigurationKeys.CONVERTER_AVRO_SCHEMA_KEY), schema); return schema; }
@Override public Extractor<String, String> getExtractor(WorkUnitState state) throws IOException { if (!state.contains(SOURCE_FILEBASED_OPTIONAL_DOWNLOADER_CLASS)) { state.setProp(SOURCE_FILEBASED_OPTIONAL_DOWNLOADER_CLASS, TokenizedFileDownloader.class.getName()); } return new FileBasedExtractor<>(state, new HadoopFsHelper(state)); }
private void addStateToWorkunit(State s, WorkUnitState wuState) { for (Map.Entry<Object, Object> prop : s.getProperties().entrySet()) { wuState.setProp((String) prop.getKey(), prop.getValue()); } }
DummyExtractor(WorkUnitState workUnitState) { this.workUnitState = workUnitState; workUnitState.setProp(FOO, BAR); this.current = Integer.parseInt(this.workUnitState.getProp(ConfigurationKeys.WORK_UNIT_LOW_WATER_MARK_KEY)); }
@Override public Integer readRecord(Integer reuse) throws DataRecordException, IOException { if (this.current > this.workUnitState.getPropAsInt(ConfigurationKeys.WORK_UNIT_HIGH_WATER_MARK_KEY)) { return null; } this.workUnitState.setProp(LAST_READ_RECORD_KEY, this.current); return this.current++; }
public void testConversionWithJsonTemplate() throws DataConversionException, IOException, JSONException { JsonParser parser = new JsonParser(); String expectedResourceKey = "/sobject/user/John"; String expectedJsonStr = "{ \"name\" : \"John\", \"favoriteNumber\" : 9, \"city\" : \"Mountain view\" }"; RestEntry<JsonObject> expected = new RestEntry<JsonObject>(expectedResourceKey, parser.parse(expectedJsonStr).getAsJsonObject()); WorkUnitState workUnitState = new WorkUnitState(); workUnitState.setProp(AvroToRestJsonEntryConverter.CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "/sobject/user/${name}"); String template = "name=${name},favoriteNumber=${favorite_number},city=${address.city}"; workUnitState.setProp(AvroToRestJsonEntryConverter.CONVERTER_AVRO_REST_JSON_ENTRY_TEMPLATE, template); testConversion(expected, workUnitState); }
@BeforeClass public void setUp() throws Exception { String avroSchemaString = IOUtils.toString(this.getClass().getResourceAsStream("/converter/jsonToAvroSchema.avsc"), StandardCharsets.UTF_8); this.jsonRecord = new JsonParser().parse(IOUtils.toString(this.getClass().getResourceAsStream( "/converter/jsonToAvroRecord.json"), StandardCharsets.UTF_8)).getAsJsonObject(); SourceState source = new SourceState(); this.state = new WorkUnitState( source.createWorkUnit(source.createExtract(TableType.SNAPSHOT_ONLY, "test_table", "test_namespace"))); this.state.setProp(ConfigurationKeys.CONVERTER_AVRO_SCHEMA_KEY, avroSchemaString); this.state.setProp(ConfigurationKeys.CONVERTER_IGNORE_FIELDS, "fieldToIgnore"); }
public static WorkUnitState createWus(String dbName, String tableName, long watermark) { WorkUnitState wus = new WorkUnitState(); wus.setActualHighWatermark(new LongWatermark(watermark)); wus.setProp(ConfigurationKeys.DATASET_URN_KEY, dbName + "@" + tableName); wus.setProp(ConfigurationKeys.JOB_ID_KEY, "jobId"); return wus; } }
public void testEqualConversion() throws DataConversionException, IOException, JSONException { JsonParser parser = new JsonParser(); String expectedResourceKey = "/sobject/user/John"; String expectedJsonStr = "{ \"name\" : \"John\", \"favorite_number\" : 9, \"favorite_color\" : \"blue\", \"date_of_birth\" : 1462387756716, \"last_modified\" : 0, \"created\" : 1462387756716, \"address\" : {\"city\" : \"Mountain view\", \"street_number\" : 2029 } }"; RestEntry<JsonObject> expected = new RestEntry<JsonObject>(expectedResourceKey, parser.parse(expectedJsonStr).getAsJsonObject()); WorkUnitState workUnitState = new WorkUnitState(); workUnitState.setProp(AvroToRestJsonEntryConverter.CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "/sobject/user/${name}"); testConversion(expected, workUnitState); }
@Test public void test() throws IOException { WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(true)); TestInstrumentedDataWriter writer = new TestInstrumentedDataWriter(state); testBase(writer); }
@Test public void test() { WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(true)); TestInstrumentedRowLevelPolicy policy = new TestInstrumentedRowLevelPolicy(state, null); testBase(policy); }
public void testBase(InstrumentedForkOperatorBase<String, String> fork) throws Exception { WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(true)); fork.init(state); fork.forkDataRecord(new WorkUnitState(), "in"); Map<String, Long> metrics = MetricsHelper.dumpMetrics(fork.getMetricContext()); Assert.assertEquals(metrics.get(MetricNames.ForkOperatorMetrics.RECORDS_IN_METER), Long.valueOf(1)); Assert.assertEquals(metrics.get(MetricNames.ForkOperatorMetrics.FORKS_OUT_METER), Long.valueOf(2)); Assert.assertEquals(metrics.get(MetricNames.ForkOperatorMetrics.FORK_TIMER), Long.valueOf(1)); Assert.assertEquals(MetricsHelper.dumpTags(fork.getMetricContext()).get("construct"), Constructs.FORK_OPERATOR.toString()); }
@Test public void testReadPreviousNullWatermarks() throws Exception { WorkUnitState previousWus = new WorkUnitState(); previousWus.setProp(ConfigurationKeys.DATASET_URN_KEY, "test_dataset_urn"); previousWus.setProp(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY, true); SourceState state = new SourceState(new State(), Lists.newArrayList(previousWus)); PartitionLevelWatermarker watermarker = new PartitionLevelWatermarker(state); Assert.assertEquals(watermarker.getPreviousWatermarks().size(), 0); }
@Test public void testBuiltInGsonDeserializer() throws ReflectiveOperationException { WorkUnitState mockWorkUnitState = getMockWorkUnitState(0L, 10L); mockWorkUnitState.setProp(KafkaDeserializerExtractor.KAFKA_DESERIALIZER_TYPE, KafkaDeserializerExtractor.Deserializers.GSON.name()); KafkaDeserializerExtractor kafkaDecoderExtractor = new KafkaDeserializerExtractor(mockWorkUnitState); Assert.assertEquals(kafkaDecoderExtractor.getKafkaDeserializer().getClass(), KafkaDeserializerExtractor.Deserializers.GSON.getDeserializerClass()); Assert.assertEquals(kafkaDecoderExtractor.getKafkaSchemaRegistry().getClass(), KafkaDeserializerExtractor.Deserializers.GSON.getSchemaRegistryClass()); }
private TaskState getStreamingTaskState() { WorkUnitState workUnitState = new WorkUnitState(WorkUnit.create( new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName()))); workUnitState.setProp(ConfigurationKeys.TASK_KEY_KEY, "1234"); TaskState taskState = new TaskState(workUnitState); taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false)); taskState.setProp(TaskConfigurationKeys.TASK_EXECUTION_MODE, ExecutionModel.STREAMING.name()); taskState.setJobId("1234"); taskState.setTaskId("testContinuousTaskId"); return taskState; }
@Test public void testDecorated() throws DataRecordException, IOException { WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(true)); InstrumentedExtractorBase instrumentedExtractor = new InstrumentedExtractorDecorator(state, new TestInstrumentedExtractor(state) ); testBase(instrumentedExtractor); InstrumentedExtractorBase nonInstrumentedExtractor = new InstrumentedExtractorDecorator(state, new TestExtractor()); testBase(nonInstrumentedExtractor); }