@SuppressWarnings("unchecked") @Override public S getSchema() { return (S) this.workUnit.getProp(ConfigurationKeys.SOURCE_SCHEMA); }
public static Partition deserialize(WorkUnit workUnit) { long lowWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; long highWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; if (workUnit.getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY) != null) { lowWatermark = workUnit.getLowWatermark(LongWatermark.class).getValue(); highWatermark = workUnit.getExpectedHighWatermark(LongWatermark.class).getValue(); } return new Partition(lowWatermark, highWatermark, workUnit.getPropAsBoolean(Partition.IS_LAST_PARTIITON), workUnit.getPropAsBoolean(Partition.HAS_USER_SPECIFIED_HIGH_WATERMARK)); } }
@Test(dependsOnMethods = "testSerializeState") public void testDeserializeState() throws IOException { WorkUnit workUnit1 = WorkUnit.createEmpty(); WorkUnit workUnit2 = WorkUnit.createEmpty(); SerializationUtils.deserializeState(this.fs, new Path(this.outputPath, "wu1"), workUnit1); SerializationUtils.deserializeState(this.fs, new Path(this.outputPath, "wu2"), workUnit2); Assert.assertEquals(workUnit1.getPropertyNames().size(), 2); Assert.assertEquals(workUnit1.getProp("foo"), "bar"); Assert.assertEquals(workUnit1.getPropAsInt("a"), 10); Assert.assertEquals(workUnit2.getPropertyNames().size(), 2); Assert.assertEquals(workUnit2.getProp("foo"), "baz"); Assert.assertEquals(workUnit2.getPropAsInt("b"), 20); }
public String getOutputFilePath() { // Search for the properties in the workunit. // This search for the property first in State and then in the Extract of this workunit. String namespace = getProp(ConfigurationKeys.EXTRACT_NAMESPACE_NAME_KEY, ""); String table = getProp(ConfigurationKeys.EXTRACT_TABLE_NAME_KEY, ""); String extractId = getProp(ConfigurationKeys.EXTRACT_EXTRACT_ID_KEY, ""); // getPropAsBoolean and other similar methods are not overridden in WorkUnit class // Thus, to enable searching in WorkUnit's Extract, we use getProp, and not getPropAsBoolean boolean isFull = Boolean.parseBoolean(getProp(ConfigurationKeys.EXTRACT_IS_FULL_KEY)); return namespace.replaceAll("\\.", "/") + "/" + table + "/" + extractId + "_" + (isFull ? "full" : "append"); } }
@Override public String getConnectionUrl() { String urlPrefix = "jdbc:teradata://"; String host = this.workUnit.getProp(ConfigurationKeys.SOURCE_CONN_HOST_NAME); checkArgument(!isNullOrEmpty(host), "Connectionn host cannot be null or empty at %s", ConfigurationKeys.SOURCE_CONN_HOST_NAME); String port = this.workUnit.getProp(ConfigurationKeys.SOURCE_CONN_PORT,"1025"); String database = this.workUnit.getProp(ConfigurationKeys.SOURCE_QUERYBASED_SCHEMA); String defaultUrl = urlPrefix + host.trim() + "/TYPE=FASTEXPORT,DATABASE=" + database.trim() + ",DBS_PORT=" + port.trim() ; // use custom url from source.conn.host if Teradata jdbc url available return host.contains(urlPrefix) ? host.trim() : defaultUrl; }
/** * Get the expected high {@link Watermark} as a {@link JsonElement}. * * @return a {@link JsonElement} representing the expected high {@link Watermark}. */ public JsonElement getExpectedHighWatermark() { return JSON_PARSER.parse(getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)).getAsJsonObject() .get(WatermarkInterval.EXPECTED_HIGH_WATERMARK_TO_JSON_KEY); }
private void addDatasetUrnOptionally(WorkUnit workUnit) { if (!this.shouldEnableDatasetStateStore) { return; } workUnit.setProp(ConfigurationKeys.DATASET_URN_KEY, workUnit.getProp(TOPIC_NAME)); }
/** * Get the low {@link Watermark} as a {@link JsonElement}. * * @return a {@link JsonElement} representing the low {@link Watermark} or * {@code null} if the low {@link Watermark} is not set. */ public JsonElement getLowWatermark() { if (!contains(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)) { return null; } return JSON_PARSER.parse(getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)).getAsJsonObject() .get(WatermarkInterval.LOW_WATERMARK_TO_JSON_KEY); }
private void addSchemaFile(PartitionAwareFileRetriever.FileInfo dataFile, WorkUnit workUnit) throws IOException { Path schemaFile = new Path(new Path(dataFile.getFilePath()).getParent(), workUnit.getProp(ConfigurationKeys.SCHEMA_FILENAME, ConfigurationKeys.DEFAULT_SCHEMA_FILENAME)); if (fs.exists(schemaFile)) { workUnit.setProp(ConfigurationKeys.SOURCE_SCHEMA, schemaFile.toString()); } else { throw new IOException("Schema file " + schemaFile + " does not exist."); } }
@Nullable @Override public String apply(@Nullable WorkUnit input) { return input.getProp(WORK_UNIT_NAME); } }));
@Override public String getProp(String key) { String value = super.getProp(key); if (value == null) { value = this.workUnit.getProp(key); } if (value == null) { value = this.jobState.getProp(key); } return value; }
@Override public String getProp(String key, String def) { String value = super.getProp(key); if (value == null) { value = this.workUnit.getProp(key); } if (value == null) { value = this.jobState.getProp(key, def); } return value; }
private String readProp(String key, WorkUnitState workUnitState) { String value = workUnitState.getWorkunit().getProp(key); if (StringUtils.isBlank(value)) { value = workUnitState.getProp(key); } if (StringUtils.isBlank(value)) { value = workUnitState.getJobState().getProp(key); } return value; }
public void userCreatedStagingTable() throws SQLException { this.state.setProp(ConfigurationKeys.WRITER_STAGING_TABLE, STAGING_TABLE); when(this.commands.isEmpty(DB, STAGING_TABLE)).thenReturn(Boolean.TRUE); this.initializer.initialize(); Assert.assertEquals(STAGING_TABLE, this.workUnit.getProp(ConfigurationKeys.WRITER_STAGING_TABLE)); verify(this.commands, never()).createTableStructure(anyString(), anyString(), anyString()); verify(this.commands, never()).truncate(anyString(), anyString()); verify(this.commands, never()).drop(anyString(), anyString()); }
public void skipStagingTable() throws SQLException { this.state.setProp(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "partial"); this.state.setProp(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.toString(false)); this.initializer.initialize(); this.initializer.close(); Assert.assertEquals(DEST_TABLE, this.workUnit.getProp(ConfigurationKeys.WRITER_STAGING_TABLE)); verify(this.commands, never()).createTableStructure(anyString(), anyString(), anyString()); verify(this.commands, never()).truncate(anyString(), anyString()); verify(this.commands, never()).drop(anyString(), anyString()); }
public void skipStagingTableTruncateDestTable() throws SQLException { this.state.setProp(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "partial"); this.state.setProp(ConfigurationKeys.PUBLISH_DATA_AT_JOB_LEVEL, Boolean.toString(false)); this.state.setProp(JdbcPublisher.JDBC_PUBLISHER_REPLACE_FINAL_TABLE, Boolean.toString(true)); this.initializer.initialize(); Assert.assertEquals(DEST_TABLE, this.workUnit.getProp(ConfigurationKeys.WRITER_STAGING_TABLE)); verify(this.commands, never()).createTableStructure(anyString(), anyString(), anyString()); InOrder inOrder = inOrder(this.commands); inOrder.verify(this.commands, times(1)).truncate(DB, DEST_TABLE); this.initializer.close(); inOrder.verify(this.commands, never()).truncate(anyString(), anyString()); verify(this.commands, never()).drop(anyString(), anyString()); }
public void userCreatedStagingTableTruncate() throws SQLException { this.state.setProp(ConfigurationKeys.WRITER_STAGING_TABLE, STAGING_TABLE); this.state.setProp(ConfigurationKeys.WRITER_TRUNCATE_STAGING_TABLE, Boolean.toString(true)); when(this.commands.isEmpty(DB, STAGING_TABLE)).thenReturn(Boolean.TRUE); this.initializer.initialize(); Assert.assertEquals(STAGING_TABLE, this.workUnit.getProp(ConfigurationKeys.WRITER_STAGING_TABLE)); InOrder inOrder = inOrder(this.commands); inOrder.verify(this.commands, times(1)).truncate(DB, STAGING_TABLE); this.initializer.close(); inOrder.verify(this.commands, times(1)).truncate(DB, STAGING_TABLE); verify(this.commands, never()).createTableStructure(anyString(), anyString(), anyString()); verify(this.commands, never()).drop(anyString(), anyString()); }
private static void computeAndSetWorkUnitGuid(WorkUnit workUnit) throws IOException { Guid guid = Guid.fromStrings(workUnit.contains(ConfigurationKeys.CONVERTER_CLASSES_KEY) ? workUnit .getProp(ConfigurationKeys.CONVERTER_CLASSES_KEY) : ""); setWorkUnitGuid(workUnit, guid.append(deserializeCopyEntity(workUnit))); }
@BeforeClass public void setUp() throws Exception { WorkUnit workUnit = WorkUnit.createEmpty(); Properties properties = new Properties(); properties.load(new StringReader(TEST_JOB_CONFIG)); workUnit.addAll(properties); workUnit.setProp(ConfigurationKeys.JOB_ID_KEY, JobLauncherUtils.newJobId("GobblinTest1")); workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, JobLauncherUtils.newTaskId(workUnit.getProp(ConfigurationKeys.JOB_ID_KEY), 0)); this.taskContext = new TaskContext(new WorkUnitState(workUnit)); }