/** * Create a CheckpointableWatermarkState object from a CheckpointableWatermark * @param watermark: the checkpointable watermark * @param gson: the instance of {@link Gson} to use for serializing the {@param watermark}. */ public CheckpointableWatermarkState(CheckpointableWatermark watermark, Gson gson) { super.setProp(watermark.getSource(), gson.toJsonTree(watermark)); super.setId(watermark.getSource()); }
public void setPlanningJobId (String jobName, String planningJobId) throws IOException { State state = getOrCreate(distributedStateStoreName, jobName); state.setId(jobName); state.setProp(GobblinClusterConfigurationKeys.PLANNING_ID_KEY, planningJobId); // fs state store use hdfs rename, which assumes the target file doesn't exist. if (stateStore instanceof FsStateStore) { this.deleteMapping(jobName); } this.stateStore.put(distributedStateStoreName, jobName, state); }
public void setActualJobId (String jobName, String planningJobId, String actualJobId) throws IOException { State state = getOrCreate(distributedStateStoreName, jobName); state.setId(jobName); state.setProp(GobblinClusterConfigurationKeys.PLANNING_ID_KEY, planningJobId); state.setProp(ConfigurationKeys.JOB_ID_KEY, actualJobId); // fs state store use hdfs rename, which assumes the target file doesn't exist. if (stateStore instanceof FsStateStore) { this.deleteMapping(jobName); } this.stateStore.put(distributedStateStoreName, jobName, state); }
/** * Test for the metadata query to see if the check for unsigned int is present */ @Test public void testUnsignedInt() throws SchemaException { State state = new WorkUnitState(); state.setId("id"); MysqlExtractor mysqlExtractor = new MysqlExtractor((WorkUnitState) state); List<Command> commands = mysqlExtractor.getSchemaMetadata("db", "table"); assertTrue(commands.get(0).getCommandType() == JdbcCommand.JdbcCommandType.QUERY); assertTrue(commands.get(0).getParams().get(0).contains("bigint")); assertTrue(commands.get(1).getCommandType() == JdbcCommand.JdbcCommandType.QUERYPARAMS); assertTrue(!commands.get(1).getParams().get(0).contains("unsigned")); // set option to promote unsigned int to bigint state.setProp(ConfigurationKeys.SOURCE_QUERYBASED_PROMOTE_UNSIGNED_INT_TO_BIGINT, "true"); commands = mysqlExtractor.getSchemaMetadata("db", "table"); assertTrue(commands.get(0).getCommandType() == JdbcCommand.JdbcCommandType.QUERY); assertTrue(commands.get(0).getParams().get(0).contains("bigint")); assertTrue(commands.get(1).getCommandType() == JdbcCommand.JdbcCommandType.QUERYPARAMS); assertTrue(commands.get(1).getParams().get(0).contains("unsigned")); }
@Test public void testPut() throws IOException { List<State> states = Lists.newArrayList(); State state1 = new State(); state1.setId("s1"); state1.setProp("k1", "v1"); states.add(state1); State state2 = new State(); state2.setId("s2"); state2.setProp("k2", "v2"); states.add(state2); State state3 = new State(); state3.setId("s3"); state3.setProp("k3", "v3"); states.add(state3); Assert.assertFalse(this.stateStore.exists("testStore", "testTable")); this.stateStore.putAll("testStore", "testTable", states); Assert.assertTrue(this.stateStore.exists("testStore", "testTable")); // for testing of getStoreNames this.stateStore.putAll("testStore2", "testTable", states); }
@Test public void testGetData() throws Exception { CommandOutput<JdbcCommand, ResultSet> output = new JdbcCommandOutput(); output.put(new JdbcCommand(), buildMockResultSet()); State state = new WorkUnitState(); state.setId("id"); JdbcExtractor jdbcExtractor = new MysqlExtractor((WorkUnitState) state); List<String> columnNames = Lists.newArrayListWithCapacity(COLUMNS.size()); for (MockJdbcColumn mockJdbcColumn:COLUMNS) { columnNames.add(mockJdbcColumn.getColumnName()); } jdbcExtractor.setHeaderRecord(columnNames); Iterator<JsonElement> itr = jdbcExtractor.getData(output); // Make sure there is an element in the iterator assertTrue(itr.hasNext()); JsonObject obj = itr.next().getAsJsonObject(); // Verify the columns for (MockJdbcColumn column : COLUMNS) { assertEquals(obj.get(column.getColumnName()).getAsString(), column.getValue()); } }
@BeforeClass public void setup() { output = new JdbcCommandOutput(); try { output.put(new JdbcCommand(), buildMockResultSet()); } catch (Exception e) { // hack for test failure assertEquals("PostgresqlExtractorTest: error initializing mock result set", "false"); } state = new WorkUnitState(); state.setId("id"); postgresqlExtractor = new PostgresqlExtractor((WorkUnitState) state); }
@BeforeClass public void setup() { output = new JdbcCommandOutput(); try { output.put(new JdbcCommand(), buildMockResultSet()); } catch (Exception e) { // hack for test failure assertEquals("OracleExtractorTest: error initializing mock result set", "false"); } state = new WorkUnitState(); state.setId("id"); oracleExtractor = new OracleExtractor((WorkUnitState) state); }
@BeforeClass public void setUp() throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); this.store = new FsCommitSequenceStore(fs, new Path("commit-sequence-store-test")); State props = new State(); props.setId("propsId"); props.setProp("prop1", "valueOfProp1"); props.setProp("prop2", "valueOfProp2"); DatasetState datasetState = new DatasetState(); datasetState.setDatasetUrn(this.datasetUrn); datasetState.incrementJobFailures(); this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn") .beginStep(FsRenameCommitStep.Builder.class).from(new Path("/ab/cd")).to(new Path("/ef/gh")).withProps(props) .endStep().beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn(this.datasetUrn) .withDatasetState(datasetState).withProps(props).endStep().build(); }
/** * Create a CheckpointableWatermarkState object from a CheckpointableWatermark * @param watermark: the checkpointable watermark * @param gson: the instance of {@link Gson} to use for serializing the {@param watermark}. */ public CheckpointableWatermarkState(CheckpointableWatermark watermark, Gson gson) { super.setProp(watermark.getSource(), gson.toJsonTree(watermark)); super.setId(watermark.getSource()); }