/** * Creates a {@link Multimap} that maps {@link Extract} to their corresponds {@link WorkUnitState}s. * * @see Multimap */ public static Multimap<Extract, WorkUnitState> createExtractToWorkUnitStateMap( Collection<? extends WorkUnitState> workUnitStates) { Multimap<Extract, WorkUnitState> extractToWorkUnitStateMap = ArrayListMultimap.create(); for (WorkUnitState workUnitState : workUnitStates) { extractToWorkUnitStateMap.put(workUnitState.getExtract(), workUnitState); } return extractToWorkUnitStateMap; }
/** * Get properties set in the previous run for the same table as the {@link WorkUnit}. * * @return properties as a {@link State} object */ public State getPreviousTableState() { return getExtract().getPreviousTableState(); }
@Override public MessageType convertSchema(JsonArray inputSchema, WorkUnitState workUnit) throws SchemaConversionException { String fieldName = workUnit.getExtract().getTable(); JsonSchema jsonSchema = new JsonSchema(inputSchema); jsonSchema.setColumnName(fieldName); recordConverter = new RecordConverter(jsonSchema, ROOT); return (MessageType) recordConverter.schema(); }
@Override public String toString() { return super.toString() + "\nWorkUnit: " + getWorkunit().toString() + "\nExtract: " + getExtract().toString() + "\nJobState: " + this.jobState.toString(); }
@Override public Schema convertSchema(JsonArray schema, WorkUnitState workUnit) throws SchemaConversionException { try { JsonSchema jsonSchema = new JsonSchema(schema); jsonSchema.setColumnName(workUnit.getExtract().getTable()); recordConverter = new RecordConverter(jsonSchema, workUnit, workUnit.getExtract().getNamespace()); } catch (UnsupportedDateTypeException e) { throw new SchemaConversionException(e); } Schema recordSchema = recordConverter.schema(); if (workUnit .getPropAsBoolean(CONVERTER_AVRO_NULLIFY_FIELDS_ENABLED, DEFAULT_CONVERTER_AVRO_NULLIFY_FIELDS_ENABLED)) { return this.generateSchemaWithNullifiedField(workUnit, recordSchema); } return recordSchema; }
@Test public void testGetDefaultWriterFilePathWithWorkUnitState() { String namespace = "gobblin.test"; String tableName = "test-table"; SourceState sourceState = new SourceState(); WorkUnit workUnit = WorkUnit.create(new Extract(sourceState, TableType.APPEND_ONLY, namespace, tableName)); WorkUnitState workUnitState = new WorkUnitState(workUnit); Assert.assertEquals(WriterUtils.getWriterFilePath(workUnitState, 0, 0), new Path(workUnitState.getExtract() .getOutputFilePath())); Assert.assertEquals(WriterUtils.getWriterFilePath(workUnitState, 2, 0), new Path(workUnitState.getExtract() .getOutputFilePath(), ConfigurationKeys.DEFAULT_FORK_BRANCH_NAME + "0")); }
@Test public void schemaWithRecordOfArray() throws Exception { String testName = "schemaWithRecordOfArray"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); JsonObject expected = getExpectedSchema(testName).getAsJsonObject(); RecordConverter converter = new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "something")); Assert.assertEquals(avroSchemaToJsonElement(converter), expected); }
@Test public void schemaWithRecordIsNullable() throws Exception { String testName = "schemaWithRecordIsNullable"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); JsonArray expected = getExpectedSchema(testName).getAsJsonArray(); RecordConverter converter = new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "something")); Assert.assertEquals(avroSchemaToJsonElement(converter), expected); }
@Test public void schemaWithRecord() throws DataConversionException, SchemaConversionException, UnsupportedDateTypeException { String testName = "schemaWithRecord"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); JsonObject expected = getExpectedSchema(testName).getAsJsonObject(); JsonSchema jsonSchema = new JsonSchema(schema); jsonSchema.setColumnName("dummy1"); RecordConverter converter = new RecordConverter(jsonSchema, state, buildNamespace(state.getExtract().getNamespace(), "something")); Assert.assertEquals(avroSchemaToJsonElement(converter), expected); }
@Test public void schemaWithRecordOfEnum() throws Exception { String testName = "schemaWithRecordOfEnum"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); JsonObject expected = getExpectedSchema(testName).getAsJsonObject(); RecordConverter converter = new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "something")); Assert.assertEquals(avroSchemaToJsonElement(converter), expected); }
@Test public void schemaWithRecordOfMap() throws Exception { String testName = "schemaWithRecordOfMap"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); JsonObject expected = getExpectedSchema(testName).getAsJsonObject(); RecordConverter converter = new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "something")); Assert.assertEquals(avroSchemaToJsonElement(converter), expected); }
@Test public void schemaWithRecordOfRecord() throws Exception { String testName = "schemaWithRecordOfRecord"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); JsonObject expected = getExpectedSchema(testName).getAsJsonObject(); RecordConverter converter = new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "something")); Assert.assertEquals(avroSchemaToJsonElement(converter), expected); }
@Test(expectedExceptions = IllegalStateException.class) public void schemaWithMapValuesAsJsonArray() throws Exception { String testName = "schemaWithMapValuesAsJsonArray"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "something")); }
@Test(expectedExceptions = UnsupportedOperationException.class) public void schemaWithMapValuesAsJsonNull() throws Exception { String testName = "schemaWithMapValuesAsJsonNull"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "something")); }
@Test public void schemaWithRecordOfRecordCheckNamespace() throws Exception { String testName = "schemaWithRecordOfRecordCheckNamespace"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); JsonObject expected = getExpectedSchema(testName).getAsJsonObject(); RecordConverter converter = new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "person")); Assert.assertEquals(avroSchemaToJsonElement(converter), expected); Assert.assertEquals(converter.schema().getField("someperson").schema().getNamespace(), "namespace.person.myrecord"); Assert.assertEquals(converter.schema().getNamespace(), "namespace.person"); }
@Test public void schemaWithRecordOfEnumCheckNamespace() throws Exception { String testName = "schemaWithRecordOfEnumCheckNamespace"; JsonObject schema = getSchemaData(testName).getAsJsonObject(); JsonObject expected = getExpectedSchema(testName).getAsJsonObject(); RecordConverter converter = new RecordConverter(new JsonSchema(schema), state, buildNamespace(state.getExtract().getNamespace(), "something")); Assert.assertEquals(avroSchemaToJsonElement(converter), expected); Assert.assertEquals(converter.schema().getField("someperson").schema().getNamespace(), "namespace.something.myrecord"); Assert.assertEquals(converter.schema().getNamespace(), "namespace.something"); }
ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 0)); fs.mkdirs(writerStagingPath0); ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 1)); fs.mkdirs(writerStagingPath1); ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 0)); fs.mkdirs(writerOutputPath0); ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 1)); fs.mkdirs(writerOutputPath1);
/** * Get properties set in the previous run for the same table as the {@link WorkUnit}. * * @return properties as a {@link State} object */ public State getPreviousTableState() { return getExtract().getPreviousTableState(); }
@Override public String toString() { return super.toString() + "\nWorkUnit: " + getWorkunit().toString() + "\nExtract: " + getExtract().toString() + "\nJobState: " + this.jobState.toString(); }
@Override public MessageType convertSchema(JsonArray inputSchema, WorkUnitState workUnit) throws SchemaConversionException { String fieldName = workUnit.getExtract().getTable(); JsonSchema jsonSchema = new JsonSchema(inputSchema); jsonSchema.setColumnName(fieldName); recordConverter = new RecordConverter(jsonSchema, ROOT); return (MessageType) recordConverter.schema(); }