@Override public DataWriterBuilder<S, D> withSchema(S schema) { super.withSchema(schema); return typedSelf(); }
/** * Build a {@link org.apache.gobblin.writer.DataWriter} for writing fetched data records. */ private DataWriter<Object> buildWriter() throws IOException { DataWriterBuilder<Object, Object> builder = this.taskContext.getDataWriterBuilder(this.branches, this.index) .writeTo(Destination.of(this.taskContext.getDestinationType(this.branches, this.index), this.taskState)) .writeInFormat(this.taskContext.getWriterOutputFormat(this.branches, this.index)).withWriterId(this.taskId) .withSchema(this.convertedSchema.orNull()).withBranches(this.branches).forBranch(this.index); if (this.taskAttemptId.isPresent()) { builder.withAttemptId(this.taskAttemptId.get()); } DataWriter<Object> writer = new PartitionedDataWriter<>(builder, this.taskContext.getTaskState()); logger.info("Wrapping writer " + writer); return new DataWriterWrapperBuilder<>(writer, this.taskState).build(); }
@BeforeClass public void setUp() throws Exception { // Making the staging and/or output dirs if necessary File stagingDir = new File(TestConstants.TEST_STAGING_DIR); File outputDir = new File(TestConstants.TEST_OUTPUT_DIR); if (!stagingDir.exists()) { stagingDir.mkdirs(); } if (!outputDir.exists()) { outputDir.mkdirs(); } this.schema = new Schema.Parser().parse(TestConstants.AVRO_SCHEMA); this.filePath = TestConstants.TEST_EXTRACT_NAMESPACE.replaceAll("\\.", "/") + "/" + TestConstants.TEST_EXTRACT_TABLE + "/" + TestConstants.TEST_EXTRACT_ID + "_" + TestConstants.TEST_EXTRACT_PULL_TYPE; properties = new State(); properties.setProp(ConfigurationKeys.WRITER_BUFFER_SIZE, ConfigurationKeys.DEFAULT_BUFFER_SIZE); properties.setProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, TestConstants.TEST_FS_URI); properties.setProp(ConfigurationKeys.WRITER_STAGING_DIR, TestConstants.TEST_STAGING_DIR); properties.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, TestConstants.TEST_OUTPUT_DIR); properties.setProp(ConfigurationKeys.WRITER_FILE_PATH, this.filePath); properties.setProp(ConfigurationKeys.WRITER_FILE_NAME, TestConstants.TEST_FILE_NAME); // Build a writer to write test records this.writer = new AvroDataWriterBuilder().writeTo(Destination.of(Destination.DestinationType.HDFS, properties)) .writeInFormat(WriterOutputFormat.AVRO).withWriterId(TestConstants.TEST_WRITER_ID).withSchema(this.schema) .withBranches(1).forBranch(0).build(); }
.withWriterId(WRITER_ID).withSchema(this.schema).withBranches(1).forBranch(0); this.writer = new PartitionedDataWriter<Schema, GenericRecord>(builder, properties);
private SimpleDataWriter buildSimpleDataWriter() throws IOException { SimpleDataWriterBuilder b = (SimpleDataWriterBuilder)new SimpleDataWriterBuilder() .writeTo(Destination.of(Destination.DestinationType.HDFS, properties)).writeInFormat(WriterOutputFormat.AVRO) .withWriterId(TestConstants.TEST_WRITER_ID).withSchema(this.schema).forBranch(0); return new SimpleDataWriter(b, properties); }
.writeInFormat(WriterOutputFormat.AVRO) .withWriterId("writer-1") .withSchema(this.schema) .withBranches(1).forBranch(0);
@Override public DataWriterBuilder<S, D> withSchema(S schema) { super.withSchema(schema); return typedSelf(); }
/** * Build a {@link org.apache.gobblin.writer.DataWriter} for writing fetched data records. */ private DataWriter<Object> buildWriter() throws IOException { DataWriterBuilder<Object, Object> builder = this.taskContext.getDataWriterBuilder(this.branches, this.index) .writeTo(Destination.of(this.taskContext.getDestinationType(this.branches, this.index), this.taskState)) .writeInFormat(this.taskContext.getWriterOutputFormat(this.branches, this.index)).withWriterId(this.taskId) .withSchema(this.convertedSchema.orNull()).withBranches(this.branches).forBranch(this.index); if (this.taskAttemptId.isPresent()) { builder.withAttemptId(this.taskAttemptId.get()); } DataWriter<Object> writer = new PartitionedDataWriter<>(builder, this.taskContext.getTaskState()); logger.info("Wrapping writer " + writer); return new DataWriterWrapperBuilder<>(writer, this.taskState).build(); }