@Override public DataWriterBuilder<S, D> withBranches(int branches) { super.withBranches(branches); return typedSelf(); }
@Override public RetryerBuilder<Void> getRetryerBuilder() { if (writer instanceof Retriable) { return ((Retriable) writer).getRetryerBuilder(); } return RetryWriter.createRetryBuilder(state); }
private SimpleDataWriter buildSimpleDataWriter() throws IOException { SimpleDataWriterBuilder b = (SimpleDataWriterBuilder)new SimpleDataWriterBuilder() .writeTo(Destination.of(Destination.DestinationType.HDFS, properties)).writeInFormat(WriterOutputFormat.AVRO) .withWriterId(TestConstants.TEST_WRITER_ID).withSchema(this.schema).forBranch(0); return new SimpleDataWriter(b, properties); }
/** * For backward compatibility on how Fork creates writer, invoke fromState when it's called writeTo method. * @param destination * @return this */ @Override public AsyncHttpWriterBuilder<D, RQ, RP> writeTo(Destination destination) { super.writeTo(destination); return fromState(destination.getProperties()); }
/** * Build a {@link org.apache.gobblin.writer.DataWriter}. * * @return the built {@link org.apache.gobblin.writer.DataWriter} * @throws java.io.IOException if there is anything wrong building the writer */ @Override public DataWriter<Object> build() throws IOException { return new MetadataWriterWrapper<byte[]>(new SimpleDataWriter(this, this.destination.getProperties()), byte[].class, this.branches, this.branch, this.destination.getProperties()); }
@Override public void commit() throws IOException { writeMetadata(); wrappedWriter.commit(); }
@Override public DataWriterBuilder<S, D> withWriterId(String writerId) { super.withWriterId(writerId); return typedSelf(); }
@Override public B writeInFormat(WriterOutputFormat format) { super.writeInFormat(format); return typedSelf(); }
private ParquetDataWriterBuilder getParquetDataWriterBuilder() { ParquetDataWriterBuilder writerBuilder = new ParquetDataWriterBuilder(); writerBuilder.destination = Destination.of(Destination.DestinationType.HDFS, properties); writerBuilder.writerId = TestConstants.TEST_WRITER_ID; writerBuilder.schema = this.schema; writerBuilder.format = WriterOutputFormat.PARQUET; return writerBuilder; }
@Override public DataWriterBuilder<S, D> withSchema(S schema) { super.withSchema(schema); return typedSelf(); }
/** * Create a derived record envelope from this one. * Derived envelopes share the same watermark. * The original envelope must be acknowledged separately. */ public AcknowledgableRecordEnvelope derivedEnvelope(D record) { _watermark.incrementAck(); return new AcknowledgableRecordEnvelope(record, _watermark); }
@Override public void commit() throws IOException { Callable<Void> commitCall = new Callable<Void>() { @Override public Void call() throws Exception { writer.commit(); return null; } }; callWithRetry(commitCall); }
/** * Wait for a buffer empty occurrence */ @Override public void flush() throws IOException { waitForBufferEmpty(); } }
@Override public RetryerBuilder<Void> getRetryerBuilder() { if (writer instanceof Retriable) { return ((Retriable) writer).getRetryerBuilder(); } return RetryWriter.createRetryBuilder(state); }