@Override public Extractor<Schema, GenericRecord> getExtractor(WorkUnitState state) throws IOException { return new DatePartitionedAvroFileExtractor(state); } }
@Override public Extractor<String, String> getExtractor(WorkUnitState state) throws IOException { return new DatePartitionedJsonFileExtractor(state); } }
/** * Read an {@link StreamEntity}. By default, just return result of {@link #readRecordEnvelope()}. */ default StreamEntity<D> readStreamEntity() throws DataRecordException, IOException { return readRecordEnvelope(); }
@Override public D readRecord(D reuse) throws DataRecordException, IOException { return actualExtractor.readRecord(reuse); }
@Override public S getSchema() throws IOException { return this.embeddedExtractor.getSchema(); }
@Override public long getExpectedRecordCount() { return this.embeddedExtractor.getExpectedRecordCount(); }
@Override public JsonElement toJson() { return WatermarkSerializerHelper.convertWatermarkToJson(this); }
@Override public long getHighWatermark() { return actualExtractor.getHighWatermark(); } }
@Override public short calculatePercentCompletion(Watermark lowWatermark, Watermark highWatermark) { TestWatermark testLowWatermark = GSON.fromJson(lowWatermark.toJson(), TestWatermark.class); TestWatermark testHighWatermark = GSON.fromJson(highWatermark.toJson(), TestWatermark.class); return (short) (100 * (this.watermark - testLowWatermark.getLongWatermark()) / (testHighWatermark .getLongWatermark() - testLowWatermark.getLongWatermark())); }
@Override public int compareTo(AcknowledgableWatermark o) { return _checkpointableWatermark.compareTo(o._checkpointableWatermark); }
@Override public short calculatePercentCompletion(Watermark lowWatermark, Watermark highWatermark) { return comparable.calculatePercentCompletion(lowWatermark, highWatermark); }
private void verifyWorkUnits(List<WorkUnit> workunits) throws IOException, DataRecordException { verifyWorkUnits(workunits, RECORD_SIZE); }
public Extractor getExtractor(WorkUnitState state) throws IOException { return new DummyExtractor(state); }
private int getNumRecords(Extractor<?, ?> extractor) throws DataRecordException, IOException { int numRecords = 0; while (extractor.readRecord(null) != null) { numRecords++; } return numRecords; }
@Override public S getSchema() throws IOException { return this.extractor.getSchema(); }
@Override public long getExpectedRecordCount() { return actualExtractor.getExpectedRecordCount(); }
@Override public JsonElement toJson() { return WatermarkSerializerHelper.convertWatermarkToJson(this); }
@Override public long getHighWatermark() { return this.embeddedExtractor.getHighWatermark(); }
@Override protected RecordEnvelope<D> readRecordEnvelopeImpl() throws DataRecordException, IOException { return this.embeddedExtractor.readRecordEnvelope(); }
@Override public JsonElement toJson() { return WatermarkSerializerHelper.convertWatermarkToJson(this); }