/** * get source record count from source * @return record count */ @Override public long getExpectedRecordCount() { return this.getSourceRecordCount(); }
/** * get schema(Metadata) corresponding to the data records * @return schema */ @Override public S getSchema() { return this.getOutputSchema(); }
@Override public String getWatermarkCondition(QueryBasedExtractor<?, ?> extractor, long watermarkValue, String operator) { return extractor .getTimestampPredicateCondition(this.watermarkColumn, watermarkValue, this.watermarkFormat, operator); }
@Override public LongWatermark getExpectedHighWatermark(Table table, long tableProcessTime) { return new LongWatermark(tableProcessTime); }
/** * close extractor read stream * update high watermark */ @Override public void close() { log.info("Updating the current state high water mark with " + this.highWatermark); this.workUnitState.setActualHighWatermark(new LongWatermark(this.highWatermark)); try { this.closeConnection(); } catch (Exception e) { log.error("Failed to close the extractor", e); } }
/** * Check if workunit needs to be created. Returns <code>true</code> If the * <code>updateTime</code> is greater than the <code>lowWatermark</code> and <code>maxLookBackTime</code> * <code>createTime</code> is not used. It exists for backward compatibility */ protected boolean shouldCreateWorkunit(long createTime, long updateTime, LongWatermark lowWatermark) { if (new DateTime(updateTime).isBefore(this.maxLookBackTime)) { return false; } return new DateTime(updateTime).isAfter(lowWatermark.getValue()); }
protected Set<SourceEntity> getFilteredSourceEntities(SourceState state) { Set<SourceEntity> unfilteredEntities = getSourceEntities(state); return getFilteredSourceEntitiesHelper(state, unfilteredEntities); }
private Set<SourceEntity> getFilteredEntities(SourceState state) { Set<SourceEntity> unfiltered = QueryBasedSource.getSourceEntitiesHelper(state); return QueryBasedSource.getFilteredSourceEntitiesHelper(state, unfiltered); }
public static SourceEntity fromSourceEntityName(String sourceEntityName) { return new SourceEntity(sourceEntityName, sanitizeEntityName(sourceEntityName)); }
protected Set<SourceEntity> getSourceEntities(State state) { return getSourceEntitiesHelper(state); }
@Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((getDatasetName() == null) ? 0 : getDatasetName().hashCode()); return result; } }
@Override public String getWatermarkCondition(QueryBasedExtractor<?, ?> extractor, long watermarkValue, String operator) { return extractor.getHourPredicateCondition(this.watermarkColumn, watermarkValue, this.watermarkFormat, operator); }
@Override public String getWatermarkCondition(QueryBasedExtractor<?, ?> extractor, long watermarkValue, String operator) { return extractor.getDatePredicateCondition(this.watermarkColumn, watermarkValue, this.watermarkFormat, operator); }
public String getWatermarkSourceFormat(QueryBasedExtractor<?, ?> extractor) { return extractor.getWatermarkSourceFormat(this.watermarkType); }
private boolean isPullRequired() { return getFetchStatus(); }
@Override public int hashCode() { final int prime = 31; return _topicPartition.hashCode() * prime + _lwm.hashCode(); }
private ArrayList<DataRecord> generateRecords(int count) { ArrayList<DataRecord> records = new ArrayList<>(); while (count > 0) { records.add(new DataRecord(count, count)); count--; } return records; }
private void verify(TestQueryBasedExtractor testExtractor, int expectedCount) { int actualCount = 0; try { while (testExtractor.readRecord(null) != null) { actualCount++; } } catch (Exception e) { Assert.fail("There should not incur any exception"); } Assert.assertEquals(actualCount, expectedCount, "Expect " + expectedCount + " records!"); }
@Override public Iterator<DataRecord> getRecordSetFromSourceApi(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws IOException { try { return getRecordSet(schema, entity, workUnit, predicateList); } catch (DataRecordException e) { e.printStackTrace(); return null; } } }
public static WorkUnitState createWus(String dbName, String tableName, long watermark) { WorkUnitState wus = new WorkUnitState(); wus.setActualHighWatermark(new LongWatermark(watermark)); wus.setProp(ConfigurationKeys.DATASET_URN_KEY, dbName + "@" + tableName); wus.setProp(ConfigurationKeys.JOB_ID_KEY, "jobId"); return wus; } }