/** * Get the {@link org.apache.gobblin.source.workunit.Extract} associated with the {@link WorkUnit}. * * @return {@link org.apache.gobblin.source.workunit.Extract} associated with the {@link WorkUnit} */ public Extract getExtract() { return new Extract(this.workUnit.getExtract()); }
/** * Get the low {@link Watermark}. A default {@link Gson} object will be used to deserialize the watermark. * * @param watermarkClass the watermark class for this {@code WorkUnit}. * @return the low watermark in this {@code WorkUnit}. */ public <T extends Watermark> T getLowWatermark(Class<T> watermarkClass) { return getLowWatermark(watermarkClass, GSON); }
/** * Set the specified key, value pair in this {@link MultiWorkUnit} only, but do not propagate it to all the inner * {@link WorkUnit}s. * * @param key property key * @param value property value */ public void setPropExcludeInnerWorkUnits(String key, Object value) { super.setProp(key, value); }
/** * Factory method. * * @param extract {@link Extract} * @return A {@link WorkUnit} with the given {@link Extract} */ public static WorkUnit create(Extract extract) { return new WorkUnit(null, extract); }
/** * Get the expected high {@link Watermark}. A default {@link Gson} object will be used to deserialize the watermark. * * @param watermarkClass the watermark class for this {@code WorkUnit}. * @return the expected high watermark in this {@code WorkUnit}. */ public <T extends Watermark> T getExpectedHighWatermark(Class<T> watermarkClass) { return getExpectedHighWatermark(watermarkClass, GSON); }
@Override public long weight(WorkUnit workUnit) { return workUnit.getPropAsLong(this.field); } }
/** * Override {@link #workUnit}'s properties with new commonProps and specProps. */ public void setWuProperties(Properties commonProps, Properties specProps) { this.workUnit.setProps(commonProps, specProps); }
/** * Returns the HWM of the workUnit * {@inheritDoc} * @see org.apache.gobblin.source.extractor.filebased.FileBasedExtractor#getHighWatermark() */ @Override public long getHighWatermark() { return this.workUnit.getHighWaterMark(); } }
/** * Set a (non-globally) unique ID for this {@link Extract}. * * @param extractId unique ID for this {@link Extract} */ public void setExtractId(String extractId) { setProp(ConfigurationKeys.EXTRACT_EXTRACT_ID_KEY, extractId); }
/** * Create a new empty {@link MultiWorkUnit} instance. * * @return a new empty {@link MultiWorkUnit} instance */ public static MultiWorkUnit createEmpty() { return new MultiWorkUnit(); } }
/** * Get the dot-separated namespace of the table. * * @return dot-separated namespace of the table */ public String getNamespace() { return getProp(ConfigurationKeys.EXTRACT_NAMESPACE_NAME_KEY, ""); }
@Override public boolean equals(Object obj) { if (!(obj instanceof WeightedMultiWorkUnit)) { return false; } WeightedMultiWorkUnit weightedMultiWorkUnit = (WeightedMultiWorkUnit) obj; return this.weight == weightedMultiWorkUnit.getWeight(); }
/** * Get an {@link ImmutableWorkUnit} that wraps the internal {@link WorkUnit}. * * @return an {@link ImmutableWorkUnit} that wraps the internal {@link WorkUnit} */ public WorkUnit getWorkunit() { return new ImmutableWorkUnit(this.workUnit); }
@Override public Long load(WorkUnit key) throws Exception { return WeightComparator.this.weighter.weight(key); } });
/** * If this {@link Extract} has extract table type defined. * * @return <code>true</code> if it has, <code>false</code> otherwise. */ public boolean hasType() { return contains(ConfigurationKeys.EXTRACT_TABLE_TYPE_KEY); }
/** * Get a materialized collection of the {@link WorkUnit}s in this stream. Note this call will fail if * {@link #isSafeToMaterialize()} is false. */ public Collection<WorkUnit> getMaterializedWorkUnitCollection() { materialize(); return this.materializedWorkUnits; }
/** * This method will allow a work unit to be skipped if needed. */ public void skip() { this.setProp(ConfigurationKeys.WORK_UNIT_SKIP_KEY, true); }
/** * Factory method. * * @return An empty {@link WorkUnit}. */ public static WorkUnit createEmpty() { return new WorkUnit(); }
/** * Get the name of the table. * * @return name of the table */ public String getTable() { return getProp(ConfigurationKeys.EXTRACT_TABLE_NAME_KEY, ""); }