/** * Get append high water mark * * @param extractType Extract type * @return append high water mark */ private long getAppendHighWatermark(ExtractType extractType) { LOG.debug("Getting append high water mark"); if (this.isFullDump()) { LOG.info("Overriding high water mark with end value:" + ConfigurationKeys.SOURCE_QUERYBASED_END_VALUE); long highWatermark = this.state.getPropAsLong(ConfigurationKeys.SOURCE_QUERYBASED_END_VALUE, 0); if (highWatermark != 0) { // User specifies SOURCE_QUERYBASED_END_VALUE hasUserSpecifiedHighWatermark = true; } return highWatermark; } return this.getAppendWatermarkCutoff(extractType); }
Stopwatch stopwatch = Stopwatch.createStarted(); int threads = this.state.getPropAsInt(CompactionVerifier.COMPACTION_VERIFICATION_THREADS, 5); long timeOutInMinute = this.state.getPropAsLong(CompactionVerifier.COMPACTION_VERIFICATION_TIMEOUT_MINUTES, 30); long iterationCountLimit = this.state.getPropAsLong(CompactionVerifier.COMPACTION_VERIFICATION_ITERATION_COUNT_LIMIT, Integer.MAX_VALUE); long iteration = 0; Map<String, String> failedReasonMap = null;
/** * Get high water mark * * @param extractType Extract type * @param watermarkType Watermark type * @return high water mark in {@link Partitioner#WATERMARKTIMEFORMAT} */ @VisibleForTesting protected long getHighWatermark(ExtractType extractType, WatermarkType watermarkType) { LOG.debug("Getting high watermark"); String timeZone = this.state.getProp(ConfigurationKeys.SOURCE_TIMEZONE); long highWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; if (this.isWatermarkOverride()) { highWatermark = this.state.getPropAsLong(ConfigurationKeys.SOURCE_QUERYBASED_END_VALUE, 0); if (highWatermark == 0) { highWatermark = Long.parseLong(Utils.dateTimeToString(getCurrentTime(timeZone), WATERMARKTIMEFORMAT, timeZone)); } else { // User specifies SOURCE_QUERYBASED_END_VALUE hasUserSpecifiedHighWatermark = true; } LOG.info("Overriding high water mark with the given end value:" + highWatermark); } else { if (isSnapshot(extractType)) { highWatermark = this.getSnapshotHighWatermark(watermarkType); } else { highWatermark = this.getAppendHighWatermark(extractType); } } return (highWatermark == 0 ? ConfigurationKeys.DEFAULT_WATERMARK_VALUE : highWatermark); }
targetFs.getUri()); long maxSizePerBin = state.getPropAsLong(MAX_SIZE_MULTI_WORKUNITS, 0); long maxWorkUnitsPerMultiWorkUnit = state.getPropAsLong(MAX_WORK_UNITS_PER_BIN, 50); final long minWorkUnitWeight = Math.max(1, maxSizePerBin / maxWorkUnitsPerMultiWorkUnit); final Optional<CopyableFileWatermarkGenerator> watermarkGenerator =
histogramAdjust.add(group); if (histogramAdjust.getTotalRecordCount() > state .getPropAsLong(EARLY_STOP_TOTAL_RECORDS_LIMIT, DEFAULT_EARLY_STOP_TOTAL_RECORDS_LIMIT)) { break;
/** * Get append high water mark * * @param extractType Extract type * @return append high water mark */ private long getAppendHighWatermark(ExtractType extractType) { LOG.debug("Getting append high water mark"); if (this.isFullDump()) { LOG.info("Overriding high water mark with end value:" + ConfigurationKeys.SOURCE_QUERYBASED_END_VALUE); long highWatermark = this.state.getPropAsLong(ConfigurationKeys.SOURCE_QUERYBASED_END_VALUE, 0); if (highWatermark != 0) { // User specifies SOURCE_QUERYBASED_END_VALUE hasUserSpecifiedHighWatermark = true; } return highWatermark; } return this.getAppendWatermarkCutoff(extractType); }
Stopwatch stopwatch = Stopwatch.createStarted(); int threads = this.state.getPropAsInt(CompactionVerifier.COMPACTION_VERIFICATION_THREADS, 5); long timeOutInMinute = this.state.getPropAsLong(CompactionVerifier.COMPACTION_VERIFICATION_TIMEOUT_MINUTES, 30); long iterationCountLimit = this.state.getPropAsLong(CompactionVerifier.COMPACTION_VERIFICATION_ITERATION_COUNT_LIMIT, Integer.MAX_VALUE); long iteration = 0; Map<String, String> failedReasonMap = null;
/** * Get high water mark * * @param extractType Extract type * @param watermarkType Watermark type * @return high water mark in {@link Partitioner#WATERMARKTIMEFORMAT} */ @VisibleForTesting protected long getHighWatermark(ExtractType extractType, WatermarkType watermarkType) { LOG.debug("Getting high watermark"); String timeZone = this.state.getProp(ConfigurationKeys.SOURCE_TIMEZONE); long highWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; if (this.isWatermarkOverride()) { highWatermark = this.state.getPropAsLong(ConfigurationKeys.SOURCE_QUERYBASED_END_VALUE, 0); if (highWatermark == 0) { highWatermark = Long.parseLong(Utils.dateTimeToString(getCurrentTime(timeZone), WATERMARKTIMEFORMAT, timeZone)); } else { // User specifies SOURCE_QUERYBASED_END_VALUE hasUserSpecifiedHighWatermark = true; } LOG.info("Overriding high water mark with the given end value:" + highWatermark); } else { if (isSnapshot(extractType)) { highWatermark = this.getSnapshotHighWatermark(watermarkType); } else { highWatermark = this.getAppendHighWatermark(extractType); } } return (highWatermark == 0 ? ConfigurationKeys.DEFAULT_WATERMARK_VALUE : highWatermark); }
targetFs.getUri()); long maxSizePerBin = state.getPropAsLong(MAX_SIZE_MULTI_WORKUNITS, 0); long maxWorkUnitsPerMultiWorkUnit = state.getPropAsLong(MAX_WORK_UNITS_PER_BIN, 50); final long minWorkUnitWeight = Math.max(1, maxSizePerBin / maxWorkUnitsPerMultiWorkUnit); final Optional<CopyableFileWatermarkGenerator> watermarkGenerator =