@Override public CopyableDataset datasetAtPath(Path path) throws IOException { return new TimeAwareRecursiveCopyableDataset(this.fs, path, this.props, this.datasetPattern); } }
public TimeAwareRecursiveCopyableDataset(FileSystem fs, Path rootPath, Properties properties, Path glob) { super(fs, rootPath, properties, glob); this.lookbackTime = properties.getProperty(LOOKBACK_TIME_KEY); PeriodFormatter periodFormatter = new PeriodFormatterBuilder().appendDays().appendSuffix("d").appendHours().appendSuffix("h").toFormatter(); this.lookbackPeriod = periodFormatter.parsePeriod(lookbackTime); this.datePattern = properties.getProperty(DATE_PATTERN_KEY); this.isPatternHourly = isDatePatternHourly(datePattern); this.currentTime = properties.containsKey(DATE_PATTERN_TIMEZONE_KEY) ? LocalDateTime.now( DateTimeZone.forID(DATE_PATTERN_TIMEZONE_KEY)) : LocalDateTime.now(DateTimeZone.forID(DEFAULT_DATE_PATTERN_TIMEZONE)); //Daily directories cannot have a "hourly" lookback pattern. But hourly directories can accept lookback pattern with days. if (!this.isPatternHourly) { Assert.assertTrue(isLookbackTimeStringDaily(this.lookbackTime), "Expected day format for lookback time; found hourly format"); } }
TimeAwareRecursiveCopyableDataset dataset = new TimeAwareRecursiveCopyableDataset(fs, baseDir1, properties, new Path("/tmp/src/*/hourly")); List<FileStatus> fileStatusList = dataset.getFilesAtPath(fs, baseDir1, pathFilter); properties.setProperty(TimeAwareRecursiveCopyableDataset.LOOKBACK_TIME_KEY, NUM_LOOKBACK_DAYS_HOURS_STR); properties.setProperty(TimeAwareRecursiveCopyableDataset.DATE_PATTERN_KEY, "yyyy/MM/dd/HH"); dataset = new TimeAwareRecursiveCopyableDataset(fs, baseDir1, properties, new Path("/tmp/src/*/hourly")); fileStatusList = dataset.getFilesAtPath(fs, baseDir1, pathFilter); candidateFiles = new HashSet<>(); datePattern = "yyyy/MM/dd/HH"; properties.setProperty(TimeAwareRecursiveCopyableDataset.DATE_PATTERN_KEY, "yyyy/MM/dd"); dataset = new TimeAwareRecursiveCopyableDataset(fs, baseDir2, properties, new Path("/tmp/src/*/daily")); fileStatusList = dataset.getFilesAtPath(fs, baseDir2, pathFilter);
public TimeAwareRecursiveCopyableDataset(FileSystem fs, Path rootPath, Properties properties, Path glob) { super(fs, rootPath, properties, glob); this.lookbackTime = properties.getProperty(LOOKBACK_TIME_KEY); PeriodFormatter periodFormatter = new PeriodFormatterBuilder().appendDays().appendSuffix("d").appendHours().appendSuffix("h").toFormatter(); this.lookbackPeriod = periodFormatter.parsePeriod(lookbackTime); this.datePattern = properties.getProperty(DATE_PATTERN_KEY); this.isPatternHourly = isDatePatternHourly(datePattern); this.currentTime = properties.containsKey(DATE_PATTERN_TIMEZONE_KEY) ? LocalDateTime.now( DateTimeZone.forID(DATE_PATTERN_TIMEZONE_KEY)) : LocalDateTime.now(DateTimeZone.forID(DEFAULT_DATE_PATTERN_TIMEZONE)); //Daily directories cannot have a "hourly" lookback pattern. But hourly directories can accept lookback pattern with days. if (!this.isPatternHourly) { Assert.assertTrue(isLookbackTimeStringDaily(this.lookbackTime), "Expected day format for lookback time; found hourly format"); } }
@Test (expectedExceptions = AssertionError.class) public void testInstantiationError() { //Daily directories, but look back time has days and hours. We should expect an assertion error. Properties properties = new Properties(); properties.setProperty(TimeAwareRecursiveCopyableDataset.LOOKBACK_TIME_KEY, NUM_LOOKBACK_DAYS_HOURS_STR); properties.setProperty(TimeAwareRecursiveCopyableDataset.DATE_PATTERN_KEY, "yyyy/MM/dd"); TimeAwareRecursiveCopyableDataset dataset = new TimeAwareRecursiveCopyableDataset(fs, baseDir2, properties, new Path("/tmp/src/*/daily")); }
@Override public CopyableDataset datasetAtPath(Path path) throws IOException { return new TimeAwareRecursiveCopyableDataset(this.fs, path, this.props, this.datasetPattern); } }