public static boolean isValidFormat(String format) { Preconditions.checkNotNull(format); String[] formatTokens = format.split(COLON_SEPARATOR, MAX_FORMAT_TOKENS); Preconditions.checkArgument(formatTokens.length == MIN_FORMAT_TOKENS || formatTokens.length == MAX_FORMAT_TOKENS, FORMAT_TOKENS_ERROR_STR); Preconditions.checkArgument(formatTokens[FORMAT_SIZE_POSITION].matches(NUMBER_REGEX), FORMAT_PATTERN_ERROR_STR); Preconditions.checkArgument(DateTimeFormatUnitSpec.isValidUnitSpec(formatTokens[FORMAT_UNIT_POSITION])); if (formatTokens.length == MIN_FORMAT_TOKENS) { Preconditions.checkArgument(formatTokens[FORMAT_TIMEFORMAT_POSITION].equals(TimeFormat.EPOCH.toString()), TIME_FORMAT_ERROR_STR); } else { Preconditions .checkArgument(formatTokens[FORMAT_TIMEFORMAT_POSITION].equals(TimeFormat.SIMPLE_DATE_FORMAT.toString()), TIME_FORMAT_ERROR_STR); } return true; }
public DateTimeFormatPatternSpec(String timeFormat, String sdfPatternWithTz) { _timeFormat = TimeFormat.valueOf(timeFormat); if (_timeFormat.equals(TimeFormat.SIMPLE_DATE_FORMAT)) { Matcher m = SDF_PATTERN_WITH_TIMEZONE.matcher(sdfPatternWithTz); _sdfPattern = sdfPatternWithTz; if (m.find()) { _sdfPattern = m.group(SDF_PATTERN_GROUP).trim(); String timezoneString = m.group(TIMEZONE_GROUP).trim(); _dateTimeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(timezoneString)); } _dateTimeFormatter = DateTimeFormat.forPattern(_sdfPattern).withZone(_dateTimeZone); } }
/** * <ul> * <li>Given a timestamp in millis, convert it to the given format * This method should not do validation of outputGranularity. * The validation should be handled by caller using {@link #isValidFormat(String)}</li> * <ul> * <li>1) given dateTimeColumnValueMS = 1498892400000 and format=1:HOURS:EPOCH, * dateTimeSpec.fromMillis(1498892400000) = 416359 (i.e. dateTimeColumnValueMS/(1000*60*60))</li> * <li>2) given dateTimeColumnValueMS = 1498892400000 and format=5:MINUTES:EPOCH, * dateTimeSpec.fromMillis(1498892400000) = 4996308 (i.e. timeColumnValueMS/(1000*60*5))</li> * <li>3) given dateTimeColumnValueMS = 1498892400000 and * format=1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd, dateTimeSpec.fromMillis(1498892400000) = 20170701</li> * </ul> * </ul> * @param dateTimeColumnValueMS * @param toFormat - the format in which to convert the millis value * @param type - type of return value (can be int/long or string depending on time format) * @return dateTime column value in dateTimeFieldSpec */ public <T extends Object> T fromMillisToFormat(Long dateTimeColumnValueMS, Class<T> type) { Preconditions.checkNotNull(dateTimeColumnValueMS); Object dateTimeColumnValue = null; if (_patternSpec.getTimeFormat().equals(TimeFormat.EPOCH)) { dateTimeColumnValue = _unitSpec.getTimeUnit().convert(dateTimeColumnValueMS, TimeUnit.MILLISECONDS) / _size; } else { dateTimeColumnValue = _patternSpec.getDateTimeFormatter().print(dateTimeColumnValueMS); } return type.cast(dateTimeColumnValue); }
/** * <ul> * <li>Convert a time value in a format, to millis. * This method should not do validation of outputGranularity. * The validation should be handled by caller using {@link #isValidFormat(String)}</li> * <ul> * <li>1) given dateTimeColumnValue = 416359 and format=1:HOURS:EPOCH * dateTimeSpec.toMillis(416359) = 1498892400000 (i.e. timeColumnValue*60*60*1000)</li> * <li>2) given dateTimeColumnValue = 4996308 and format=5:MINUTES:EPOCH * dateTimeSpec.toMillis(4996308) = 1498892400000 (i.e. timeColumnValue*5*60*1000)</li> * <li>3) given dateTimeColumnValue = 20170701 and format=1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd * dateTimeSpec.toMillis(20170701) = 1498892400000</li> * </ul> * <ul> * @param dateTimeColumnValue - datetime Column value to convert to millis * @param fromFormat - the format in which the date time column value is expressed * @return datetime value in millis */ public Long fromFormatToMillis(Object dateTimeColumnValue) { Preconditions.checkNotNull(dateTimeColumnValue); Long timeColumnValueMS = 0L; if (_patternSpec.getTimeFormat().equals(TimeFormat.EPOCH)) { timeColumnValueMS = TimeUnit.MILLISECONDS.convert((Long) dateTimeColumnValue * _size, _unitSpec.getTimeUnit()); } else { timeColumnValueMS = _patternSpec.getDateTimeFormatter().parseMillis(String.valueOf(dateTimeColumnValue)); } return timeColumnValueMS; }
@Test public void testAppendWithPrefix() throws Exception { ColumnStatistics columnStatisticsClass = Mockito.mock(ColumnStatistics.class); when(columnStatisticsClass.getMaxValue()).thenReturn(3L); when(columnStatisticsClass.getMinValue()).thenReturn(1L); NormalizedDateSegmentNameGenerator normalizedDataSegmentNameGenerator = new NormalizedDateSegmentNameGenerator(TABLE_NAME, SEQUENCE_ID, TIME_COLUMN_TYPE, TABLE_PUSH_FREQUENCY, APPEND_PUSH_TYPE, PREFIX, null, DateTimeFieldSpec.TimeFormat.EPOCH.toString()); Assert.assertEquals(normalizedDataSegmentNameGenerator.generateSegmentName(columnStatisticsClass), "myTable_daily_1970-01-02_1970-01-04_1"); }
@Test public void testAppend() throws Exception { ColumnStatistics columnStatisticsClass = Mockito.mock(ColumnStatistics.class); when(columnStatisticsClass.getMaxValue()).thenReturn(3L); when(columnStatisticsClass.getMinValue()).thenReturn(1L); NormalizedDateSegmentNameGenerator normalizedDataSegmentNameGenerator = new NormalizedDateSegmentNameGenerator(TABLE_NAME, SEQUENCE_ID, TIME_COLUMN_TYPE, TABLE_PUSH_FREQUENCY, APPEND_PUSH_TYPE, null, null, DateTimeFieldSpec.TimeFormat.EPOCH.toString()); Assert.assertEquals(normalizedDataSegmentNameGenerator.generateSegmentName(columnStatisticsClass), "myTable_1970-01-02_1970-01-04_1"); }
@Test public void testAppendWithPrefixNoSequenceId() throws Exception { ColumnStatistics columnStatisticsClass = Mockito.mock(ColumnStatistics.class); when(columnStatisticsClass.getMaxValue()).thenReturn(3L); when(columnStatisticsClass.getMinValue()).thenReturn(1L); NormalizedDateSegmentNameGenerator normalizedDataSegmentNameGenerator = new NormalizedDateSegmentNameGenerator(TABLE_NAME, SEQUENCE_ID, TIME_COLUMN_TYPE, TABLE_PUSH_FREQUENCY, APPEND_PUSH_TYPE, "prefix", "true", DateTimeFieldSpec.TimeFormat.EPOCH.toString()); Assert.assertEquals(normalizedDataSegmentNameGenerator.generateSegmentName(columnStatisticsClass), "prefix_1970-01-02_1970-01-04"); }
@Test public void testUntrimmedPrefix() throws Exception { ColumnStatistics columnStatisticsClass = Mockito.mock(ColumnStatistics.class); when(columnStatisticsClass.getMaxValue()).thenReturn(3L); when(columnStatisticsClass.getMinValue()).thenReturn(1L); NormalizedDateSegmentNameGenerator normalizedDataSegmentNameGenerator = new NormalizedDateSegmentNameGenerator(TABLE_NAME, SEQUENCE_ID, TIME_COLUMN_TYPE, TABLE_PUSH_FREQUENCY, APPEND_PUSH_TYPE, "mirrorShareEvents_daily ", null, DateTimeFieldSpec.TimeFormat.EPOCH.toString()); Assert.assertEquals(normalizedDataSegmentNameGenerator.generateSegmentName(columnStatisticsClass), "mirrorShareEvents_daily_1970-01-02_1970-01-04_1"); }
@Test public void testMirrorShare() throws Exception { ColumnStatistics columnStatisticsClass = Mockito.mock(ColumnStatistics.class); when(columnStatisticsClass.getMaxValue()).thenReturn(3L); when(columnStatisticsClass.getMinValue()).thenReturn(1L); NormalizedDateSegmentNameGenerator normalizedDataSegmentNameGenerator = new NormalizedDateSegmentNameGenerator(TABLE_NAME, SEQUENCE_ID, TIME_COLUMN_TYPE, TABLE_PUSH_FREQUENCY, APPEND_PUSH_TYPE, "mirrorShareEvents_daily", null, DateTimeFieldSpec.TimeFormat.EPOCH.toString()); Assert.assertEquals(normalizedDataSegmentNameGenerator.generateSegmentName(columnStatisticsClass), "mirrorShareEvents_daily_1970-01-02_1970-01-04_1"); }