@Test(dataProvider = "testGetFromFormatDataProvider") public void testGetFromFormat(String format, int columnSizeFromFormatExpected, TimeUnit columnUnitFromFormatExpected, TimeFormat timeFormatFromFormatExpected, String sdfPatternFromFormatExpected, DateTimeZone dateTimeZoneFromFormatExpected) { DateTimeFormatSpec dateTimeFormatSpec = new DateTimeFormatSpec(format); int columnSizeFromFormat = dateTimeFormatSpec.getColumnSize(); Assert.assertEquals(columnSizeFromFormat, columnSizeFromFormatExpected); TimeUnit columnUnitFromFormat = dateTimeFormatSpec.getColumnUnit(); Assert.assertEquals(columnUnitFromFormat, columnUnitFromFormatExpected); TimeFormat timeFormatFromFormat = dateTimeFormatSpec.getTimeFormat(); Assert.assertEquals(timeFormatFromFormat, timeFormatFromFormatExpected); String sdfPatternFromFormat = null; DateTimeZone dateTimeZoneFromFormat = DateTimeZone.UTC; try { sdfPatternFromFormat = dateTimeFormatSpec.getSDFPattern(); dateTimeZoneFromFormat = dateTimeFormatSpec.getDateTimezone(); } catch (Exception e) { // No sdf pattern } Assert.assertEquals(sdfPatternFromFormat, sdfPatternFromFormatExpected); Assert.assertEquals(dateTimeZoneFromFormat, dateTimeZoneFromFormatExpected); }
@Test(dataProvider = "testFromMillisToFormatDataProvider") public void testFromMillisToFormat(String format, long timeColumnValueMS, Class<?> type, Object timeColumnValueExpected) { DateTimeFormatSpec dateTimeFormatSpec = new DateTimeFormatSpec(format); Object timeColumnValueActual = dateTimeFormatSpec.fromMillisToFormat(timeColumnValueMS, type); Assert.assertEquals(timeColumnValueActual, timeColumnValueExpected); }
@Test(dataProvider = "testFromFormatToMillisDataProvider") public void testFromFormatToMillis(String format, Object timeColumnValue, long millisExpected) { DateTimeFormatSpec dateTimeFormatSpec = new DateTimeFormatSpec(format); long millisActual = dateTimeFormatSpec.fromFormatToMillis(timeColumnValue); Assert.assertEquals(millisActual, millisExpected); }
public static BaseDateTimeTransformer getDateTimeTransformer(String inputFormatStr, String outputFormatStr, String outputGranularityStr) { DateTimeFormatSpec inputFormat = new DateTimeFormatSpec(inputFormatStr); DateTimeFormatSpec outputFormat = new DateTimeFormatSpec(outputFormatStr); DateTimeGranularitySpec outputGranularity = new DateTimeGranularitySpec(outputGranularityStr); TimeFormat inputTimeFormat = inputFormat.getTimeFormat(); TimeFormat outputTimeFormat = outputFormat.getTimeFormat(); if (inputTimeFormat == TimeFormat.EPOCH) { if (outputTimeFormat == TimeFormat.EPOCH) { return new EpochToEpochTransformer(inputFormat, outputFormat, outputGranularity); } else { return new EpochToSDFTransformer(inputFormat, outputFormat, outputGranularity); } } else { if (outputTimeFormat == TimeFormat.EPOCH) { return new SDFToEpochTransformer(inputFormat, outputFormat, outputGranularity); } else { return new SDFToSDFTransformer(inputFormat, outputFormat, outputGranularity); } } } }
/** * Converts the time column value from timeFieldSpec to dateTimeFieldSpec * @param timeColumnValue - time column value from timeFieldSpec * @return */ private Object convertTimeFieldToDateTimeFieldSpec(Object timeColumnValue) { TimeGranularitySpec timeGranularitySpec = _timeFieldSpec.getOutgoingGranularitySpec(); DateTimeFormatSpec formatFromTimeSpec = new DateTimeFormatSpec(timeGranularitySpec.getTimeUnitSize(), timeGranularitySpec.getTimeType().toString(), timeGranularitySpec.getTimeFormat()); if (formatFromTimeSpec.getFormat().equals(_dateTimeFieldSpec.getFormat())) { return timeColumnValue; } long timeColumnValueMS = timeGranularitySpec.toMillis(timeColumnValue); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(_dateTimeFieldSpec.getFormat()); return toFormat.fromMillisToFormat(timeColumnValueMS, Object.class); }
public BaseDateTimeTransformer(@Nonnull DateTimeFormatSpec inputFormat, @Nonnull DateTimeFormatSpec outputFormat, @Nonnull DateTimeGranularitySpec outputGranularity) { _inputTimeSize = inputFormat.getColumnSize(); _inputTimeUnit = inputFormat.getColumnUnit(); _inputDateTimeFormatter = inputFormat.getDateTimeFormatter(); _outputTimeSize = outputFormat.getColumnSize(); _outputTimeUnit = outputFormat.getColumnDateTimeTransformUnit(); _outputDateTimeFormatter = outputFormat.getDateTimeFormatter(); _outputGranularity = outputGranularity; _outputGranularityMillis = outputGranularity.granularityToMillis();
@Test(dataProvider = "testConstructFormatDataProvider") public void testConstructFormat(int columnSize, TimeUnit columnUnit, String columnTimeFormat, String pattern, DateTimeFormatSpec formatExpected1, DateTimeFormatSpec formatExpected2) { DateTimeFormatSpec formatActual1 = null; try { formatActual1 = new DateTimeFormatSpec(columnSize, columnUnit.toString(), columnTimeFormat); } catch (Exception e) { // invalid arguments } Assert.assertEquals(formatActual1, formatExpected1); DateTimeFormatSpec formatActual2 = null; try { formatActual2 = new DateTimeFormatSpec(columnSize, columnUnit.toString(), columnTimeFormat, pattern); } catch (Exception e) { // invalid arguments } Assert.assertEquals(formatActual2, formatExpected2); }
@DataProvider(name = "testConstructFormatDataProvider") public Object[][] provideTestConstructFormatData() { List<Object[]> entries = new ArrayList<>(); entries.add(new Object[]{1, TimeUnit.HOURS, "EPOCH", null, new DateTimeFormatSpec("1:HOURS:EPOCH"), null}); entries.add(new Object[]{1, TimeUnit.HOURS, "EPOCH", "yyyyMMdd", new DateTimeFormatSpec("1:HOURS:EPOCH"), null}); entries.add(new Object[]{5, TimeUnit.MINUTES, "EPOCH", null, new DateTimeFormatSpec("5:MINUTES:EPOCH"), null}); entries.add(new Object[]{0, TimeUnit.HOURS, "EPOCH", null, null, null}); entries.add(new Object[]{1, null, "EPOCH", null, null, null}); entries.add(new Object[]{1, TimeUnit.HOURS, null, null, null, null}); entries.add(new Object[]{1, TimeUnit.HOURS, "DUMMY", "yyyyMMdd", null, null}); entries.add(new Object[]{1, TimeUnit.HOURS, "SIMPLE_DATE_FORMAT", "yyyyMMdd", null, new DateTimeFormatSpec( "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd")}); entries.add( new Object[]{1, TimeUnit.HOURS, "SIMPLE_DATE_FORMAT", "yyyyMMdd tz(America/Los_Angeles)", null, new DateTimeFormatSpec( "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(America/Los_Angeles)")}); entries.add(new Object[]{1, TimeUnit.HOURS, "SIMPLE_DATE_FORMAT", null, null, null}); entries.add(new Object[]{-1, TimeUnit.HOURS, "SIMPLE_DATE_FORMAT", "yyyyMMDD", null, null}); entries.add( new Object[]{1, TimeUnit.HOURS, "SIMPLE_DATE_FORMAT", "M/d/yyyy h:mm:ss a", null, new DateTimeFormatSpec( "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a")}); return entries.toArray(new Object[entries.size()][]); } }
private List<GenericRow> createTestDataWithTimespec(TimeFieldSpec timeFieldSpec, DateTimeFieldSpec dateTimeFieldSpec) { List<GenericRow> rows = new ArrayList<>(); Random random = new Random(); Map<String, Object> fields; for (int i = 0; i < NUM_ROWS; i++) { fields = new HashMap<>(); fields.put(D1, RandomStringUtils.randomAlphabetic(2)); fields.put(D2, RandomStringUtils.randomAlphabetic(5)); fields.put(M1, Math.abs(random.nextInt())); fields.put(M2, Math.abs(random.nextFloat())); long timestamp = System.currentTimeMillis(); Object timeColumnValue = timeFieldSpec.getIncomingGranularitySpec().fromMillis(timestamp); fields.put(timeFieldSpec.getName(), timeColumnValue); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(dateTimeFieldSpec.getFormat()); Object dateTimeColumnValue = toFormat.fromMillisToFormat(timestamp, Object.class); fields.put(dateTimeFieldSpec.getName(), dateTimeColumnValue); GenericRow row = new GenericRow(); row.init(fields); rows.add(row); } return rows; }
@Test(dataProvider = "backfillRecordReaderDataProvider") public void testBackfillDateTimeRecordReader(RecordReader baseRecordReader, TimeFieldSpec timeFieldSpec, DateTimeFieldSpec dateTimeFieldSpec, Schema schemaExpected) throws Exception { BackfillDateTimeColumn backfillDateTimeColumn = new BackfillDateTimeColumn(new File("original"), new File("backup"), timeFieldSpec, dateTimeFieldSpec); try (BackfillDateTimeRecordReader wrapperReader = backfillDateTimeColumn .getBackfillDateTimeRecordReader(baseRecordReader)) { // check that schema has new column Schema schemaActual = wrapperReader.getSchema(); Assert.assertEquals(schemaActual, schemaExpected); DateTimeFieldSpec dateTimeFieldSpecActual = schemaActual.getDateTimeSpec(dateTimeFieldSpec.getName()); TimeFieldSpec timeFieldSpecActual = schemaActual.getTimeFieldSpec(); Assert.assertEquals(dateTimeFieldSpecActual, dateTimeFieldSpec); Assert.assertEquals(timeFieldSpecActual, timeFieldSpec); while (wrapperReader.hasNext()) { GenericRow next = wrapperReader.next(); // check that new datetime column is generated Object dateTimeColumnValueActual = next.getValue(dateTimeFieldSpec.getName()); Assert.assertNotNull(dateTimeColumnValueActual); Object timeColumnValueActual = next.getValue(timeFieldSpec.getName()); Assert.assertNotNull(timeColumnValueActual); // check that datetime column has correct value as per its format Long timeColumnValueMS = timeFieldSpec.getIncomingGranularitySpec().toMillis(timeColumnValueActual); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(dateTimeFieldSpec.getFormat()); Object dateTimeColumnValueExpected = toFormat.fromMillisToFormat(timeColumnValueMS, Object.class); Assert.assertEquals(dateTimeColumnValueActual, dateTimeColumnValueExpected); } } }