@JsonIgnore public TimeUnit getOutgoingTimeUnit() { return (_timeFieldSpec != null) ? _timeFieldSpec.getOutgoingGranularitySpec().getTimeType() : null; }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, @Nonnull TimeUnit incomingTimeUnit, @Nonnull String outgoingName, @Nonnull DataType outgoingDataType, @Nonnull TimeUnit outgoingTimeUnit) { super(outgoingName, outgoingDataType, true); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnit, incomingName); _outgoingGranularitySpec = new TimeGranularitySpec(outgoingDataType, outgoingTimeUnit, outgoingName); }
public TimeConverter(TimeGranularitySpec timeGranularitySpec) { Preconditions .checkArgument(timeGranularitySpec.getTimeFormat().equals(TimeGranularitySpec.TimeFormat.EPOCH.toString()), "Cannot perform time conversion for time format other than EPOCH"); _timeGranularitySpec = timeGranularitySpec; }
public TimeFieldSpec(@Nonnull TimeGranularitySpec incomingGranularitySpec) { super(incomingGranularitySpec.getName(), incomingGranularitySpec.getDataType(), true); _incomingGranularitySpec = incomingGranularitySpec; }
public long toMillisSinceEpoch(Object value) { long duration; if (value instanceof Number) { duration = ((Number) value).longValue(); } else { duration = Long.parseLong(value.toString()); } return _timeGranularitySpec.getTimeType().toMillis(duration * _timeGranularitySpec.getTimeUnitSize()); }
public Object fromMillisSinceEpoch(long value) { long duration = _timeGranularitySpec.getTimeType().convert(value, TimeUnit.MILLISECONDS) / _timeGranularitySpec .getTimeUnitSize(); switch (_timeGranularitySpec.getDataType()) { case INT: return (int) duration; case LONG: return duration; case FLOAT: return (float) duration; case DOUBLE: return (double) duration; case STRING: return Long.toString(duration); default: throw new IllegalStateException(); } } }
/** * Converts the time column value from timeFieldSpec to dateTimeFieldSpec * @param timeColumnValue - time column value from timeFieldSpec * @return */ private Object convertTimeFieldToDateTimeFieldSpec(Object timeColumnValue) { TimeGranularitySpec timeGranularitySpec = _timeFieldSpec.getOutgoingGranularitySpec(); DateTimeFormatSpec formatFromTimeSpec = new DateTimeFormatSpec(timeGranularitySpec.getTimeUnitSize(), timeGranularitySpec.getTimeType().toString(), timeGranularitySpec.getTimeFormat()); if (formatFromTimeSpec.getFormat().equals(_dateTimeFieldSpec.getFormat())) { return timeColumnValue; } long timeColumnValueMS = timeGranularitySpec.toMillis(timeColumnValue); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(_dateTimeFieldSpec.getFormat()); return toFormat.fromMillisToFormat(timeColumnValueMS, Object.class); }
public TimeTransformer(Schema schema) { TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); if (timeFieldSpec != null) { TimeGranularitySpec incomingGranularitySpec = timeFieldSpec.getIncomingGranularitySpec(); TimeGranularitySpec outgoingGranularitySpec = timeFieldSpec.getOutgoingGranularitySpec(); // Perform time conversion only if incoming and outgoing granularity spec are different if (!incomingGranularitySpec.equals(outgoingGranularitySpec)) { _incomingTimeColumn = incomingGranularitySpec.getName(); _outgoingTimeColumn = outgoingGranularitySpec.getName(); _incomingTimeConverter = new TimeConverter(incomingGranularitySpec); _outgoingTimeConverter = new TimeConverter(outgoingGranularitySpec); } } }
@Nonnull @Override public ObjectNode toJsonObject() { ObjectNode jsonObject = JsonUtils.newObjectNode(); jsonObject.set("incomingGranularitySpec", _incomingGranularitySpec.toJsonObject()); if (!getOutgoingGranularitySpec().equals(_incomingGranularitySpec)) { jsonObject.set("outgoingGranularitySpec", _outgoingGranularitySpec.toJsonObject()); } appendDefaultNullValue(jsonObject); return jsonObject; }
@JsonIgnore @Nonnull public String getIncomingTimeColumnName() { return _incomingGranularitySpec.getName(); }
@Test(dataProvider = "backfillRecordReaderDataProvider") public void testBackfillDateTimeRecordReader(RecordReader baseRecordReader, TimeFieldSpec timeFieldSpec, DateTimeFieldSpec dateTimeFieldSpec, Schema schemaExpected) throws Exception { BackfillDateTimeColumn backfillDateTimeColumn = new BackfillDateTimeColumn(new File("original"), new File("backup"), timeFieldSpec, dateTimeFieldSpec); try (BackfillDateTimeRecordReader wrapperReader = backfillDateTimeColumn .getBackfillDateTimeRecordReader(baseRecordReader)) { // check that schema has new column Schema schemaActual = wrapperReader.getSchema(); Assert.assertEquals(schemaActual, schemaExpected); DateTimeFieldSpec dateTimeFieldSpecActual = schemaActual.getDateTimeSpec(dateTimeFieldSpec.getName()); TimeFieldSpec timeFieldSpecActual = schemaActual.getTimeFieldSpec(); Assert.assertEquals(dateTimeFieldSpecActual, dateTimeFieldSpec); Assert.assertEquals(timeFieldSpecActual, timeFieldSpec); while (wrapperReader.hasNext()) { GenericRow next = wrapperReader.next(); // check that new datetime column is generated Object dateTimeColumnValueActual = next.getValue(dateTimeFieldSpec.getName()); Assert.assertNotNull(dateTimeColumnValueActual); Object timeColumnValueActual = next.getValue(timeFieldSpec.getName()); Assert.assertNotNull(timeColumnValueActual); // check that datetime column has correct value as per its format Long timeColumnValueMS = timeFieldSpec.getIncomingGranularitySpec().toMillis(timeColumnValueActual); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(dateTimeFieldSpec.getFormat()); Object dateTimeColumnValueExpected = toFormat.fromMillisToFormat(timeColumnValueMS, Object.class); Assert.assertEquals(dateTimeColumnValueActual, dateTimeColumnValueExpected); } } }
public TimeFieldSpec(@Nonnull TimeGranularitySpec incomingGranularitySpec, @Nonnull TimeGranularitySpec outgoingGranularitySpec) { super(outgoingGranularitySpec.getName(), outgoingGranularitySpec.getDataType(), true); _incomingGranularitySpec = incomingGranularitySpec; _outgoingGranularitySpec = outgoingGranularitySpec; }
@JsonIgnore public TimeUnit getIncomingTimeUnit() { return (_timeFieldSpec != null) ? _timeFieldSpec.getIncomingGranularitySpec().getTimeType() : null; }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, int incomingTimeUnitSize, @Nonnull TimeUnit incomingTimeUnit, @Nonnull String outgoingName, @Nonnull DataType outgoingDataType, int outgoingTimeUnitSize, @Nonnull TimeUnit outgoingTimeUnit, @Nonnull Object defaultNullValue) { super(outgoingName, outgoingDataType, true, defaultNullValue); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnitSize, incomingTimeUnit, incomingName); _outgoingGranularitySpec = new TimeGranularitySpec(outgoingDataType, outgoingTimeUnitSize, outgoingTimeUnit, outgoingName); }
public TimeFieldSpec(@Nonnull TimeGranularitySpec incomingGranularitySpec, @Nonnull Object defaultNullValue) { super(incomingGranularitySpec.getName(), incomingGranularitySpec.getDataType(), true, defaultNullValue); _incomingGranularitySpec = incomingGranularitySpec; }
private String getDefaultSegmentName(TableConfig tableConfig, Schema schema, List<File> inputIndexDirs, long minStartTime, long maxEndTime) throws Exception { String tableName = tableConfig.getTableName(); // Fetch time related configurations from schema and table config. String pushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency(); String timeColumnType = tableConfig.getValidationConfig().getTimeType(); String pushType = tableConfig.getValidationConfig().getSegmentPushType(); String timeFormat = schema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeFormat(); // Generate the final segment name using segment name generator NormalizedDateSegmentNameGenerator segmentNameGenerator = new NormalizedDateSegmentNameGenerator(tableName, DEFAULT_SEQUENCE_ID, timeColumnType, pushFrequency, pushType, null, null, timeFormat); return segmentNameGenerator.generateSegmentName(minStartTime, maxEndTime); }
public TimeUnit getSegmentTimeUnit() { if (_segmentTimeUnit != null) { return _segmentTimeUnit; } else { TimeFieldSpec timeFieldSpec = _schema.getTimeFieldSpec(); if (timeFieldSpec != null) { // Outgoing granularity is always non-null. return timeFieldSpec.getOutgoingGranularitySpec().getTimeType(); } return TimeUnit.DAYS; } }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, @Nonnull TimeUnit incomingTimeUnit) { super(incomingName, incomingDataType, true); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnit, incomingName); }
public TimeFieldSpec(@Nonnull TimeGranularitySpec incomingGranularitySpec, @Nonnull TimeGranularitySpec outgoingGranularitySpec, @Nonnull Object defaultNullValue) { super(outgoingGranularitySpec.getName(), outgoingGranularitySpec.getDataType(), true, defaultNullValue); _incomingGranularitySpec = incomingGranularitySpec; _outgoingGranularitySpec = outgoingGranularitySpec; }
timeUnits.put(col, tfs.getIncomingGranularitySpec().getTimeType()); break;