public SchemaBuilder addTime(@Nonnull String incomingName, @Nonnull TimeUnit incomingTimeUnit, @Nonnull DataType incomingDataType) { _schema.addField(new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnit)); return this; }
/** * Returns a new schema based on the original one. The new schema removes columns as needed (for ex, virtual cols) * and adds the new timespec to the schema. */ @VisibleForTesting public Schema getUpdatedSchema(Schema original) { TimeFieldSpec tfs = original.getTimeFieldSpec(); // Use outgoing granularity for creating segment TimeGranularitySpec outgoing = tfs.getOutgoingGranularitySpec(); TimeFieldSpec newTimeSpec = new TimeFieldSpec(outgoing); Schema newSchema = new Schema(); newSchema.addField(newTimeSpec); for (String col : original.getPhysicalColumnNames()) { if (!col.equals(tfs.getName())) { newSchema.addField(original.getFieldSpecFor(col)); } } return newSchema; } }
public AvroRecordToPinotRowGenerator(@Nonnull Schema schema) { _schema = schema; // For time field, we use the incoming time field spec TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); Preconditions.checkNotNull(timeFieldSpec); _incomingTimeFieldSpec = new TimeFieldSpec(timeFieldSpec.getIncomingGranularitySpec()); }
@Nonnull @Override public ObjectNode toJsonObject() { ObjectNode jsonObject = JsonUtils.newObjectNode(); jsonObject.set("incomingGranularitySpec", _incomingGranularitySpec.toJsonObject()); if (!getOutgoingGranularitySpec().equals(_incomingGranularitySpec)) { jsonObject.set("outgoingGranularitySpec", _outgoingGranularitySpec.toJsonObject()); } appendDefaultNullValue(jsonObject); return jsonObject; }
private List<GenericRow> createTestDataWithTimespec(TimeFieldSpec timeFieldSpec) { List<GenericRow> rows = new ArrayList<>(); Random random = new Random(); Map<String, Object> fields; for (int i = 0; i < NUM_ROWS; i++) { fields = new HashMap<>(); fields.put(D1, RandomStringUtils.randomAlphabetic(2)); fields.put(D2, RandomStringUtils.randomAlphabetic(5)); fields.put(M1, Math.abs(random.nextInt())); fields.put(M2, Math.abs(random.nextFloat())); long timestamp = System.currentTimeMillis(); Object timeColumnValue = timeFieldSpec.getIncomingGranularitySpec().fromMillis(timestamp); fields.put(timeFieldSpec.getName(), timeColumnValue); GenericRow row = new GenericRow(); row.init(fields); rows.add(row); } return rows; }
if (timeFieldSpec == null || timeFieldSpec.getDataType() == FieldSpec.DataType.STRING) { LOGGER.info("Table: {}, skip adding inverted index because it does not have a numeric time column", tableNameWithType); continue; String timeColumnName = timeFieldSpec.getName(); TimeUnit timeUnit = timeFieldSpec.getOutgoingGranularitySpec().getTimeType(); if (timeUnit != TimeUnit.DAYS) { LOGGER.warn("Table: {}, time column {] has non-DAYS time unit: {}", timeColumnName, timeUnit);
@Override public String toString() { return "< field type: TIME, incoming granularity spec: " + _incomingGranularitySpec + ", outgoing granularity spec: " + getOutgoingGranularitySpec() + ", default null value: " + _defaultNullValue + " >"; }
Assert.assertEquals(timeFieldSpec.getFieldType(), FieldSpec.FieldType.TIME); Assert.assertEquals(timeFieldSpec.getName(), "time"); Assert.assertEquals(timeFieldSpec.getDataType(), FieldSpec.DataType.LONG); Assert.assertEquals(timeFieldSpec.isSingleValueField(), true); Assert.assertEquals(timeFieldSpec.getDefaultNullValue(), Long.MIN_VALUE);
public TimeTransformer(Schema schema) { TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); if (timeFieldSpec != null) { TimeGranularitySpec incomingGranularitySpec = timeFieldSpec.getIncomingGranularitySpec(); TimeGranularitySpec outgoingGranularitySpec = timeFieldSpec.getOutgoingGranularitySpec(); // Perform time conversion only if incoming and outgoing granularity spec are different if (!incomingGranularitySpec.equals(outgoingGranularitySpec)) { _incomingTimeColumn = incomingGranularitySpec.getName(); _outgoingTimeColumn = outgoingGranularitySpec.getName(); _incomingTimeConverter = new TimeConverter(incomingGranularitySpec); _outgoingTimeConverter = new TimeConverter(outgoingGranularitySpec); } } }
@JsonIgnore public String getTimeColumnName() { return (_timeFieldSpec != null) ? _timeFieldSpec.getName() : null; }
TimeFieldSpec timeFieldSpec1 = new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnit); TimeFieldSpec timeFieldSpec2 = new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnit, defaultNullValue); TimeFieldSpec timeFieldSpec3 = new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnit, outgoingName, outgoingDataType, outgoingTimeUnit); TimeFieldSpec timeFieldSpec4 = new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnit, outgoingName, outgoingDataType, outgoingTimeUnit, defaultNullValue); TimeFieldSpec timeFieldSpec5 = new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnitSize, incomingTimeUnit); TimeFieldSpec timeFieldSpec6 = new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnitSize, incomingTimeUnit, defaultNullValue); TimeFieldSpec timeFieldSpec7 = new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnitSize, incomingTimeUnit, outgoingName, outgoingDataType, outgoingTimeUnitSize, outgoingTimeUnit); TimeFieldSpec timeFieldSpec8 = new TimeFieldSpec(incomingName, incomingDataType, incomingTimeUnitSize, incomingTimeUnit, outgoingName, outgoingDataType, outgoingTimeUnitSize, outgoingTimeUnit, defaultNullValue); TimeFieldSpec timeFieldSpec9 = new TimeFieldSpec(incomingTimeGranularitySpec); TimeFieldSpec timeFieldSpec10 = new TimeFieldSpec(incomingTimeGranularitySpec, defaultNullValue); TimeFieldSpec timeFieldSpec11 = new TimeFieldSpec(incomingTimeGranularitySpec, outgoingTimeGranularitySpec); TimeFieldSpec timeFieldSpec12 = new TimeFieldSpec(incomingTimeGranularitySpec, outgoingTimeGranularitySpec, defaultNullValue); Assert.assertFalse(timeFieldSpec1.equals(timeFieldSpec2)); Assert.assertFalse(timeFieldSpec3.equals(timeFieldSpec4)); Assert.assertFalse(timeFieldSpec5.equals(timeFieldSpec6)); Assert.assertFalse(timeFieldSpec7.equals(timeFieldSpec8));
@JsonIgnore public TimeUnit getIncomingTimeUnit() { return (_timeFieldSpec != null) ? _timeFieldSpec.getIncomingGranularitySpec().getTimeType() : null; }
TimeFieldSpec timeFieldSpec1 = JsonUtils.stringToObject(getRandomOrderJsonString(timeFields), TimeFieldSpec.class); TimeFieldSpec timeFieldSpec2 = new TimeFieldSpec("incomingTime", LONG, TimeUnit.MILLISECONDS, "outgoingTime", INT, TimeUnit.SECONDS, -1); Assert.assertEquals(timeFieldSpec1, timeFieldSpec2, ERROR_MESSAGE); Assert.assertEquals(timeFieldSpec1.getDefaultNullValue(), -1, ERROR_MESSAGE);
genConfig.setTimeColumnName(dataSchema.getTimeFieldSpec().getOutgoingTimeColumnName()); genConfig.setSegmentTimeUnit(dataSchema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeType()); if (segmentVersion != null) { genConfig.setSegmentVersion(segmentVersion);
private List<GenericRow> createTestDataWithTimespec(TimeFieldSpec timeFieldSpec, DateTimeFieldSpec dateTimeFieldSpec) { List<GenericRow> rows = new ArrayList<>(); Random random = new Random(); Map<String, Object> fields; for (int i = 0; i < NUM_ROWS; i++) { fields = new HashMap<>(); fields.put(D1, RandomStringUtils.randomAlphabetic(2)); fields.put(D2, RandomStringUtils.randomAlphabetic(5)); fields.put(M1, Math.abs(random.nextInt())); fields.put(M2, Math.abs(random.nextFloat())); long timestamp = System.currentTimeMillis(); Object timeColumnValue = timeFieldSpec.getIncomingGranularitySpec().fromMillis(timestamp); fields.put(timeFieldSpec.getName(), timeColumnValue); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(dateTimeFieldSpec.getFormat()); Object dateTimeColumnValue = toFormat.fromMillisToFormat(timestamp, Object.class); fields.put(dateTimeFieldSpec.getName(), dateTimeColumnValue); GenericRow row = new GenericRow(); row.init(fields); rows.add(row); } return rows; }
@JsonIgnore public TimeUnit getOutgoingTimeUnit() { return (_timeFieldSpec != null) ? _timeFieldSpec.getOutgoingGranularitySpec().getTimeType() : null; }
@JsonIgnore @Nonnull public String getOutgoingTimeColumnName() { return getName(); }
timeUnits.put(col, tfs.getIncomingGranularitySpec().getTimeType()); break;
public SchemaBuilder addTime(@Nonnull TimeGranularitySpec incomingTimeGranularitySpec, @Nonnull Object defaultNullValue) { _schema.addField(new TimeFieldSpec(incomingTimeGranularitySpec, defaultNullValue)); return this; }
@Test(dataProvider = "backfillRecordReaderDataProvider") public void testBackfillDateTimeRecordReader(RecordReader baseRecordReader, TimeFieldSpec timeFieldSpec, DateTimeFieldSpec dateTimeFieldSpec, Schema schemaExpected) throws Exception { BackfillDateTimeColumn backfillDateTimeColumn = new BackfillDateTimeColumn(new File("original"), new File("backup"), timeFieldSpec, dateTimeFieldSpec); try (BackfillDateTimeRecordReader wrapperReader = backfillDateTimeColumn .getBackfillDateTimeRecordReader(baseRecordReader)) { // check that schema has new column Schema schemaActual = wrapperReader.getSchema(); Assert.assertEquals(schemaActual, schemaExpected); DateTimeFieldSpec dateTimeFieldSpecActual = schemaActual.getDateTimeSpec(dateTimeFieldSpec.getName()); TimeFieldSpec timeFieldSpecActual = schemaActual.getTimeFieldSpec(); Assert.assertEquals(dateTimeFieldSpecActual, dateTimeFieldSpec); Assert.assertEquals(timeFieldSpecActual, timeFieldSpec); while (wrapperReader.hasNext()) { GenericRow next = wrapperReader.next(); // check that new datetime column is generated Object dateTimeColumnValueActual = next.getValue(dateTimeFieldSpec.getName()); Assert.assertNotNull(dateTimeColumnValueActual); Object timeColumnValueActual = next.getValue(timeFieldSpec.getName()); Assert.assertNotNull(timeColumnValueActual); // check that datetime column has correct value as per its format Long timeColumnValueMS = timeFieldSpec.getIncomingGranularitySpec().toMillis(timeColumnValueActual); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(dateTimeFieldSpec.getFormat()); Object dateTimeColumnValueExpected = toFormat.fromMillisToFormat(timeColumnValueMS, Object.class); Assert.assertEquals(dateTimeColumnValueActual, dateTimeColumnValueExpected); } } }