public AvroRecordToPinotRowGenerator(@Nonnull Schema schema) { _schema = schema; // For time field, we use the incoming time field spec TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); Preconditions.checkNotNull(timeFieldSpec); _incomingTimeFieldSpec = new TimeFieldSpec(timeFieldSpec.getIncomingGranularitySpec()); }
public TimeUnit getSegmentTimeUnit() { if (_segmentTimeUnit != null) { return _segmentTimeUnit; } else { TimeFieldSpec timeFieldSpec = _schema.getTimeFieldSpec(); if (timeFieldSpec != null) { // Outgoing granularity is always non-null. return timeFieldSpec.getOutgoingGranularitySpec().getTimeType(); } return TimeUnit.DAYS; } }
TimeFieldSpec timeFieldSpec = tableSchema.getTimeFieldSpec(); if (timeFieldSpec == null || timeFieldSpec.getDataType() == FieldSpec.DataType.STRING) { LOGGER.info("Table: {}, skip adding inverted index because it does not have a numeric time column",
TimeFieldSpec spec = pinotSchema.getTimeFieldSpec(); String timeColumn = spec.getIncomingTimeColumnName(); message.put(timeColumn, currentTimeValue);
/** * Returns a new schema based on the original one. The new schema removes columns as needed (for ex, virtual cols) * and adds the new timespec to the schema. */ @VisibleForTesting public Schema getUpdatedSchema(Schema original) { TimeFieldSpec tfs = original.getTimeFieldSpec(); // Use outgoing granularity for creating segment TimeGranularitySpec outgoing = tfs.getOutgoingGranularitySpec(); TimeFieldSpec newTimeSpec = new TimeFieldSpec(outgoing); Schema newSchema = new Schema(); newSchema.addField(newTimeSpec); for (String col : original.getPhysicalColumnNames()) { if (!col.equals(tfs.getName())) { newSchema.addField(original.getFieldSpecFor(col)); } } return newSchema; } }
/** * @deprecated Load outside the class and use the setter for schema setting. * @throws IOException */ @Deprecated public void loadConfigFiles() throws IOException { Schema schema; if (_schemaFile != null) { schema = Schema.fromFile(new File(_schemaFile)); setSchema(schema); } else if (_format == FileFormat.AVRO) { schema = AvroUtils.getPinotSchemaFromAvroDataFile(new File(_inputFilePath)); setSchema(schema); } else { throw new RuntimeException("Input format " + _format + " requires schema."); } setTimeColumnName(schema.getTimeColumnName()); TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); if (timeFieldSpec != null) { setSegmentTimeUnit(timeFieldSpec.getIncomingGranularitySpec().getTimeType()); } else { setSegmentTimeUnit(TimeUnit.DAYS); } if (_readerConfigFile != null) { setReaderConfig(JsonUtils.fileToObject(new File(_readerConfigFile), CSVRecordReaderConfig.class)); } }
public TimeTransformer(Schema schema) { TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); if (timeFieldSpec != null) { TimeGranularitySpec incomingGranularitySpec = timeFieldSpec.getIncomingGranularitySpec(); TimeGranularitySpec outgoingGranularitySpec = timeFieldSpec.getOutgoingGranularitySpec(); // Perform time conversion only if incoming and outgoing granularity spec are different if (!incomingGranularitySpec.equals(outgoingGranularitySpec)) { _incomingTimeColumn = incomingGranularitySpec.getName(); _outgoingTimeColumn = outgoingGranularitySpec.getName(); _incomingTimeConverter = new TimeConverter(incomingGranularitySpec); _outgoingTimeConverter = new TimeConverter(outgoingGranularitySpec); } } }
long segEndTime = realtimeSegment.getMaxTime(); TimeUnit timeUnit = schema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeType(); ImmutableSegment segment = ImmutableSegmentLoader.load(new File(resourceDir, segmentMetatdaZk.getSegmentName()), indexLoadingConfig);
@Test public void testNoVirtualColumnsInSchema() { Schema schema = new Schema(); FieldSpec spec = new DimensionFieldSpec("col1", FieldSpec.DataType.STRING, true); schema.addField(spec); TimeFieldSpec tfs = new TimeFieldSpec("col1", FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "col2", FieldSpec.DataType.LONG, TimeUnit.DAYS); schema.addField(tfs); VirtualColumnProviderFactory.addBuiltInVirtualColumnsToSchema(schema); Assert.assertEquals(schema.getColumnNames().size(), 5); Assert.assertEquals(schema.getTimeFieldSpec().getIncomingGranularitySpec().getTimeType(), TimeUnit.MILLISECONDS); RealtimeSegmentConverter converter = new RealtimeSegmentConverter(null, "", schema, "testTable", "col1", "segment1", "col1"); Schema newSchema = converter.getUpdatedSchema(schema); Assert.assertEquals(newSchema.getColumnNames().size(), 2); Assert.assertEquals(newSchema.getTimeFieldSpec().getIncomingGranularitySpec().getTimeType(), TimeUnit.DAYS); } }
@Test(dataProvider = "backfillRecordReaderDataProvider") public void testBackfillDateTimeRecordReader(RecordReader baseRecordReader, TimeFieldSpec timeFieldSpec, DateTimeFieldSpec dateTimeFieldSpec, Schema schemaExpected) throws Exception { BackfillDateTimeColumn backfillDateTimeColumn = new BackfillDateTimeColumn(new File("original"), new File("backup"), timeFieldSpec, dateTimeFieldSpec); try (BackfillDateTimeRecordReader wrapperReader = backfillDateTimeColumn .getBackfillDateTimeRecordReader(baseRecordReader)) { // check that schema has new column Schema schemaActual = wrapperReader.getSchema(); Assert.assertEquals(schemaActual, schemaExpected); DateTimeFieldSpec dateTimeFieldSpecActual = schemaActual.getDateTimeSpec(dateTimeFieldSpec.getName()); TimeFieldSpec timeFieldSpecActual = schemaActual.getTimeFieldSpec(); Assert.assertEquals(dateTimeFieldSpecActual, dateTimeFieldSpec); Assert.assertEquals(timeFieldSpecActual, timeFieldSpec); while (wrapperReader.hasNext()) { GenericRow next = wrapperReader.next(); // check that new datetime column is generated Object dateTimeColumnValueActual = next.getValue(dateTimeFieldSpec.getName()); Assert.assertNotNull(dateTimeColumnValueActual); Object timeColumnValueActual = next.getValue(timeFieldSpec.getName()); Assert.assertNotNull(timeColumnValueActual); // check that datetime column has correct value as per its format Long timeColumnValueMS = timeFieldSpec.getIncomingGranularitySpec().toMillis(timeColumnValueActual); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(dateTimeFieldSpec.getFormat()); Object dateTimeColumnValueExpected = toFormat.fromMillisToFormat(timeColumnValueMS, Object.class); Assert.assertEquals(dateTimeColumnValueActual, dateTimeColumnValueExpected); } } }
.addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec, defaultNullValue).build(); Assert.assertNotNull(schema1.getTimeFieldSpec()); Assert.assertNotNull(schema2.getTimeFieldSpec()); Assert.assertNotNull(schema3.getTimeFieldSpec()); Assert.assertNotNull(schema4.getTimeFieldSpec()); Assert.assertNotNull(schema5.getTimeFieldSpec()); Assert.assertNotNull(schema6.getTimeFieldSpec()); Assert.assertNotNull(schema7.getTimeFieldSpec()); Assert.assertNotNull(schema8.getTimeFieldSpec()); Assert.assertNotNull(schema9.getTimeFieldSpec()); Assert.assertNotNull(schema10.getTimeFieldSpec()); Assert.assertNotNull(schema11.getTimeFieldSpec()); Assert.assertNotNull(schema12.getTimeFieldSpec()); schema1.getTimeFieldSpec().setDefaultNullValue(defaultNullValue); schema3.getTimeFieldSpec().setDefaultNullValue(defaultNullValue); schema5.getTimeFieldSpec().setDefaultNullValue(defaultNullValue); schema7.getTimeFieldSpec().setDefaultNullValue(defaultNullValue); schema9.getTimeFieldSpec().setDefaultNullValue(defaultNullValue); schema11.getTimeFieldSpec().setDefaultNullValue(defaultNullValue); Assert.assertEquals(schema1, schema2); Assert.assertEquals(schema3, schema4);
private String getDefaultSegmentName(TableConfig tableConfig, Schema schema, List<File> inputIndexDirs, long minStartTime, long maxEndTime) throws Exception { String tableName = tableConfig.getTableName(); // Fetch time related configurations from schema and table config. String pushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency(); String timeColumnType = tableConfig.getValidationConfig().getTimeType(); String pushType = tableConfig.getValidationConfig().getSegmentPushType(); String timeFormat = schema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeFormat(); // Generate the final segment name using segment name generator NormalizedDateSegmentNameGenerator segmentNameGenerator = new NormalizedDateSegmentNameGenerator(tableName, DEFAULT_SEQUENCE_ID, timeColumnType, pushFrequency, pushType, null, null, timeFormat); return segmentNameGenerator.generateSegmentName(minStartTime, maxEndTime); }
genConfig.setTimeColumnName(dataSchema.getTimeFieldSpec().getOutgoingTimeColumnName()); genConfig.setSegmentTimeUnit(dataSchema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeType()); if (segmentVersion != null) { genConfig.setSegmentVersion(segmentVersion);
Assert.assertNotNull(metricFieldSpec.getDerivedMetricType()); TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); Assert.assertNotNull(timeFieldSpec); Assert.assertEquals(timeFieldSpec.getFieldType(), FieldSpec.FieldType.TIME);