@JsonIgnore public TimeUnit getOutgoingTimeUnit() { return (_timeFieldSpec != null) ? _timeFieldSpec.getOutgoingGranularitySpec().getTimeType() : null; }
public long toMillisSinceEpoch(Object value) { long duration; if (value instanceof Number) { duration = ((Number) value).longValue(); } else { duration = Long.parseLong(value.toString()); } return _timeGranularitySpec.getTimeType().toMillis(duration * _timeGranularitySpec.getTimeUnitSize()); }
@JsonIgnore public TimeUnit getIncomingTimeUnit() { return (_timeFieldSpec != null) ? _timeFieldSpec.getIncomingGranularitySpec().getTimeType() : null; }
public TimeUnit getSegmentTimeUnit() { if (_segmentTimeUnit != null) { return _segmentTimeUnit; } else { TimeFieldSpec timeFieldSpec = _schema.getTimeFieldSpec(); if (timeFieldSpec != null) { // Outgoing granularity is always non-null. return timeFieldSpec.getOutgoingGranularitySpec().getTimeType(); } return TimeUnit.DAYS; } }
public Object fromMillisSinceEpoch(long value) { long duration = _timeGranularitySpec.getTimeType().convert(value, TimeUnit.MILLISECONDS) / _timeGranularitySpec .getTimeUnitSize(); switch (_timeGranularitySpec.getDataType()) { case INT: return (int) duration; case LONG: return duration; case FLOAT: return (float) duration; case DOUBLE: return (double) duration; case STRING: return Long.toString(duration); default: throw new IllegalStateException(); } } }
timeUnits.put(col, tfs.getIncomingGranularitySpec().getTimeType()); break;
TimeUnit timeUnit = timeFieldSpec.getOutgoingGranularitySpec().getTimeType(); if (timeUnit != TimeUnit.DAYS) { LOGGER.warn("Table: {}, time column {] has non-DAYS time unit: {}", timeColumnName, timeUnit);
/** * @deprecated Load outside the class and use the setter for schema setting. * @throws IOException */ @Deprecated public void loadConfigFiles() throws IOException { Schema schema; if (_schemaFile != null) { schema = Schema.fromFile(new File(_schemaFile)); setSchema(schema); } else if (_format == FileFormat.AVRO) { schema = AvroUtils.getPinotSchemaFromAvroDataFile(new File(_inputFilePath)); setSchema(schema); } else { throw new RuntimeException("Input format " + _format + " requires schema."); } setTimeColumnName(schema.getTimeColumnName()); TimeFieldSpec timeFieldSpec = schema.getTimeFieldSpec(); if (timeFieldSpec != null) { setSegmentTimeUnit(timeFieldSpec.getIncomingGranularitySpec().getTimeType()); } else { setSegmentTimeUnit(TimeUnit.DAYS); } if (_readerConfigFile != null) { setReaderConfig(JsonUtils.fileToObject(new File(_readerConfigFile), CSVRecordReaderConfig.class)); } }
/** * Converts the time column value from timeFieldSpec to dateTimeFieldSpec * @param timeColumnValue - time column value from timeFieldSpec * @return */ private Object convertTimeFieldToDateTimeFieldSpec(Object timeColumnValue) { TimeGranularitySpec timeGranularitySpec = _timeFieldSpec.getOutgoingGranularitySpec(); DateTimeFormatSpec formatFromTimeSpec = new DateTimeFormatSpec(timeGranularitySpec.getTimeUnitSize(), timeGranularitySpec.getTimeType().toString(), timeGranularitySpec.getTimeFormat()); if (formatFromTimeSpec.getFormat().equals(_dateTimeFieldSpec.getFormat())) { return timeColumnValue; } long timeColumnValueMS = timeGranularitySpec.toMillis(timeColumnValue); DateTimeFormatSpec toFormat = new DateTimeFormatSpec(_dateTimeFieldSpec.getFormat()); return toFormat.fromMillisToFormat(timeColumnValueMS, Object.class); }
long segEndTime = realtimeSegment.getMaxTime(); TimeUnit timeUnit = schema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeType(); ImmutableSegment segment = ImmutableSegmentLoader.load(new File(resourceDir, segmentMetatdaZk.getSegmentName()), indexLoadingConfig);
@Test public void testNoVirtualColumnsInSchema() { Schema schema = new Schema(); FieldSpec spec = new DimensionFieldSpec("col1", FieldSpec.DataType.STRING, true); schema.addField(spec); TimeFieldSpec tfs = new TimeFieldSpec("col1", FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "col2", FieldSpec.DataType.LONG, TimeUnit.DAYS); schema.addField(tfs); VirtualColumnProviderFactory.addBuiltInVirtualColumnsToSchema(schema); Assert.assertEquals(schema.getColumnNames().size(), 5); Assert.assertEquals(schema.getTimeFieldSpec().getIncomingGranularitySpec().getTimeType(), TimeUnit.MILLISECONDS); RealtimeSegmentConverter converter = new RealtimeSegmentConverter(null, "", schema, "testTable", "col1", "segment1", "col1"); Schema newSchema = converter.getUpdatedSchema(schema); Assert.assertEquals(newSchema.getColumnNames().size(), 2); Assert.assertEquals(newSchema.getTimeFieldSpec().getIncomingGranularitySpec().getTimeType(), TimeUnit.DAYS); } }
genConfig.setSegmentTimeUnit(dataSchema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeType()); if (segmentVersion != null) { genConfig.setSegmentVersion(segmentVersion);