public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, int incomingTimeUnitSize, @Nonnull TimeUnit incomingTimeUnit, @Nonnull String outgoingName, @Nonnull DataType outgoingDataType, int outgoingTimeUnitSize, @Nonnull TimeUnit outgoingTimeUnit, @Nonnull Object defaultNullValue) { super(outgoingName, outgoingDataType, true, defaultNullValue); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnitSize, incomingTimeUnit, incomingName); _outgoingGranularitySpec = new TimeGranularitySpec(outgoingDataType, outgoingTimeUnitSize, outgoingTimeUnit, outgoingName); }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, @Nonnull TimeUnit incomingTimeUnit, @Nonnull String outgoingName, @Nonnull DataType outgoingDataType, @Nonnull TimeUnit outgoingTimeUnit) { super(outgoingName, outgoingDataType, true); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnit, incomingName); _outgoingGranularitySpec = new TimeGranularitySpec(outgoingDataType, outgoingTimeUnit, outgoingName); }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, int incomingTimeUnitSize, @Nonnull TimeUnit incomingTimeUnit) { super(incomingName, incomingDataType, true); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnitSize, incomingTimeUnit, incomingName); }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, int incomingTimeUnitSize, @Nonnull TimeUnit incomingTimeUnit, @Nonnull String outgoingName, @Nonnull DataType outgoingDataType, int outgoingTimeUnitSize, @Nonnull TimeUnit outgoingTimeUnit) { super(outgoingName, outgoingDataType, true); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnitSize, incomingTimeUnit, incomingName); _outgoingGranularitySpec = new TimeGranularitySpec(outgoingDataType, outgoingTimeUnitSize, outgoingTimeUnit, outgoingName); }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, @Nonnull TimeUnit incomingTimeUnit) { super(incomingName, incomingDataType, true); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnit, incomingName); }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, @Nonnull TimeUnit incomingTimeUnit, @Nonnull String outgoingName, @Nonnull DataType outgoingDataType, @Nonnull TimeUnit outgoingTimeUnit, @Nonnull Object defaultNullValue) { super(outgoingName, outgoingDataType, true, defaultNullValue); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnit, incomingName); _outgoingGranularitySpec = new TimeGranularitySpec(outgoingDataType, outgoingTimeUnit, outgoingName); }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, @Nonnull TimeUnit incomingTimeUnit, @Nonnull Object defaultNullValue) { super(incomingName, incomingDataType, true, defaultNullValue); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnit, incomingName); }
public TimeFieldSpec(@Nonnull String incomingName, @Nonnull DataType incomingDataType, int incomingTimeUnitSize, @Nonnull TimeUnit incomingTimeUnit, @Nonnull Object defaultNullValue) { super(incomingName, incomingDataType, true, defaultNullValue); _incomingGranularitySpec = new TimeGranularitySpec(incomingDataType, incomingTimeUnitSize, incomingTimeUnit, incomingName); }
@Test public void testFloatTimeColumn() { TimeConverter converter = new TimeConverter(new TimeGranularitySpec(FieldSpec.DataType.FLOAT, 1, TimeUnit.SECONDS, "time")); // Should support conversion from all data types assertEquals(converter.toMillisSinceEpoch(123), 123 * TimeUnit.SECONDS.toMillis(1)); assertEquals(converter.toMillisSinceEpoch(123L), 123 * TimeUnit.SECONDS.toMillis(1)); assertEquals(converter.toMillisSinceEpoch(123f), 123 * TimeUnit.SECONDS.toMillis(1)); assertEquals(converter.toMillisSinceEpoch(123d), 123 * TimeUnit.SECONDS.toMillis(1)); assertEquals(converter.toMillisSinceEpoch("123"), 123 * TimeUnit.SECONDS.toMillis(1)); assertEquals(converter.fromMillisSinceEpoch(123 * TimeUnit.SECONDS.toMillis(1)), 123f); }
@Test public void testLongTimeColumn() { TimeConverter converter = new TimeConverter(new TimeGranularitySpec(FieldSpec.DataType.LONG, 10, TimeUnit.DAYS, "time")); // Should support conversion from all data types assertEquals(converter.toMillisSinceEpoch(123), 123 * TimeUnit.DAYS.toMillis(10)); assertEquals(converter.toMillisSinceEpoch(123L), 123 * TimeUnit.DAYS.toMillis(10)); assertEquals(converter.toMillisSinceEpoch(123f), 123 * TimeUnit.DAYS.toMillis(10)); assertEquals(converter.toMillisSinceEpoch(123d), 123 * TimeUnit.DAYS.toMillis(10)); assertEquals(converter.toMillisSinceEpoch("123"), 123 * TimeUnit.DAYS.toMillis(10)); assertEquals(converter.fromMillisSinceEpoch(123 * TimeUnit.DAYS.toMillis(10)), 123L); }
@Test public void testDoubleTimeColumn() { TimeConverter converter = new TimeConverter(new TimeGranularitySpec(FieldSpec.DataType.DOUBLE, 3, TimeUnit.MINUTES, "time")); // Should support conversion from all data types assertEquals(converter.toMillisSinceEpoch(123), 123 * TimeUnit.MINUTES.toMillis(3)); assertEquals(converter.toMillisSinceEpoch(123L), 123 * TimeUnit.MINUTES.toMillis(3)); assertEquals(converter.toMillisSinceEpoch(123f), 123 * TimeUnit.MINUTES.toMillis(3)); assertEquals(converter.toMillisSinceEpoch(123d), 123 * TimeUnit.MINUTES.toMillis(3)); assertEquals(converter.toMillisSinceEpoch("123"), 123 * TimeUnit.MINUTES.toMillis(3)); assertEquals(converter.fromMillisSinceEpoch(123 * TimeUnit.MINUTES.toMillis(3)), 123d); }
@Test public void testIntTimeColumn() { TimeConverter converter = new TimeConverter(new TimeGranularitySpec(FieldSpec.DataType.INT, 5, TimeUnit.HOURS, "time")); // Should support conversion from all data types assertEquals(converter.toMillisSinceEpoch(123), 123 * TimeUnit.HOURS.toMillis(5)); assertEquals(converter.toMillisSinceEpoch(123L), 123 * TimeUnit.HOURS.toMillis(5)); assertEquals(converter.toMillisSinceEpoch(123f), 123 * TimeUnit.HOURS.toMillis(5)); assertEquals(converter.toMillisSinceEpoch(123d), 123 * TimeUnit.HOURS.toMillis(5)); assertEquals(converter.toMillisSinceEpoch("123"), 123 * TimeUnit.HOURS.toMillis(5)); assertEquals(converter.fromMillisSinceEpoch(123 * TimeUnit.HOURS.toMillis(5)), 123); }
@Test public void testStringTimeColumn() { TimeConverter converter = new TimeConverter(new TimeGranularitySpec(FieldSpec.DataType.STRING, 100, TimeUnit.MILLISECONDS, "time")); // Should support conversion from all data types assertEquals(converter.toMillisSinceEpoch(123), 123 * TimeUnit.MILLISECONDS.toMillis(100)); assertEquals(converter.toMillisSinceEpoch(123L), 123 * TimeUnit.MILLISECONDS.toMillis(100)); assertEquals(converter.toMillisSinceEpoch(123f), 123 * TimeUnit.MILLISECONDS.toMillis(100)); assertEquals(converter.toMillisSinceEpoch(123d), 123 * TimeUnit.MILLISECONDS.toMillis(100)); assertEquals(converter.toMillisSinceEpoch("123"), 123 * TimeUnit.MILLISECONDS.toMillis(100)); assertEquals(converter.fromMillisSinceEpoch(123 * TimeUnit.MILLISECONDS.toMillis(100)), "123"); } }
@Test public void testTimeConversion() { // When incoming time exists and is valid, do the conversion Schema schema = new Schema(); schema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "incoming"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "outgoing"))); TimeTransformer transformer = new TimeTransformer(schema); GenericRow record = new GenericRow(); record.putField("incoming", TimeUnit.MILLISECONDS.toDays(VALID_TIME)); record = transformer.transform(record); assertNotNull(record); assertEquals(record.getValue("outgoing"), VALID_TIME); // When incoming and outgoing time column is the same, and the value is not yet converted, do the conversion schema = new Schema(); schema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "time"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "time"))); transformer = new TimeTransformer(schema); record = new GenericRow(); record.putField("time", TimeUnit.MILLISECONDS.toDays(VALID_TIME)); record = transformer.transform(record); assertNotNull(record); assertEquals(record.getValue("time"), VALID_TIME); } }
@Test public void testTimeFormat() { // When incoming and outgoing spec are the same, any time format should work Schema schema = new Schema(); schema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, TimeGranularitySpec.TimeFormat.SIMPLE_DATE_FORMAT.toString(), "time"))); TimeTransformer transformer = new TimeTransformer(schema); GenericRow record = new GenericRow(); record.putField("time", 20180101); record = transformer.transform(record); assertNotNull(record); assertEquals(record.getValue("time"), 20180101); // When incoming and outgoing spec are not the same, simple date format is not allowed schema = new Schema(); schema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, TimeGranularitySpec.TimeFormat.SIMPLE_DATE_FORMAT.toString(), "incoming"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "outgoing"))); try { new TimeTransformer(schema); fail(); } catch (IllegalArgumentException e) { // Expected } }
@Test public void testSimpleDateFormat() throws Exception { TimeGranularitySpec incomingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); TimeGranularitySpec outgoingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build(); String jsonSchema = schema.getJSONSchema(); Schema schemaFromJson = Schema.fromString(jsonSchema); Assert.assertEquals(schemaFromJson, schema); Assert.assertEquals(schemaFromJson.hashCode(), schema.hashCode()); }
schema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "incoming"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "outgoing"))); TimeTransformer transformer = new TimeTransformer(schema); GenericRow record = new GenericRow(); schema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "time"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "time"))); transformer = new TimeTransformer(schema); record = new GenericRow(); schema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "incoming"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "outgoing"))); transformer = new TimeTransformer(schema); record = new GenericRow(); schema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "time"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "time"))); transformer = new TimeTransformer(schema); record = new GenericRow();
final TimeGranularitySpec gSpec = new TimeGranularitySpec(getColumnType(field), granularity, columnName); final TimeFieldSpec fSpec = new TimeFieldSpec(gSpec); schema.addField(fSpec);
private Schema createPinotSchema() { Schema testSchema = new Schema(); testSchema.setSchemaName("schema"); testSchema.addField(new DimensionFieldSpec(D_SV_1, DataType.STRING, true)); testSchema.addField(new DimensionFieldSpec(D_MV_1, FieldSpec.DataType.STRING, false)); testSchema.addField(new MetricFieldSpec(M1, FieldSpec.DataType.INT)); testSchema.addField(new MetricFieldSpec(M2, FieldSpec.DataType.FLOAT)); testSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS, TIME))); return testSchema; }
private Schema createPinotSchema() { Schema testSchema = new Schema(); testSchema.setSchemaName("schema"); testSchema.addField(new DimensionFieldSpec(D_SV_1, FieldSpec.DataType.STRING, true)); testSchema.addField(new DimensionFieldSpec(D_SV_2, FieldSpec.DataType.INT, true)); testSchema.addField(new DimensionFieldSpec(D_MV_1, FieldSpec.DataType.STRING, false)); testSchema.addField(new MetricFieldSpec(M1, FieldSpec.DataType.INT)); testSchema.addField(new MetricFieldSpec(M2, FieldSpec.DataType.FLOAT)); testSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS, TIME))); return testSchema; }