/** * An example on how to use this utility class. */ public static void main(String[] args) { Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addSingleValueDimension("dimension", FieldSpec.DataType.DOUBLE).addMetric("metric", FieldSpec.DataType.INT) .addTime("time", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); System.out.println(postSchema("localhost", 8100, schema)); Schema fetchedSchema = getSchema("localhost", 8100, "testSchema"); Preconditions.checkNotNull(fetchedSchema); System.out.println(fetchedSchema); System.out.println(equalsIgnoreVersion(schema, fetchedSchema)); System.out.println(deleteSchema("localhost", 8100, "testSchema")); } }
public static void main(String[] args) throws IOException { SchemaBuilder schemaBuilder = new SchemaBuilder(); schemaBuilder.addSingleValueDimension("name", DataType.STRING); schemaBuilder.addSingleValueDimension("age", DataType.INT); schemaBuilder.addMetric("percent", DataType.FLOAT); schemaBuilder.addTime("days", TimeUnit.DAYS, DataType.LONG); Schema schema = schemaBuilder.build(); System.out.println(JsonUtils.objectToPrettyString(schema)); } }
.addMetric("derivedMetric", FieldSpec.DataType.STRING, 10, MetricFieldSpec.DerivedMetricType.HLL) .addMetric("derivedMetricWithDefault", FieldSpec.DataType.STRING, 10, MetricFieldSpec.DerivedMetricType.HLL, defaultString).addTime("time", TimeUnit.DAYS, FieldSpec.DataType.LONG) .addDateTime("dateTime", FieldSpec.DataType.LONG, "1:HOURS:EPOCH", "1:HOURS").build();
new Schema.SchemaBuilder().setSchemaName("testSchema").addTime(incomingName, incomingTimeUnit, incomingDataType) .build(); Schema schema2 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, defaultNullValue).build(); Schema schema3 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnit, outgoingDataType) .build(); Schema schema4 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnit, outgoingDataType, defaultNullValue).build(); Schema schema5 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType).build(); Schema schema6 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, defaultNullValue).build(); Schema schema7 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnitSize, outgoingTimeUnit, outgoingDataType).build(); Schema schema8 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnitSize, outgoingTimeUnit, outgoingDataType, defaultNullValue).build(); Schema schema9 = new Schema.SchemaBuilder().setSchemaName("testSchema").addTime(incomingTimeGranularitySpec).build(); Schema schema10 = new Schema.SchemaBuilder().setSchemaName("testSchema").addTime(incomingTimeGranularitySpec, defaultNullValue) .build(); Schema schema11 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build(); Schema schema12 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec, defaultNullValue).build();
.addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", 1, TimeUnit.DAYS, DataType.INT) .build(); .addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", 1, TimeUnit.DAYS, DataType.INT) .build();
.addSingleValueDimension("column12", FieldSpec.DataType.STRING) .addMetric("column17", FieldSpec.DataType.INT).addMetric("column18", FieldSpec.DataType.INT) .addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT); if (hasPreGeneratedHllColumns) { schemaBuilder.addSingleValueDimension("column17_HLL", FieldSpec.DataType.STRING)
@BeforeTest public void buildSegment() throws Exception { FileUtils.deleteQuietly(INDEX_DIR); // Get resource file path. URL resource = getClass().getClassLoader().getResource(AVRO_DATA); Assert.assertNotNull(resource); String filePath = resource.getFile(); // Build the segment schema. Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT) .addSingleValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column9", FieldSpec.DataType.INT) .addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT) .build(); // Create the segment generator config. SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable"); segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath()); segmentGeneratorConfig .setInvertedIndexCreationColumns(Arrays.asList("column6", "column7", "column11", "column17", "column18")); // Build the index segment. SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig); driver.build(); }
@BeforeTest public void buildSegment() throws Exception { FileUtils.deleteQuietly(INDEX_DIR); // Get resource file path. URL resource = getClass().getClassLoader().getResource(AVRO_DATA); Assert.assertNotNull(resource); String filePath = resource.getFile(); // Build the segment schema. Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column2", FieldSpec.DataType.INT).addSingleValueDimension("column3", FieldSpec.DataType.STRING) .addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addMultiValueDimension("column6", FieldSpec.DataType.INT) .addMultiValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column8", FieldSpec.DataType.INT).addMetric("column9", FieldSpec.DataType.INT) .addMetric("column10", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT) .build(); // Create the segment generator config. SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable"); segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath()); segmentGeneratorConfig.setInvertedIndexCreationColumns(Arrays.asList("column3", "column7", "column8", "column9")); // Build the index segment. SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig); driver.build(); }
@Test public void testIncomingTimeColumn() throws Exception { List<Schema.Field> avroFields = Collections.singletonList(new Schema.Field("incomingTime", Schema.create(Schema.Type.LONG), null, null)); Schema avroSchema = Schema.createRecord(avroFields); GenericData.Record avroRecord = new GenericData.Record(avroSchema); avroRecord.put("incomingTime", 12345L); org.apache.pinot.common.data.Schema pinotSchema = new org.apache.pinot.common.data.Schema.SchemaBuilder().setSchemaName("testSchema") .addTime("incomingTime", TimeUnit.MILLISECONDS, FieldSpec.DataType.LONG, "outgoingTime", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); AvroRecordToPinotRowGenerator avroRecordToPinotRowGenerator = new AvroRecordToPinotRowGenerator(pinotSchema); GenericRow genericRow = new GenericRow(); avroRecordToPinotRowGenerator.transform(avroRecord, genericRow); Assert.assertEquals(genericRow.getFieldNames(), new String[]{"incomingTime"}); Assert.assertEquals(genericRow.getValue("incomingTime"), 12345L); } }
@Test public void testSimpleDateFormat() throws Exception { TimeGranularitySpec incomingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); TimeGranularitySpec outgoingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build(); String jsonSchema = schema.getJSONSchema(); Schema schemaFromJson = Schema.fromString(jsonSchema); Assert.assertEquals(schemaFromJson, schema); Assert.assertEquals(schemaFromJson.hashCode(), schema.hashCode()); }