/** * An example on how to use this utility class. */ public static void main(String[] args) { Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addSingleValueDimension("dimension", FieldSpec.DataType.DOUBLE).addMetric("metric", FieldSpec.DataType.INT) .addTime("time", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); System.out.println(postSchema("localhost", 8100, schema)); Schema fetchedSchema = getSchema("localhost", 8100, "testSchema"); Preconditions.checkNotNull(fetchedSchema); System.out.println(fetchedSchema); System.out.println(equalsIgnoreVersion(schema, fetchedSchema)); System.out.println(deleteSchema("localhost", 8100, "testSchema")); } }
@BeforeClass public void setUp() { Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addSingleValueDimension(DIMENSION_1, FieldSpec.DataType.INT) .addSingleValueDimension(DIMENSION_2, FieldSpec.DataType.STRING).addMetric(METRIC, FieldSpec.DataType.LONG) .build(); _mutableSegmentImpl = MutableSegmentImplTestUtils .createMutableSegmentImpl(schema, new HashSet<>(Arrays.asList(DIMENSION_1, METRIC)), Collections.singleton(DIMENSION_1), true); }
@Test public void testIncomingTimeColumn() throws Exception { List<Schema.Field> avroFields = Collections.singletonList(new Schema.Field("incomingTime", Schema.create(Schema.Type.LONG), null, null)); Schema avroSchema = Schema.createRecord(avroFields); GenericData.Record avroRecord = new GenericData.Record(avroSchema); avroRecord.put("incomingTime", 12345L); org.apache.pinot.common.data.Schema pinotSchema = new org.apache.pinot.common.data.Schema.SchemaBuilder().setSchemaName("testSchema") .addTime("incomingTime", TimeUnit.MILLISECONDS, FieldSpec.DataType.LONG, "outgoingTime", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); AvroRecordToPinotRowGenerator avroRecordToPinotRowGenerator = new AvroRecordToPinotRowGenerator(pinotSchema); GenericRow genericRow = new GenericRow(); avroRecordToPinotRowGenerator.transform(avroRecord, genericRow); Assert.assertEquals(genericRow.getFieldNames(), new String[]{"incomingTime"}); Assert.assertEquals(genericRow.getValue("incomingTime"), 12345L); } }
new Schema.SchemaBuilder().setSchemaName("testSchema").addTime(incomingName, incomingTimeUnit, incomingDataType) .build(); Schema schema2 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, defaultNullValue).build(); Schema schema3 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnit, outgoingDataType) .build(); Schema schema4 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnit, outgoingDataType, defaultNullValue).build(); Schema schema5 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType).build(); Schema schema6 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, defaultNullValue).build(); Schema schema7 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnitSize, outgoingTimeUnit, outgoingDataType).build(); Schema schema8 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnitSize, outgoingTimeUnit, outgoingDataType, defaultNullValue).build(); Schema schema9 = new Schema.SchemaBuilder().setSchemaName("testSchema").addTime(incomingTimeGranularitySpec).build(); Schema schema10 = new Schema.SchemaBuilder().setSchemaName("testSchema").addTime(incomingTimeGranularitySpec, defaultNullValue) .build(); Schema schema11 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build(); Schema schema12 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec, defaultNullValue).build();
Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT) schema = new Schema.SchemaBuilder().setSchemaName("testTableStarTree").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT)
new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT)
@BeforeTest public void buildSegment() throws Exception { FileUtils.deleteQuietly(INDEX_DIR); // Get resource file path. URL resource = getClass().getClassLoader().getResource(AVRO_DATA); Assert.assertNotNull(resource); String filePath = resource.getFile(); // Build the segment schema. Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT) .addSingleValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column9", FieldSpec.DataType.INT) .addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT) .build(); // Create the segment generator config. SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable"); segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath()); segmentGeneratorConfig .setInvertedIndexCreationColumns(Arrays.asList("column6", "column7", "column11", "column17", "column18")); // Build the index segment. SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig); driver.build(); }
@BeforeTest public void buildSegment() throws Exception { FileUtils.deleteQuietly(INDEX_DIR); // Get resource file path. URL resource = getClass().getClassLoader().getResource(AVRO_DATA); Assert.assertNotNull(resource); String filePath = resource.getFile(); // Build the segment schema. Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column2", FieldSpec.DataType.INT).addSingleValueDimension("column3", FieldSpec.DataType.STRING) .addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addMultiValueDimension("column6", FieldSpec.DataType.INT) .addMultiValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column8", FieldSpec.DataType.INT).addMetric("column9", FieldSpec.DataType.INT) .addMetric("column10", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT) .build(); // Create the segment generator config. SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable"); segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath()); segmentGeneratorConfig.setInvertedIndexCreationColumns(Arrays.asList("column3", "column7", "column8", "column9")); // Build the index segment. SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig); driver.build(); }
@Test public void testSimpleDateFormat() throws Exception { TimeGranularitySpec incomingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); TimeGranularitySpec outgoingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build(); String jsonSchema = schema.getJSONSchema(); Schema schemaFromJson = Schema.fromString(jsonSchema); Assert.assertEquals(schemaFromJson, schema); Assert.assertEquals(schemaFromJson.hashCode(), schema.hashCode()); }
private Schema getSchema() { return new Schema.SchemaBuilder().setSchemaName("ThriftSampleData") .addSingleValueDimension("id", FieldSpec.DataType.INT) .addSingleValueDimension("name", FieldSpec.DataType.STRING) .addSingleValueDimension("created_at", FieldSpec.DataType.LONG) .addSingleValueDimension("active", FieldSpec.DataType.BOOLEAN) .addMultiValueDimension("groups", FieldSpec.DataType.INT) .addMultiValueDimension("map_values", FieldSpec.DataType.STRING) .addMultiValueDimension("set_values", FieldSpec.DataType.STRING).build(); }