/** * An example on how to use this utility class. */ public static void main(String[] args) { Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addSingleValueDimension("dimension", FieldSpec.DataType.DOUBLE).addMetric("metric", FieldSpec.DataType.INT) .addTime("time", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); System.out.println(postSchema("localhost", 8100, schema)); Schema fetchedSchema = getSchema("localhost", 8100, "testSchema"); Preconditions.checkNotNull(fetchedSchema); System.out.println(fetchedSchema); System.out.println(equalsIgnoreVersion(schema, fetchedSchema)); System.out.println(deleteSchema("localhost", 8100, "testSchema")); } }
@BeforeClass public void setUp() { Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addSingleValueDimension(DIMENSION_1, FieldSpec.DataType.INT) .addSingleValueDimension(DIMENSION_2, FieldSpec.DataType.STRING).addMetric(METRIC, FieldSpec.DataType.LONG) .build(); _mutableSegmentImpl = MutableSegmentImplTestUtils .createMutableSegmentImpl(schema, new HashSet<>(Arrays.asList(DIMENSION_1, METRIC)), Collections.singleton(DIMENSION_1), true); }
.addSingleValueDimension("column2", DataType.STRING).build(); final SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema);
.addMetric("derivedMetricWithDefault", FieldSpec.DataType.STRING, 10, MetricFieldSpec.DerivedMetricType.HLL, defaultString).addTime("time", TimeUnit.DAYS, FieldSpec.DataType.LONG) .addDateTime("dateTime", FieldSpec.DataType.LONG, "1:HOURS:EPOCH", "1:HOURS").build();
.build(); Schema schema2 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, defaultNullValue).build(); Schema schema3 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnit, outgoingDataType) .build(); Schema schema4 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnit, outgoingDataType, defaultNullValue).build(); Schema schema5 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType).build(); Schema schema6 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, defaultNullValue).build(); Schema schema7 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnitSize, outgoingTimeUnit, outgoingDataType).build(); Schema schema8 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingName, incomingTimeUnitSize, incomingTimeUnit, incomingDataType, outgoingName, outgoingTimeUnitSize, outgoingTimeUnit, outgoingDataType, defaultNullValue).build(); Schema schema9 = new Schema.SchemaBuilder().setSchemaName("testSchema").addTime(incomingTimeGranularitySpec).build(); Schema schema10 = new Schema.SchemaBuilder().setSchemaName("testSchema").addTime(incomingTimeGranularitySpec, defaultNullValue) .build(); Schema schema11 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build(); Schema schema12 = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec, defaultNullValue).build();
.addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", 1, TimeUnit.DAYS, DataType.INT) .build(); .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", 1, TimeUnit.DAYS, DataType.INT) .build();
SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schemaBuilder.build()); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable");
schemaBuilder.addMetric(METRIC, rawValueType); Schema schema = schemaBuilder.build();
@Test public void TestReadAvro() throws Exception { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA)); final String jsonPath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(JSON_DATA)); Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("column3", DataType.STRING) .addSingleValueDimension("column2", DataType.STRING).build(); final SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema); config.setFormat(FileFormat.AVRO); config.setInputFilePath(filePath); config.setSegmentVersion(SegmentVersion.v1); AvroRecordReader avroDataPublisher = (AvroRecordReader) RecordReaderFactory.getRecordReader(config); int cnt = 0; for (String line : FileUtils.readLines(new File(jsonPath))) { JsonNode jsonNode = JsonUtils.stringToJsonNode(line); if (avroDataPublisher.hasNext()) { GenericRow recordRow = avroDataPublisher.next(); for (String column : recordRow.getFieldNames()) { String valueFromJson = jsonNode.get(column).asText(); String valueFromAvro = recordRow.getValue(column).toString(); if (cnt > 1) { Assert.assertEquals(valueFromJson, valueFromAvro); } } } cnt++; } Assert.assertEquals(cnt, 10001); }
@BeforeTest public void buildSegment() throws Exception { FileUtils.deleteQuietly(INDEX_DIR); // Get resource file path. URL resource = getClass().getClassLoader().getResource(AVRO_DATA); Assert.assertNotNull(resource); String filePath = resource.getFile(); // Build the segment schema. Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT) .addSingleValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column9", FieldSpec.DataType.INT) .addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT) .build(); // Create the segment generator config. SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable"); segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath()); segmentGeneratorConfig .setInvertedIndexCreationColumns(Arrays.asList("column6", "column7", "column11", "column17", "column18")); // Build the index segment. SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig); driver.build(); }
@BeforeTest public void buildSegment() throws Exception { FileUtils.deleteQuietly(INDEX_DIR); // Get resource file path. URL resource = getClass().getClassLoader().getResource(AVRO_DATA); Assert.assertNotNull(resource); String filePath = resource.getFile(); // Build the segment schema. Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column2", FieldSpec.DataType.INT).addSingleValueDimension("column3", FieldSpec.DataType.STRING) .addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addMultiValueDimension("column6", FieldSpec.DataType.INT) .addMultiValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column8", FieldSpec.DataType.INT).addMetric("column9", FieldSpec.DataType.INT) .addMetric("column10", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT) .build(); // Create the segment generator config. SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable"); segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath()); segmentGeneratorConfig.setInvertedIndexCreationColumns(Arrays.asList("column3", "column7", "column8", "column9")); // Build the index segment. SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig); driver.build(); }
@Test public void testIncomingTimeColumn() throws Exception { List<Schema.Field> avroFields = Collections.singletonList(new Schema.Field("incomingTime", Schema.create(Schema.Type.LONG), null, null)); Schema avroSchema = Schema.createRecord(avroFields); GenericData.Record avroRecord = new GenericData.Record(avroSchema); avroRecord.put("incomingTime", 12345L); org.apache.pinot.common.data.Schema pinotSchema = new org.apache.pinot.common.data.Schema.SchemaBuilder().setSchemaName("testSchema") .addTime("incomingTime", TimeUnit.MILLISECONDS, FieldSpec.DataType.LONG, "outgoingTime", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); AvroRecordToPinotRowGenerator avroRecordToPinotRowGenerator = new AvroRecordToPinotRowGenerator(pinotSchema); GenericRow genericRow = new GenericRow(); avroRecordToPinotRowGenerator.transform(avroRecord, genericRow); Assert.assertEquals(genericRow.getFieldNames(), new String[]{"incomingTime"}); Assert.assertEquals(genericRow.getValue("incomingTime"), 12345L); } }
@Test public void testSimpleDateFormat() throws Exception { TimeGranularitySpec incomingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); TimeGranularitySpec outgoingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build(); String jsonSchema = schema.getJSONSchema(); Schema schemaFromJson = Schema.fromString(jsonSchema); Assert.assertEquals(schemaFromJson, schema); Assert.assertEquals(schemaFromJson.hashCode(), schema.hashCode()); }
private Schema getSchema() { return new Schema.SchemaBuilder().setSchemaName("ThriftSampleData") .addSingleValueDimension("id", FieldSpec.DataType.INT) .addSingleValueDimension("name", FieldSpec.DataType.STRING) .addSingleValueDimension("created_at", FieldSpec.DataType.LONG) .addSingleValueDimension("active", FieldSpec.DataType.BOOLEAN) .addMultiValueDimension("groups", FieldSpec.DataType.INT) .addMultiValueDimension("map_values", FieldSpec.DataType.STRING) .addMultiValueDimension("set_values", FieldSpec.DataType.STRING).build(); }
public static void main(String[] args) throws IOException { SchemaBuilder schemaBuilder = new SchemaBuilder(); schemaBuilder.addSingleValueDimension("name", DataType.STRING); schemaBuilder.addSingleValueDimension("age", DataType.INT); schemaBuilder.addMetric("percent", DataType.FLOAT); schemaBuilder.addTime("days", TimeUnit.DAYS, DataType.LONG); Schema schema = schemaBuilder.build(); System.out.println(JsonUtils.objectToPrettyString(schema)); } }