public SchemaBuilder setSchemaName(@Nonnull String schemaName) { _schema.setSchemaName(schemaName); return this; }
@Override public boolean execute() throws Exception { if (_dimensions == null && _metrics == null && _timeColumnName == null) { LOGGER.error( "Error: Missing required argument, please specify at least one of -dimensions, -metrics, -timeColumnName"); return false; } Schema schema; if (_avroSchemaFile != null) { schema = AvroUtils.getPinotSchemaFromAvroSchemaFile(new File(_avroSchemaFile), buildFieldTypesMap(), _timeUnit); } else if (_avroDataFile != null) { schema = AvroUtils.getPinotSchemaFromAvroDataFile(new File(_avroDataFile), buildFieldTypesMap(), _timeUnit); } else { LOGGER.error("Error: Missing required argument, please specify either -avroSchemaFile, or -avroDataFile"); return false; } schema.setSchemaName(_pinotSchemaName); File outputDir = new File(_outputDir); if (!outputDir.isDirectory()) { LOGGER.error("ERROR: Output directory: %s does not exist or is not a directory", _outputDir); } File outputFile = new File(outputDir, _pinotSchemaName + ".json"); LOGGER.info("Store Pinot schema to file: {}", outputFile.getAbsolutePath()); try (FileWriter writer = new FileWriter(outputFile)) { writer.write(schema.toString()); } return true; }
@Test public void testSerializeDeserialize() throws Exception { URL resourceUrl = getClass().getClassLoader().getResource("schemaTest.schema"); Assert.assertNotNull(resourceUrl); Schema schema = Schema.fromFile(new File(resourceUrl.getFile())); Schema schemaToCompare = Schema.fromString(schema.getJSONSchema()); Assert.assertEquals(schemaToCompare, schema); Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode()); schemaToCompare = SchemaUtils.fromZNRecord(SchemaUtils.toZNRecord(schema)); Assert.assertEquals(schemaToCompare, schema); Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode()); // When setting new fields, schema string should be updated String JSONSchema = schemaToCompare.getJSONSchema(); schemaToCompare.setSchemaName("newSchema"); String JSONSchemaToCompare = schemaToCompare.getJSONSchema(); Assert.assertFalse(JSONSchema.equals(JSONSchemaToCompare)); }
@Test public void testByteType() throws DecoderException, IOException { Schema expectedSchema = new Schema(); byte[] expectedEmptyDefault = new byte[0]; byte[] expectedNonEmptyDefault = Hex.decodeHex("abcd1234".toCharArray()); expectedSchema.setSchemaName("test"); expectedSchema.addField(new MetricFieldSpec("noDefault", FieldSpec.DataType.BYTES)); expectedSchema.addField(new MetricFieldSpec("emptyDefault", FieldSpec.DataType.BYTES, expectedEmptyDefault)); expectedSchema.addField(new MetricFieldSpec("nonEmptyDefault", FieldSpec.DataType.BYTES, expectedNonEmptyDefault)); // Ensure that schema can be serialized and de-serialized (ie byte[] converted to String and back). String jsonSchema = expectedSchema.getJSONSchema(); Schema actualSchema = Schema.fromString(jsonSchema); Assert.assertEquals(actualSchema.getFieldSpecFor("noDefault").getDefaultNullValue(), expectedEmptyDefault); Assert.assertEquals(actualSchema.getFieldSpecFor("emptyDefault").getDefaultNullValue(), expectedEmptyDefault); Assert.assertEquals(actualSchema.getFieldSpecFor("nonEmptyDefault").getDefaultNullValue(), expectedNonEmptyDefault); Assert.assertEquals(actualSchema, expectedSchema); Assert.assertEquals(actualSchema.hashCode(), expectedSchema.hashCode()); } }
private void setupRealtimeTable() throws IOException { // Set up the realtime table. Map<String, String> streamConfigs = new HashMap<>(); streamConfigs.put("streamType", "kafka"); streamConfigs.put("stream.kafka.consumer.type", "highLevel"); streamConfigs.put("stream.kafka.topic.name", "kafkaTopic"); streamConfigs .put("stream.kafka.decoder.class.name", "org.apache.pinot.core.realtime.impl.kafka.KafkaAvroMessageDecoder"); streamConfigs.put("stream.kafka.hlc.zk.connect.string", "localhost:1111/zkConnect"); streamConfigs.put("stream.kafka.decoder.prop.schema.registry.rest.url", "http://localhost:2222/schemaRegistry"); TableConfig realtimeTimeConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(RAW_DINING_TABLE_NAME) .setTimeColumnName("timeColumn").setTimeType("DAYS"). setStreamConfigs(streamConfigs).build(); Schema schema = new Schema(); schema.setSchemaName(RAW_DINING_TABLE_NAME); _pinotResourceManager.addOrUpdateSchema(schema); // Fake an PinotLLCRealtimeSegmentManager instance: required for a realtime table creation. PinotLLCRealtimeSegmentManager .create(_pinotResourceManager, new ControllerConf(), new ControllerMetrics(new MetricsRegistry())); _pinotResourceManager.addTable(realtimeTimeConfig); _helixBrokerStarter.getHelixExternalViewBasedRouting() .markDataResourceOnline(realtimeTimeConfig, null, new ArrayList<InstanceConfig>()); }
schema.setSchemaName("differentSchemaName"); putMethod = sendMultipartPutRequest(_controllerRequestURLBuilder.forSchemaUpdate(schemaName), schema.toString()); Assert.assertEquals(putMethod.getStatusCode(), 400);
protected Schema createDummySchema(String tableName) { Schema schema = new Schema(); schema.setSchemaName(tableName); schema.addField(new DimensionFieldSpec("dimA", FieldSpec.DataType.STRING, true, "")); schema.addField(new DimensionFieldSpec("dimB", FieldSpec.DataType.STRING, true, 0)); schema.addField(new MetricFieldSpec("metricA", FieldSpec.DataType.INT, 0)); schema.addField(new MetricFieldSpec("metricB", FieldSpec.DataType.DOUBLE, -1)); return schema; }
private Schema createPinotSchemaWithTimeSpec(TimeFieldSpec timeSpec) { Schema testSchema = new Schema(); testSchema.setSchemaName("schema"); FieldSpec spec; spec = new DimensionFieldSpec(D1, DataType.STRING, true); testSchema.addField(spec); spec = new DimensionFieldSpec(D2, DataType.STRING, true); testSchema.addField(spec); spec = new MetricFieldSpec(M1, DataType.INT); testSchema.addField(spec); spec = new MetricFieldSpec(M2, DataType.FLOAT); testSchema.addField(spec); testSchema.addField(timeSpec); return testSchema; }
private Schema createPinotSchema() { Schema testSchema = new Schema(); testSchema.setSchemaName("schema"); testSchema.addField(new DimensionFieldSpec(D_SV_1, DataType.STRING, true)); testSchema.addField(new DimensionFieldSpec(D_MV_1, FieldSpec.DataType.STRING, false)); testSchema.addField(new MetricFieldSpec(M1, FieldSpec.DataType.INT)); testSchema.addField(new MetricFieldSpec(M2, FieldSpec.DataType.FLOAT)); testSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS, TIME))); return testSchema; }
private Schema createPinotSchema() { Schema testSchema = new Schema(); testSchema.setSchemaName("schema"); testSchema.addField(new DimensionFieldSpec(D_SV_1, FieldSpec.DataType.STRING, true)); testSchema.addField(new DimensionFieldSpec(D_SV_2, FieldSpec.DataType.INT, true)); testSchema.addField(new DimensionFieldSpec(D_MV_1, FieldSpec.DataType.STRING, false)); testSchema.addField(new MetricFieldSpec(M1, FieldSpec.DataType.INT)); testSchema.addField(new MetricFieldSpec(M2, FieldSpec.DataType.FLOAT)); testSchema.addField(new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS, TIME))); return testSchema; }