/** * Fetch {@link Schema} from a {@link ZNRecord}. */ public static Schema fromZNRecord(@Nonnull ZNRecord record) throws IOException { String schemaJSON = record.getSimpleField("schemaJSON"); return Schema.fromString(schemaJSON); }
return Schema.fromString(response); } finally { httpGet.releaseConnection();
schema = Schema.fromString(schemaString); } catch (Exception e) { LOGGER.error("Could not get schema from string for value: " + schemaString);
@Test public void testValidation() throws Exception { Schema schemaToValidate; schemaToValidate = Schema.fromString(makeSchema(FieldSpec.DataType.LONG, FieldSpec.DataType.STRING, true)); Assert.assertTrue(schemaToValidate.validate(LOGGER)); schemaToValidate = Schema.fromString(makeSchema(FieldSpec.DataType.BOOLEAN, FieldSpec.DataType.STRING, true)); Assert.assertFalse(schemaToValidate.validate(LOGGER)); schemaToValidate = Schema.fromString(makeSchema(FieldSpec.DataType.STRING, FieldSpec.DataType.STRING, false)); Assert.assertFalse(schemaToValidate.validate(LOGGER)); schemaToValidate = Schema.fromString(makeSchema(FieldSpec.DataType.LONG, FieldSpec.DataType.BOOLEAN, false)); Assert.assertTrue(schemaToValidate.validate(LOGGER)); }
private void validate(Map<Integer, Emp> inputMap) throws Exception { File segmentTarOutput = new File(outputTempDir, segmentTarPath); File untarOutput = Files.createTempDirectory(PinotOutputFormatTest.class.getName() + "_segmentUnTar").toFile(); for (File tarFile : segmentTarOutput.listFiles()) { TarGzCompressionUtils.unTar(tarFile, untarOutput); } File outputDir = new File(untarOutput, PinotOutputFormat.getSegmentName(fakeTaskAttemptContext)); RecordReader recordReader = new PinotSegmentRecordReader(outputDir, Schema.fromString(getSchema()), null); Map<Integer, GenericRow> resultMap = new HashMap<>(); while (recordReader.hasNext()) { GenericRow row = recordReader.next(); resultMap.put((Integer) row.getValue("id"), row); } Assert.assertEquals(resultMap.size(), inputMap.size()); Assert.assertEquals(resultMap.get(8).getValue("name"), inputMap.get(8).name); }
private void init(String indexType) throws IOException { conf = new Configuration(); job = Job.getInstance(conf); fakeTaskAttemptContext = mock(TaskAttemptContext.class); outputFormat = new JsonPinotOutputFormat(); outputTempDir = Files.createTempDirectory(PinotOutputFormatTest.class.getName() + indexType + "_io_output").toFile(); File workingTempDir = Files.createTempDirectory(PinotOutputFormatTest.class.getName() + indexType + "_io_working_dir").toFile(); // output path Path outDir = new Path(outputTempDir.getAbsolutePath()); PinotOutputFormat.setOutputPath(job, outDir); PinotOutputFormat.setTableName(job, "emp"); PinotOutputFormat.setSegmentName(job, indexType + "segment_one"); PinotOutputFormat.setTempSegmentDir(job, workingTempDir.getAbsolutePath()); Schema schema = Schema.fromString(getSchema()); PinotOutputFormat.setSchema(job, schema); mockTaskAttemptContext(indexType); segmentTarPath = "_temporary/0/_temporary/attempt_foo_task_" + indexType + "_0123_r_000002_2/part-r-00002/segmentTar"; }
@Test public void testSerializeDeserialize() throws Exception { URL resourceUrl = getClass().getClassLoader().getResource("schemaTest.schema"); Assert.assertNotNull(resourceUrl); Schema schema = Schema.fromFile(new File(resourceUrl.getFile())); Schema schemaToCompare = Schema.fromString(schema.getJSONSchema()); Assert.assertEquals(schemaToCompare, schema); Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode()); schemaToCompare = SchemaUtils.fromZNRecord(SchemaUtils.toZNRecord(schema)); Assert.assertEquals(schemaToCompare, schema); Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode()); // When setting new fields, schema string should be updated String JSONSchema = schemaToCompare.getJSONSchema(); schemaToCompare.setSchemaName("newSchema"); String JSONSchemaToCompare = schemaToCompare.getJSONSchema(); Assert.assertFalse(JSONSchema.equals(JSONSchemaToCompare)); }
@Test public void testSimpleDateFormat() throws Exception { TimeGranularitySpec incomingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); TimeGranularitySpec outgoingTimeGranularitySpec = new TimeGranularitySpec(FieldSpec.DataType.STRING, 1, TimeUnit.DAYS, TimeFormat.SIMPLE_DATE_FORMAT + ":yyyyMMdd", "Date"); Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addTime(incomingTimeGranularitySpec, outgoingTimeGranularitySpec).build(); String jsonSchema = schema.getJSONSchema(); Schema schemaFromJson = Schema.fromString(jsonSchema); Assert.assertEquals(schemaFromJson, schema); Assert.assertEquals(schemaFromJson.hashCode(), schema.hashCode()); }
@Test public void testByteType() throws DecoderException, IOException { Schema expectedSchema = new Schema(); byte[] expectedEmptyDefault = new byte[0]; byte[] expectedNonEmptyDefault = Hex.decodeHex("abcd1234".toCharArray()); expectedSchema.setSchemaName("test"); expectedSchema.addField(new MetricFieldSpec("noDefault", FieldSpec.DataType.BYTES)); expectedSchema.addField(new MetricFieldSpec("emptyDefault", FieldSpec.DataType.BYTES, expectedEmptyDefault)); expectedSchema.addField(new MetricFieldSpec("nonEmptyDefault", FieldSpec.DataType.BYTES, expectedNonEmptyDefault)); // Ensure that schema can be serialized and de-serialized (ie byte[] converted to String and back). String jsonSchema = expectedSchema.getJSONSchema(); Schema actualSchema = Schema.fromString(jsonSchema); Assert.assertEquals(actualSchema.getFieldSpecFor("noDefault").getDefaultNullValue(), expectedEmptyDefault); Assert.assertEquals(actualSchema.getFieldSpecFor("emptyDefault").getDefaultNullValue(), expectedEmptyDefault); Assert.assertEquals(actualSchema.getFieldSpecFor("nonEmptyDefault").getDefaultNullValue(), expectedNonEmptyDefault); Assert.assertEquals(actualSchema, expectedSchema); Assert.assertEquals(actualSchema.hashCode(), expectedSchema.hashCode()); } }
private void initSegmentConfig(JobContext context) throws IOException { _segmentConfig.setFormat(FileFormat.JSON); _segmentConfig.setOutDir(PinotOutputFormat.getTempSegmentDir(context) + "/segmentDir"); _segmentConfig.setOverwrite(true); _segmentConfig.setTableName(PinotOutputFormat.getTableName(context)); _segmentConfig.setSegmentName(PinotOutputFormat.getSegmentName(context)); _segmentConfig.setSchema(Schema.fromString(PinotOutputFormat.getSchema(context))); _segmentConfig.setReaderConfigFile(PinotOutputFormat.getReaderConfig(context)); initStarTreeIndex(context); initHllConfig(context); }
Schema readSchema = Schema.fromString(schemaStr); Schema inputSchema = Schema.fromString(schema.toString()); Assert.assertEquals(readSchema, inputSchema); Assert.assertTrue(readSchema.getFieldSpecMap().containsKey("NewColumn")); readSchema = Schema.fromString(schemaStr); inputSchema = Schema.fromString(schema.toString()); Assert.assertEquals(readSchema, inputSchema); Assert.assertTrue(readSchema.getFieldSpecMap().containsKey(yetAnotherColumn));
private FakeLLRealtimeSegmentDataManager createFakeSegmentManager() throws Exception { LLCRealtimeSegmentZKMetadata segmentZKMetadata = createZkMetadata(); TableConfig tableConfig = createTableConfig(); InstanceZKMetadata instanceZKMetadata = new InstanceZKMetadata(); RealtimeTableDataManager tableDataManager = createTableDataManager(); String resourceDir = _segmentDir; Schema schema = Schema.fromString(makeSchema()); ServerMetrics serverMetrics = new ServerMetrics(new MetricsRegistry()); FakeLLRealtimeSegmentDataManager segmentDataManager = new FakeLLRealtimeSegmentDataManager(segmentZKMetadata, tableConfig, instanceZKMetadata, tableDataManager, resourceDir, schema, serverMetrics); return segmentDataManager; }