@Override public Schema getWrappedSchema(@NotEmpty final String schemaName) { return getSchema(schemaName); }
private String getSchemaNameFromFileName(@NotEmpty final String schemaFile) { return getFileParts(schemaFile)[0]; }
private Schema getSchemaFromFile(@NonNull final LocatedFileStatus resultSchemaFile) throws IOException { return getSchemaFromPath(resultSchemaFile.getPath()); }
@Test public void testInvalidBytes() throws Exception { final HDFSSchemaService ss = getHdfsSchemaService(); final Schema wrongSchema = ss.getSchema("wrongSchema"); final GenericRecord data = new GenericRecordBuilder(wrongSchema).set("foo", "boo").build(); final byte[] bytes = ss.getWriter("wrongSchema", 1).write(data); try { ss.getReader(SCHEMA_NAME, 1).read(bytes); Assert.fail("Didn't throw error trying to read data"); } catch (InvalidDataException e) { // pass } }
final LocatedFileStatus f = fileIterator.next(); final String schemaFile = f.getPath().getName(); final String schemaFileSchemaName = getSchemaNameFromFileName(schemaFile); if (schemaFileSchemaName.equals(schemaName)) { final int schemaVersion = getSchemaVersionFromFileName(schemaFile); if (schemaVersion > resultSchemaVersion) { resultSchemaFile = f; String.format("Unable to find schema %s in %s", schemaName, conf.getPath())); } else { return getSchemaFromFile(resultSchemaFile);
@Test(expected = InvalidDataException.class) public void testInvalidGR() throws Exception { final HDFSSchemaService ss = getHdfsSchemaService(); final Schema wrongSchema = ss.getSchema("wrongSchema"); final GenericRecord data = new GenericRecordBuilder(wrongSchema).set("foo", "boo").build(); ss.getWriter(SCHEMA_NAME, 1).write(data); }
private HDFSSchemaService getHdfsSchemaService() { final Configuration conf = new Configuration(); conf.setProperty(HDFSSchemaServiceConfiguration.PATH, "src/test/resources/schemas/schemasource"); return new HDFSSchemaService(conf); } }
@Test public void testSchema() throws Exception { final HDFSSchemaService ss = getHdfsSchemaService(); final Schema schema1 = ss.getSchema(SCHEMA_NAME, 1); final GenericRecord data1 = new GenericRecordBuilder(schema1).set("firstName", "Eric").set("lastName", "Sayle").build(); final byte[] bytes1 = ss.getWriter(SCHEMA_NAME, 1).write(data1); final GenericRecord output1 = ss.getReader(SCHEMA_NAME, 1).read(bytes1); Assert.assertEquals(output1.get("firstName").toString(), "Eric"); Assert.assertEquals(output1.get("lastName").toString(), "Sayle"); final Schema schema2 = ss.getSchema(SCHEMA_NAME); final GenericRecord data2 = new GenericRecordBuilder(schema2).set("firstName", "Eason").set("lastName", "Lu").set("middleName", "Fitzgerald").build(); final byte[] bytes2 = ss.getWriter(SCHEMA_NAME).write(data2); final GenericRecord output2 = ss.getReader(SCHEMA_NAME).read(bytes2); Assert.assertEquals(output2.get("firstName").toString(), "Eason"); Assert.assertEquals(output2.get("lastName").toString(), "Lu"); Assert.assertEquals(output2.get("middleName").toString(), "Fitzgerald"); }
final ISchemaService schemaService = new HDFSSchemaService(conf); final String schemaName = conf.getProperty(FileSourceConfiguration.SCHEMA).get(); final Schema schema = schemaService.getSchema(schemaName);
/** * It returns a writer with latest schema * @param schemaName Fully qualified schema name * @return An instance of {@link HDFSSchemaServiceWriter} */ public HDFSSchemaServiceWriter getWriter(@NotEmpty final String schemaName) { return new HDFSSchemaServiceWriter(getSchema(schemaName)); }
@Override public HDFSSchemaServiceReader getReader(@NotEmpty final String schemaName, final int schemaVersion) { HDFSSchemaServiceReader reader; try { final HDFSSchemaServiceConfiguration conf = new HDFSSchemaServiceConfiguration(this.conf); Path schemaPath = new Path(conf.getPath(), String.format(AVRO_SCHEMA_FILE_PATTERN, schemaName, schemaVersion)); reader = new HDFSSchemaServiceReader(getSchemaFromPath(schemaPath)); } catch (IOException e) { throw new JobRuntimeException("Unable to load schema", e); } return reader; }
private int getSchemaVersionFromFileName(@NotEmpty final String schemaFile) { return Integer.valueOf(getFileParts(schemaFile)[1]); }
/** * It returns a reader with latest schema * @param schemaName Fully qualified schema name * @return An instance of {@link HDFSSchemaServiceReader} */ public HDFSSchemaServiceReader getReader(@NotEmpty final String schemaName) { return new HDFSSchemaServiceReader(getSchema(schemaName)); }
@Override public HDFSSchemaServiceWriter getWriter(@NotEmpty final String schemaName, final int schemaVersion) { HDFSSchemaServiceWriter writer; try { final HDFSSchemaServiceConfiguration conf = new HDFSSchemaServiceConfiguration(this.conf); Path schemaPath = new Path(conf.getPath(), String.format(AVRO_SCHEMA_FILE_PATTERN, schemaName, schemaVersion)); writer = new HDFSSchemaServiceWriter(getSchemaFromPath(schemaPath)); } catch (IOException e) { throw new JobRuntimeException("Unable to load schema", e); } return writer; }
@Test(expected = JobRuntimeException.class) public void testSchemaNotFound() throws Exception { final HDFSSchemaService ss = getHdfsSchemaService(); ss.getSchema("schemaDNE"); }
/** * It fetches latest version of the schema. * @param schemaName Fully qualified schema name * @param schemaVersion schema version * @return Avro schema */ public Schema getSchema(@NotEmpty final String schemaName, final int schemaVersion) { try { final HDFSSchemaServiceConfiguration conf = new HDFSSchemaServiceConfiguration(this.conf); Path schemaPath = new Path(conf.getPath(), String.format(AVRO_SCHEMA_FILE_PATTERN, schemaName, schemaVersion)); return getSchemaFromPath(schemaPath); } catch (IOException e) { throw new JobRuntimeException("Unable to load schema", e); } }