private RecordSchema createRecordSchema(final JsonNode schemaNode) throws SchemaNotFoundException { final String subject = schemaNode.get(SUBJECT_FIELD_NAME).asText(); final int version = schemaNode.get(VERSION_FIELD_NAME).asInt(); final int id = schemaNode.get(ID_FIELD_NAME).asInt(); final String schemaText = schemaNode.get(SCHEMA_TEXT_FIELD_NAME).asText(); try { final Schema avroSchema = new Schema.Parser().parse(schemaText); final SchemaIdentifier schemaId = SchemaIdentifier.builder().name(subject).id(Long.valueOf(id)).version(version).build(); final RecordSchema recordSchema = AvroTypeUtil.createSchema(avroSchema, schemaText, schemaId); return recordSchema; } catch (final SchemaParseException spe) { throw new SchemaNotFoundException("Obtained Schema with id " + id + " and name " + subject + " from Confluent Schema Registry but the Schema Text that was returned is not a valid Avro Schema"); } }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves and returns the RecordSchema based on the provided name of the schema available in Schema Registry. The RecordSchema * that is returned must have the Schema's name populated in its SchemaIdentifier. I.e., a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getName() getName()} * will always return an {@link java.util.Optional} that is not empty. * * @return the latest version of the schema with the given name, or <code>null</code> if no schema can be found with the given name. * @throws SchemaNotFoundException if unable to find the schema with the given name */ default RecordSchema retrieveSchema(String schemaName) throws IOException, SchemaNotFoundException { return retrieveSchema(SchemaIdentifier.builder().name(schemaName).build()); }
@Override public RecordSchema getSchema(final Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException, IOException { final byte[] buffer = new byte[13]; try { StreamUtils.fillBuffer(contentStream, buffer); } catch (final IOException ioe) { throw new SchemaNotFoundException("Could not read first 13 bytes from stream", ioe); } // This encoding follows the pattern that is provided for serializing data by the Hortonworks Schema Registry serializer // as it is provided at: // https://github.com/hortonworks/registry/blob/master/schema-registry/serdes/src/main/java/com/hortonworks/registries/schemaregistry/serdes/avro/AvroSnapshotSerializer.java final ByteBuffer bb = ByteBuffer.wrap(buffer); final int protocolVersion = bb.get(); if (protocolVersion != 1) { throw new SchemaNotFoundException("Schema Encoding appears to be of an incompatible version. The latest known Protocol is Version " + LATEST_PROTOCOL_VERSION + " but the data was encoded with version " + protocolVersion + " or was not encoded with this data format"); } final long schemaId = bb.getLong(); final int schemaVersion = bb.getInt(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().id(schemaId).version(schemaVersion).build(); return schemaRegistry.retrieveSchema(schemaIdentifier); }
@Override public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) { if(descriptor.isDynamic()) { // Dynamic property = schema, validate it if (newValue == null) { recordSchemas.remove(descriptor.getName()); } else { try { // Use a non-strict parser here, a strict parse can be done (if specified) in customValidate(). final Schema avroSchema = new Schema.Parser().setValidate(false).parse(newValue); final SchemaIdentifier schemaId = SchemaIdentifier.builder().name(descriptor.getName()).build(); final RecordSchema recordSchema = AvroTypeUtil.createSchema(avroSchema, newValue, schemaId); recordSchemas.put(descriptor.getName(), recordSchema); } catch (final Exception e) { // not a problem - the service won't be valid and the validation message will indicate what is wrong. } } } }
@Override public RecordSchema getSchema(final Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException, IOException { final byte[] buffer = new byte[5]; try { StreamUtils.fillBuffer(contentStream, buffer); } catch (final IOException ioe) { throw new SchemaNotFoundException("Could not read first 5 bytes from stream", ioe); } // This encoding follows the pattern that is provided for serializing data by the Confluent Schema Registry serializer // as it is provided at: // http://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format // The format consists of the first byte always being 0, to indicate a 'magic byte' followed by 4 bytes // representing the schema id. final ByteBuffer bb = ByteBuffer.wrap(buffer); final int magicByte = bb.get(); if (magicByte != 0) { throw new SchemaNotFoundException("Schema Encoding appears to be of an incompatible version. " + "Expected stream to begin with a Magic Byte of 0 but first byte was " + magicByte); } final int schemaId = bb.getInt(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder() .id(Long.valueOf(schemaId)) .version(1) .build(); return schemaRegistry.retrieveSchema(schemaIdentifier); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves and returns the textual representation of the schema based on * the provided name of the schema available in Schema Registry. * * @return the text that corresponds to the latest version of the schema with the given name * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given name */ default String retrieveSchemaText(String schemaName) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().name(schemaName).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with name '" + schemaName + "'"); } return recordSchema.getSchemaText().get(); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the schema with the given ID and version. The RecordSchema that is returned must have the Schema's identifier and version * populated in its SchemaIdentifier. I.e., a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getIdentifier() getIdentifier()} * will always return an {@link java.util.Optional} that is not empty, as will a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getVersion() getVersion()}. * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the schema with the given ID and version or <code>null</code> if no schema * can be found with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default RecordSchema retrieveSchema(long schemaId, int version) throws IOException, SchemaNotFoundException { return retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the textual representation of the schema with the given ID and version * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the textual representation of the schema with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default String retrieveSchemaText(long schemaId, int version) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'"); } return recordSchema.getSchemaText().get(); }
final int version = Integer.parseInt(schemaVersion); final SchemaIdentifier identifier = SchemaIdentifier.builder().id(schemaId).version(version).build(); final RecordSchema schema = schemaRegistry.retrieveSchema(identifier); if (schema == null) {
final String schemaVersion = schemaVersionPropertyValue.evaluateAttributeExpressions(variables).getValue(); final SchemaIdentifier.Builder identifierBuilder = SchemaIdentifier.builder(); identifierBuilder.name(schemaName);
protected RecordSchema getValidationSchema(final ProcessContext context, final FlowFile flowFile, final RecordReader reader) throws MalformedRecordException, IOException, SchemaNotFoundException { final String schemaAccessStrategy = context.getProperty(SCHEMA_ACCESS_STRATEGY).getValue(); if (schemaAccessStrategy.equals(READER_SCHEMA.getValue())) { return reader.getSchema(); } else if (schemaAccessStrategy.equals(SCHEMA_NAME_PROPERTY.getValue())) { final SchemaRegistry schemaRegistry = context.getProperty(SCHEMA_REGISTRY).asControllerService(SchemaRegistry.class); final String schemaName = context.getProperty(SCHEMA_NAME).evaluateAttributeExpressions(flowFile).getValue(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name(schemaName).build(); return schemaRegistry.retrieveSchema(schemaIdentifier); } else if (schemaAccessStrategy.equals(SCHEMA_TEXT_PROPERTY.getValue())) { final String schemaText = context.getProperty(SCHEMA_TEXT).evaluateAttributeExpressions(flowFile).getValue(); final Parser parser = new Schema.Parser(); final Schema avroSchema = parser.parse(schemaText); return AvroTypeUtil.createSchema(avroSchema); } else { throw new ProcessException("Invalid Schema Access Strategy: " + schemaAccessStrategy); } } }
final SchemaIdentifier resultSchemaIdentifier = SchemaIdentifier.builder() .id(schemaId) .name(schemaName.get())
final SchemaIdentifier resultSchemaIdentifier = SchemaIdentifier.builder() .name(schemaName) .id(schemaId.getAsLong())
private RecordSchema createRecordSchema(final JsonNode schemaNode) throws SchemaNotFoundException { final String subject = schemaNode.get(SUBJECT_FIELD_NAME).asText(); final int version = schemaNode.get(VERSION_FIELD_NAME).asInt(); final int id = schemaNode.get(ID_FIELD_NAME).asInt(); final String schemaText = schemaNode.get(SCHEMA_TEXT_FIELD_NAME).asText(); try { final Schema avroSchema = new Schema.Parser().parse(schemaText); final SchemaIdentifier schemaId = SchemaIdentifier.builder().name(subject).id(Long.valueOf(id)).version(version).build(); final RecordSchema recordSchema = AvroTypeUtil.createSchema(avroSchema, schemaText, schemaId); return recordSchema; } catch (final SchemaParseException spe) { throw new SchemaNotFoundException("Obtained Schema with id " + id + " and name " + subject + " from Confluent Schema Registry but the Schema Text that was returned is not a valid Avro Schema"); } }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves and returns the RecordSchema based on the provided name of the schema available in Schema Registry. The RecordSchema * that is returned must have the Schema's name populated in its SchemaIdentifier. I.e., a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getName() getName()} * will always return an {@link java.util.Optional} that is not empty. * * @return the latest version of the schema with the given name, or <code>null</code> if no schema can be found with the given name. * @throws SchemaNotFoundException if unable to find the schema with the given name */ default RecordSchema retrieveSchema(String schemaName) throws IOException, SchemaNotFoundException { return retrieveSchema(SchemaIdentifier.builder().name(schemaName).build()); }
@Override public RecordSchema getSchema(final Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException, IOException { final byte[] buffer = new byte[13]; try { StreamUtils.fillBuffer(contentStream, buffer); } catch (final IOException ioe) { throw new SchemaNotFoundException("Could not read first 13 bytes from stream", ioe); } // This encoding follows the pattern that is provided for serializing data by the Hortonworks Schema Registry serializer // as it is provided at: // https://github.com/hortonworks/registry/blob/master/schema-registry/serdes/src/main/java/com/hortonworks/registries/schemaregistry/serdes/avro/AvroSnapshotSerializer.java final ByteBuffer bb = ByteBuffer.wrap(buffer); final int protocolVersion = bb.get(); if (protocolVersion != 1) { throw new SchemaNotFoundException("Schema Encoding appears to be of an incompatible version. The latest known Protocol is Version " + LATEST_PROTOCOL_VERSION + " but the data was encoded with version " + protocolVersion + " or was not encoded with this data format"); } final long schemaId = bb.getLong(); final int schemaVersion = bb.getInt(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().id(schemaId).version(schemaVersion).build(); return schemaRegistry.retrieveSchema(schemaIdentifier); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves and returns the textual representation of the schema based on * the provided name of the schema available in Schema Registry. * * @return the text that corresponds to the latest version of the schema with the given name * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given name */ default String retrieveSchemaText(String schemaName) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().name(schemaName).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with name '" + schemaName + "'"); } return recordSchema.getSchemaText().get(); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the schema with the given ID and version. The RecordSchema that is returned must have the Schema's identifier and version * populated in its SchemaIdentifier. I.e., a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getIdentifier() getIdentifier()} * will always return an {@link java.util.Optional} that is not empty, as will a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getVersion() getVersion()}. * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the schema with the given ID and version or <code>null</code> if no schema * can be found with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default RecordSchema retrieveSchema(long schemaId, int version) throws IOException, SchemaNotFoundException { return retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the textual representation of the schema with the given ID and version * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the textual representation of the schema with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default String retrieveSchemaText(long schemaId, int version) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'"); } return recordSchema.getSchemaText().get(); }
protected RecordSchema getValidationSchema(final ProcessContext context, final FlowFile flowFile, final RecordReader reader) throws MalformedRecordException, IOException, SchemaNotFoundException { final String schemaAccessStrategy = context.getProperty(SCHEMA_ACCESS_STRATEGY).getValue(); if (schemaAccessStrategy.equals(READER_SCHEMA.getValue())) { return reader.getSchema(); } else if (schemaAccessStrategy.equals(SCHEMA_NAME_PROPERTY.getValue())) { final SchemaRegistry schemaRegistry = context.getProperty(SCHEMA_REGISTRY).asControllerService(SchemaRegistry.class); final String schemaName = context.getProperty(SCHEMA_NAME).evaluateAttributeExpressions(flowFile).getValue(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name(schemaName).build(); return schemaRegistry.retrieveSchema(schemaIdentifier); } else if (schemaAccessStrategy.equals(SCHEMA_TEXT_PROPERTY.getValue())) { final String schemaText = context.getProperty(SCHEMA_TEXT).evaluateAttributeExpressions(flowFile).getValue(); final Parser parser = new Schema.Parser(); final Schema avroSchema = parser.parse(schemaText); return AvroTypeUtil.createSchema(avroSchema); } else { throw new ProcessException("Invalid Schema Access Strategy: " + schemaAccessStrategy); } } }