identifierBuilder.name(schemaName); identifierBuilder.branch(schemaBranchName); identifierBuilder.version(Integer.valueOf(schemaVersion)); } catch (NumberFormatException nfe) { throw new SchemaNotFoundException("Could not retrieve schema with name '" + schemaName final RecordSchema recordSchema = schemaRegistry.retrieveSchema(identifierBuilder.build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find a schema with name '" + schemaName + "' in the configured Schema Registry");
@Override public RecordSchema getSchema(final Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException, IOException { final byte[] buffer = new byte[5]; try { StreamUtils.fillBuffer(contentStream, buffer); } catch (final IOException ioe) { throw new SchemaNotFoundException("Could not read first 5 bytes from stream", ioe); } // This encoding follows the pattern that is provided for serializing data by the Confluent Schema Registry serializer // as it is provided at: // http://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format // The format consists of the first byte always being 0, to indicate a 'magic byte' followed by 4 bytes // representing the schema id. final ByteBuffer bb = ByteBuffer.wrap(buffer); final int magicByte = bb.get(); if (magicByte != 0) { throw new SchemaNotFoundException("Schema Encoding appears to be of an incompatible version. " + "Expected stream to begin with a Magic Byte of 0 but first byte was " + magicByte); } final int schemaId = bb.getInt(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder() .id(Long.valueOf(schemaId)) .version(1) .build(); return schemaRegistry.retrieveSchema(schemaIdentifier); }
@Override public RecordSchema getSchema(final Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException, IOException { final byte[] buffer = new byte[13]; try { StreamUtils.fillBuffer(contentStream, buffer); } catch (final IOException ioe) { throw new SchemaNotFoundException("Could not read first 13 bytes from stream", ioe); } // This encoding follows the pattern that is provided for serializing data by the Hortonworks Schema Registry serializer // as it is provided at: // https://github.com/hortonworks/registry/blob/master/schema-registry/serdes/src/main/java/com/hortonworks/registries/schemaregistry/serdes/avro/AvroSnapshotSerializer.java final ByteBuffer bb = ByteBuffer.wrap(buffer); final int protocolVersion = bb.get(); if (protocolVersion != 1) { throw new SchemaNotFoundException("Schema Encoding appears to be of an incompatible version. The latest known Protocol is Version " + LATEST_PROTOCOL_VERSION + " but the data was encoded with version " + protocolVersion + " or was not encoded with this data format"); } final long schemaId = bb.getLong(); final int schemaVersion = bb.getInt(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().id(schemaId).version(schemaVersion).build(); return schemaRegistry.retrieveSchema(schemaIdentifier); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the schema with the given ID and version. The RecordSchema that is returned must have the Schema's identifier and version * populated in its SchemaIdentifier. I.e., a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getIdentifier() getIdentifier()} * will always return an {@link java.util.Optional} that is not empty, as will a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getVersion() getVersion()}. * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the schema with the given ID and version or <code>null</code> if no schema * can be found with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default RecordSchema retrieveSchema(long schemaId, int version) throws IOException, SchemaNotFoundException { return retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the textual representation of the schema with the given ID and version * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the textual representation of the schema with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default String retrieveSchemaText(long schemaId, int version) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'"); } return recordSchema.getSchemaText().get(); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves and returns the textual representation of the schema based on * the provided name of the schema available in Schema Registry. * * @return the text that corresponds to the latest version of the schema with the given name * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given name */ default String retrieveSchemaText(String schemaName) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().name(schemaName).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with name '" + schemaName + "'"); } return recordSchema.getSchemaText().get(); }
protected RecordSchema getValidationSchema(final ProcessContext context, final FlowFile flowFile, final RecordReader reader) throws MalformedRecordException, IOException, SchemaNotFoundException { final String schemaAccessStrategy = context.getProperty(SCHEMA_ACCESS_STRATEGY).getValue(); if (schemaAccessStrategy.equals(READER_SCHEMA.getValue())) { return reader.getSchema(); } else if (schemaAccessStrategy.equals(SCHEMA_NAME_PROPERTY.getValue())) { final SchemaRegistry schemaRegistry = context.getProperty(SCHEMA_REGISTRY).asControllerService(SchemaRegistry.class); final String schemaName = context.getProperty(SCHEMA_NAME).evaluateAttributeExpressions(flowFile).getValue(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name(schemaName).build(); return schemaRegistry.retrieveSchema(schemaIdentifier); } else if (schemaAccessStrategy.equals(SCHEMA_TEXT_PROPERTY.getValue())) { final String schemaText = context.getProperty(SCHEMA_TEXT).evaluateAttributeExpressions(flowFile).getValue(); final Parser parser = new Schema.Parser(); final Schema avroSchema = parser.parse(schemaText); return AvroTypeUtil.createSchema(avroSchema); } else { throw new ProcessException("Invalid Schema Access Strategy: " + schemaAccessStrategy); } } }
private RecordSchema createRecordSchema(final JsonNode schemaNode) throws SchemaNotFoundException { final String subject = schemaNode.get(SUBJECT_FIELD_NAME).asText(); final int version = schemaNode.get(VERSION_FIELD_NAME).asInt(); final int id = schemaNode.get(ID_FIELD_NAME).asInt(); final String schemaText = schemaNode.get(SCHEMA_TEXT_FIELD_NAME).asText(); try { final Schema avroSchema = new Schema.Parser().parse(schemaText); final SchemaIdentifier schemaId = SchemaIdentifier.builder().name(subject).id(Long.valueOf(id)).version(version).build(); final RecordSchema recordSchema = AvroTypeUtil.createSchema(avroSchema, schemaText, schemaId); return recordSchema; } catch (final SchemaParseException spe) { throw new SchemaNotFoundException("Obtained Schema with id " + id + " and name " + subject + " from Confluent Schema Registry but the Schema Text that was returned is not a valid Avro Schema"); } }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves and returns the RecordSchema based on the provided name of the schema available in Schema Registry. The RecordSchema * that is returned must have the Schema's name populated in its SchemaIdentifier. I.e., a call to * {@link RecordSchema}.{@link RecordSchema#getIdentifier() getIdentifier()}.{@link SchemaIdentifier#getName() getName()} * will always return an {@link java.util.Optional} that is not empty. * * @return the latest version of the schema with the given name, or <code>null</code> if no schema can be found with the given name. * @throws SchemaNotFoundException if unable to find the schema with the given name */ default RecordSchema retrieveSchema(String schemaName) throws IOException, SchemaNotFoundException { return retrieveSchema(SchemaIdentifier.builder().name(schemaName).build()); }
public static RecordSchema createSchema(final Schema avroSchema, final boolean includeText) { if (avroSchema == null) { throw new IllegalArgumentException("Avro Schema cannot be null"); } SchemaIdentifier identifier = new StandardSchemaIdentifier.Builder().name(avroSchema.getName()).build(); return createSchema(avroSchema, includeText ? avroSchema.toString() : null, identifier); }
identifierBuilder.name(schemaName); identifierBuilder.branch(schemaBranchName); identifierBuilder.version(Integer.valueOf(schemaVersion)); } catch (NumberFormatException nfe) { throw new SchemaNotFoundException("Could not retrieve schema with name '" + schemaName final RecordSchema recordSchema = schemaRegistry.retrieveSchema(identifierBuilder.build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find a schema with name '" + schemaName + "' in the configured Schema Registry");
@Override public RecordSchema getSchema(final Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException, IOException { final byte[] buffer = new byte[5]; try { StreamUtils.fillBuffer(contentStream, buffer); } catch (final IOException ioe) { throw new SchemaNotFoundException("Could not read first 5 bytes from stream", ioe); } // This encoding follows the pattern that is provided for serializing data by the Confluent Schema Registry serializer // as it is provided at: // http://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format // The format consists of the first byte always being 0, to indicate a 'magic byte' followed by 4 bytes // representing the schema id. final ByteBuffer bb = ByteBuffer.wrap(buffer); final int magicByte = bb.get(); if (magicByte != 0) { throw new SchemaNotFoundException("Schema Encoding appears to be of an incompatible version. " + "Expected stream to begin with a Magic Byte of 0 but first byte was " + magicByte); } final int schemaId = bb.getInt(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder() .id(Long.valueOf(schemaId)) .version(1) .build(); return schemaRegistry.retrieveSchema(schemaIdentifier); }
@Override public RecordSchema getSchema(final Map<String, String> variables, final InputStream contentStream, final RecordSchema readSchema) throws SchemaNotFoundException, IOException { final byte[] buffer = new byte[13]; try { StreamUtils.fillBuffer(contentStream, buffer); } catch (final IOException ioe) { throw new SchemaNotFoundException("Could not read first 13 bytes from stream", ioe); } // This encoding follows the pattern that is provided for serializing data by the Hortonworks Schema Registry serializer // as it is provided at: // https://github.com/hortonworks/registry/blob/master/schema-registry/serdes/src/main/java/com/hortonworks/registries/schemaregistry/serdes/avro/AvroSnapshotSerializer.java final ByteBuffer bb = ByteBuffer.wrap(buffer); final int protocolVersion = bb.get(); if (protocolVersion != 1) { throw new SchemaNotFoundException("Schema Encoding appears to be of an incompatible version. The latest known Protocol is Version " + LATEST_PROTOCOL_VERSION + " but the data was encoded with version " + protocolVersion + " or was not encoded with this data format"); } final long schemaId = bb.getLong(); final int schemaVersion = bb.getInt(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().id(schemaId).version(schemaVersion).build(); return schemaRegistry.retrieveSchema(schemaIdentifier); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the textual representation of the schema with the given ID and version * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the textual representation of the schema with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default String retrieveSchemaText(long schemaId, int version) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'"); } return recordSchema.getSchemaText().get(); }
protected RecordSchema getValidationSchema(final ProcessContext context, final FlowFile flowFile, final RecordReader reader) throws MalformedRecordException, IOException, SchemaNotFoundException { final String schemaAccessStrategy = context.getProperty(SCHEMA_ACCESS_STRATEGY).getValue(); if (schemaAccessStrategy.equals(READER_SCHEMA.getValue())) { return reader.getSchema(); } else if (schemaAccessStrategy.equals(SCHEMA_NAME_PROPERTY.getValue())) { final SchemaRegistry schemaRegistry = context.getProperty(SCHEMA_REGISTRY).asControllerService(SchemaRegistry.class); final String schemaName = context.getProperty(SCHEMA_NAME).evaluateAttributeExpressions(flowFile).getValue(); final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name(schemaName).build(); return schemaRegistry.retrieveSchema(schemaIdentifier); } else if (schemaAccessStrategy.equals(SCHEMA_TEXT_PROPERTY.getValue())) { final String schemaText = context.getProperty(SCHEMA_TEXT).evaluateAttributeExpressions(flowFile).getValue(); final Parser parser = new Schema.Parser(); final Schema avroSchema = parser.parse(schemaText); return AvroTypeUtil.createSchema(avroSchema); } else { throw new ProcessException("Invalid Schema Access Strategy: " + schemaAccessStrategy); } } }
static RecordSchema createRecordSchema() { final List<RecordField> fields = new ArrayList<>(); fields.add(new RecordField(SyslogAttributes.PRIORITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.SEVERITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.FACILITY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.VERSION.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.TIMESTAMP.key(), RecordFieldType.TIMESTAMP.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.HOSTNAME.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(SyslogAttributes.BODY.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.APP_NAME.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.PROCID.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.MESSAGEID.key(), RecordFieldType.STRING.getDataType(), true)); fields.add(new RecordField(Syslog5424Attributes.STRUCTURED_BASE.key(), RecordFieldType.MAP.getMapDataType(RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType())))); SchemaIdentifier schemaIdentifier = new StandardSchemaIdentifier.Builder().name(RFC_5424_SCHEMA_NAME).build(); final RecordSchema schema = new SimpleRecordSchema(fields,schemaIdentifier); return schema; }