/** Return the value of a metadata property. */ public String getMetaString(String key) { byte[] value = getMeta(key); if (value == null) { return null; } return new String(value, StandardCharsets.UTF_8); } /** Return the value of a metadata property. */
@Override public void init(final DataFileStream<GenericRecord> reader, final String codec, final OutputStream out) throws IOException { writer = new DataFileWriter<>(new GenericDatumWriter<GenericRecord>()); if (transferMetadata) { for (String metaKey : reader.getMetaKeys()) { if (!RESERVED_METADATA.contains(metaKey)) { writer.setMeta(metaKey, reader.getMeta(metaKey)); } } } writer.setCodec(CodecFactory.fromString(codec)); writer.create(reader.getSchema(), out); }
public static void checkMeta(DataFileStream<?> in) throws Exception { assertEquals(STRING_META_VALUE, in.getMetaString(STRING_KEY)); assertEquals(LONG_META_VALUE, in.getMetaLong(LONG_KEY)); assertTrue(Arrays.equals(BYTES_META_VALUE, in.getMeta(BYTES_KEY))); }
for (String key : reader.getMetaKeys()) { if (!DataFileWriter.isReservedMeta(key)) { byte[] metadatum = reader.getMeta(key); metadata.put(key, metadatum); writer.setMeta(key, metadatum); byte[] metadatum = reader.getMeta(key); byte[] writersMetadatum = metadata.get(key); if(!Arrays.equals(metadatum, writersMetadatum)) {
for (String key : reader.getMetaKeys()) { if (!DataFileWriter.isReservedMeta(key)) { writer.setMeta(key, reader.getMeta(key));
@Test @SuppressWarnings("unchecked") @Category(NeedsRunner.class) public void testMetadata() throws Exception { List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); File outputFile = tmpFolder.newFile("output.avro"); writePipeline .apply(Create.of(values)) .apply( AvroIO.write(GenericClass.class) .to(outputFile.getAbsolutePath()) .withoutSharding() .withMetadata( ImmutableMap.of( "stringKey", "stringValue", "longKey", 100L, "bytesKey", "bytesValue".getBytes(Charsets.UTF_8)))); writePipeline.run(); try (DataFileStream dataFileStream = new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) { assertEquals("stringValue", dataFileStream.getMetaString("stringKey")); assertEquals(100L, dataFileStream.getMetaLong("longKey")); assertArrayEquals("bytesValue".getBytes(Charsets.UTF_8), dataFileStream.getMeta("bytesKey")); } }
private byte[] initAvroWriter(ProcessSession session, String codec, DataFileStream<GenericRecord> reader, DataFileWriter<GenericRecord> writer, AtomicReference<FlowFile> flowFileRef) { writer.setCodec(CodecFactory.fromString(codec)); // Transfer metadata (this is a subset of the incoming file) for (String metaKey : reader.getMetaKeys()) { if (!RESERVED_METADATA.contains(metaKey)) { writer.setMeta(metaKey, reader.getMeta(metaKey)); } } final ByteArrayOutputStream avroHeader = new ByteArrayOutputStream(); flowFileRef.set(session.append(flowFileRef.get(), (out) -> { // Create writer so that records can be appended later. writer.create(reader.getSchema(), avroHeader); writer.close(); final byte[] header = avroHeader.toByteArray(); out.write(header); })); // Capture the Avro header byte array that is just written to the FlowFile. // This is needed when Avro records are appended to the same FlowFile. return avroHeader.toByteArray(); }
/** Return the value of a metadata property. */ public String getMetaString(String key) { byte[] value = getMeta(key); if (value == null) { return null; } try { return new String(value, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } /** Return the value of a metadata property. */
for (String key : reader.getMetaKeys()) { if (!DataFileWriter.isReservedMeta(key)) { writer.setMeta(key, reader.getMeta(key));
for (String key : reader.getMetaKeys()) { if (!DataFileWriter.isReservedMeta(key)) { writer.setMeta(key, reader.getMeta(key));
/** Return the value of a metadata property. */ public String getMetaString(String key) { byte[] value = getMeta(key); if (value == null) { return null; } try { return new String(value, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } /** Return the value of a metadata property. */
for (String key : reader.getMetaKeys()) { if (!DataFileWriter.isReservedMeta(key)) { writer.setMeta(key, reader.getMeta(key));
private byte[] initAvroWriter(ProcessSession session, String codec, DataFileStream<GenericRecord> reader, DataFileWriter<GenericRecord> writer, AtomicReference<FlowFile> flowFileRef) { writer.setCodec(CodecFactory.fromString(codec)); // Transfer metadata (this is a subset of the incoming file) for (String metaKey : reader.getMetaKeys()) { if (!RESERVED_METADATA.contains(metaKey)) { writer.setMeta(metaKey, reader.getMeta(metaKey)); } } final ByteArrayOutputStream avroHeader = new ByteArrayOutputStream(); flowFileRef.set(session.append(flowFileRef.get(), (out) -> { // Create writer so that records can be appended later. writer.create(reader.getSchema(), avroHeader); writer.close(); final byte[] header = avroHeader.toByteArray(); out.write(header); })); // Capture the Avro header byte array that is just written to the FlowFile. // This is needed when Avro records are appended to the same FlowFile. return avroHeader.toByteArray(); }
/** Return the value of a metadata property. */ public String getMetaString(String key) { byte[] value = getMeta(key); if (value == null) { return null; } try { return new String(value, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } /** Return the value of a metadata property. */
for (String key : reader.getMetaKeys()) { if (!DataFileWriter.isReservedMeta(key)) { byte[] metadatum = reader.getMeta(key); metadata.put(key, metadatum); writer.setMeta(key, metadatum); byte[] metadatum = reader.getMeta(key); byte[] writersMetadatum = metadata.get(key); if(!Arrays.equals(metadatum, writersMetadatum)) {
for (String key : reader.getMetaKeys()) { if (!DataFileWriter.isReservedMeta(key)) { writer.setMeta(key, reader.getMeta(key));
/** Return the value of a metadata property. */ public String getMetaString(String key) { byte[] value = getMeta(key); if (value == null) { return null; } try { return new String(value, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } /** Return the value of a metadata property. */
for (String key : reader.getMetaKeys()) { if (!DataFileWriter.isReservedMeta(key)) { writer.setMeta(key, reader.getMeta(key));
/** Return the value of a metadata property. */ public String getMetaString(String key) { byte[] value = getMeta(key); if (value == null) { return null; } try { return new String(value, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } /** Return the value of a metadata property. */