@DefaultCoder(AvroCoder.class) static class Singer { long singerId; String firstName; String lastName; Singer() {} Singer(long singerId, String firstName, String lastName) { this.singerId = singerId; this.firstName = firstName; this.lastName = lastName; } }
@Override public void encode(ReadableDuration value, OutputStream outStream) throws CoderException, IOException { if (value == null) { throw new CoderException("cannot encode a null ReadableDuration"); } LONG_CODER.encode(toLong(value), outStream); }
@Override public TypeDescriptor<Set<T>> getEncodedTypeDescriptor() { return new TypeDescriptor<Set<T>>() {}.where( new TypeParameter<T>() {}, getElemCoder().getEncodedTypeDescriptor()); }
@Override public void encode(String value, OutputStream outStream) throws CoderException, IOException { throwIfPresent(encodingException); StringUtf8Coder.of().encode(value, outStream); }
@Override public String decode(InputStream inStream) throws CoderException, IOException { throwIfPresent(decodingException); return StringUtf8Coder.of().decode(inStream); }
@Override public void verifyDeterministic() { VarIntCoder.of().verifyDeterministic(); VarLongCoder.of().verifyDeterministic(); }
@Override public void encode(OutgoingMessage value, OutputStream outStream) throws CoderException, IOException { ByteArrayCoder.of().encode(value.elementBytes, outStream); ATTRIBUTES_CODER.encode(value.attributes, outStream); BigEndianLongCoder.of().encode(value.timestampMsSinceEpoch, outStream); RECORD_ID_CODER.encode(value.recordId, outStream); }
public static void runCsvToAvro(SampleOptions options) throws IOException, IllegalArgumentException { FileSystems.setDefaultPipelineOptions(options); // Get Avro Schema String schemaJson = getSchema(options.getAvroSchema()); Schema schema = new Schema.Parser().parse(schemaJson); // Check schema field types before starting the Dataflow job checkFieldTypes(schema); // Create the Pipeline object with the options we defined above. Pipeline pipeline = Pipeline.create(options); // Convert CSV to Avro pipeline.apply("Read CSV files", TextIO.read().from(options.getInputFile())) .apply("Convert CSV to Avro formatted data", ParDo.of(new ConvertCsvToAvro(schemaJson, options.getCsvDelimiter()))) .setCoder(AvroCoder.of(GenericRecord.class, schema)) .apply("Write Avro formatted data", AvroIO.writeGenericRecords(schemaJson) .to(options.getOutput()).withCodec(CodecFactory.snappyCodec()).withSuffix(".avro")); // Run the pipeline. pipeline.run().waitUntilFinish(); }
@Override public void encode(T value, OutputStream outStream, Context context) throws CoderException, IOException { coder.encode(applyAndWrapExceptions(toFn, value), outStream, context); }
@Override public TypeDescriptor<Iterable<T>> getEncodedTypeDescriptor() { return new TypeDescriptor<Iterable<T>>() {}.where( new TypeParameter<T>() {}, getElemCoder().getEncodedTypeDescriptor()); } }
@Override public void verifyDeterministic() throws NonDeterministicException { INTERVAL_WINDOW_CODER.verifyDeterministic(); VAR_INT_CODER.verifyDeterministic(); } }
@Override public TypeDescriptor<Collection<T>> getEncodedTypeDescriptor() { return new TypeDescriptor<Collection<T>>() {}.where( new TypeParameter<T>() {}, getElemCoder().getEncodedTypeDescriptor()); } }
private static <T> AnyCoderAndData coderAndData(Coder<T> coder, List<T> data) { CoderAndData<T> coderAndData = new CoderAndData<>(); coderAndData.coder = coder; coderAndData.data = data; AnyCoderAndData res = new AnyCoderAndData(); res.coderAndData = coderAndData; return res; }
@DefaultCoder(AvroCoder.class) static class Album { long singerId; long albumId; String albumTitle; Album() {} Album(long singerId, long albumId, String albumTitle) { this.singerId = singerId; this.albumId = albumId; this.albumTitle = albumTitle; } }
@DefaultCoder(AvroCoder.class) static class Singer { long singerId; String firstName; String lastName; Singer() { } Singer(long singerId, String firstName, String lastName) { this.singerId = singerId; this.firstName = firstName; this.lastName = lastName; } }
@DefaultCoder(AvroCoder.class) static class Album { long singerId; long albumId; String albumTitle; Album() { } Album(long singerId, long albumId, String albumTitle) { this.singerId = singerId; this.albumId = albumId; this.albumTitle = albumTitle; } }