@Override public void handle(Event event, Throwable cause) throws EventDeliveryException { try { if (writer == null) { writer = dataset.newWriter(); } final AvroFlumeEvent avroEvent = new AvroFlumeEvent(); avroEvent.setBody(ByteBuffer.wrap(event.getBody())); avroEvent.setHeaders(toCharSeqMap(event.getHeaders())); writer.write(avroEvent); nEventsHandled++; } catch (RuntimeException ex) { throw new EventDeliveryException(ex); } }
@Override public void process(InputStream in) throws IOException { try (DataFileStream<Record> stream = new DataFileStream<>( in, AvroUtil.newDatumReader(schema, Record.class))) { IncompatibleSchemaException.check( SchemaValidationUtil.canRead(stream.getSchema(), schema), "Incompatible file schema %s, expected %s", stream.getSchema(), schema); long written = 0L; try (DatasetWriter<Record> writer = target.newWriter()) { for (Record record : stream) { writer.write(record); written += 1; } } finally { session.adjustCounter("Stored records", written, true /* cannot roll back the write */); } } } });
this.writer = view.newWriter();
@Override public void run() { smallEvents.newWriter(); } });
@Override public void run() { smallEvents.newWriter(); } });
@Override public void run() { smallEvents.newWriter(); } });
@Override public int run(List<String> args) throws Exception { Preconditions.checkState(!Datasets.exists(uri), "events dataset already exists"); DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schema(StandardEvent.class).build(); View<StandardEvent> events = Datasets.create(uri, descriptor, StandardEvent.class); DatasetWriter<StandardEvent> writer = events.newWriter(); try { while (System.currentTimeMillis() - baseTimestamp < 36000) { writer.write(generateRandomEvent()); } } finally { writer.close(); } System.out.println("Generated " + counter + " events"); return 0; }
public void writeUserToView(View<GenericRecord> dataset) { DatasetWriter<GenericRecord> writer = null; try { writer = dataset.newWriter(); writer.write(USER); } finally { if (writer != null) { writer.close(); } } }
public DatasetRecordWriter(View<E> view, boolean copyRecords) { this.datasetWriter = view.newWriter(); this.schema = view.getDataset().getDescriptor().getSchema(); this.dataModel = DataModelUtil.getDataModelForType( view.getType()); this.copyRecords = copyRecords; }
public DatasetRecordWriter(View<E> view, boolean copyRecords) { this.datasetWriter = view.newWriter(); this.schema = view.getDataset().getDescriptor().getSchema(); this.dataModel = DataModelUtil.getDataModelForType( view.getType()); this.copyRecords = copyRecords; }
private DatasetWriter<GenericRecord> newWriter( final UserGroupInformation login, final URI uri) { View<GenericRecord> view = KerberosUtil.runPrivileged(login, new PrivilegedExceptionAction<Dataset<GenericRecord>>() { @Override public Dataset<GenericRecord> run() { return Datasets.load(uri); } }); DatasetDescriptor descriptor = view.getDataset().getDescriptor(); String formatName = descriptor.getFormat().getName(); Preconditions.checkArgument(allowedFormats().contains(formatName), "Unsupported format: " + formatName); Schema newSchema = descriptor.getSchema(); if (targetSchema == null || !newSchema.equals(targetSchema)) { this.targetSchema = descriptor.getSchema(); // target dataset schema has changed, invalidate all readers based on it readers.invalidateAll(); } this.reuseDatum = !("parquet".equals(formatName)); this.datasetName = view.getDataset().getName(); return view.newWriter(); }
@Override public void handle(Event event, Throwable cause) throws EventDeliveryException { try { if (writer == null) { writer = dataset.newWriter(); } final AvroFlumeEvent avroEvent = new AvroFlumeEvent(); avroEvent.setBody(ByteBuffer.wrap(event.getBody())); avroEvent.setHeaders(toCharSeqMap(event.getHeaders())); writer.write(avroEvent); nEventsHandled++; } catch (RuntimeException ex) { throw new EventDeliveryException(ex); } }
@Test public void testReaderWriterCompatibleSchema() throws IOException { DatasetWriter<TestValue> writer = null; try { writer = Datasets.load(testValueView.getUri(), TestValue.class).newWriter(); writer.write(testValue); } finally { Closeables.close(writer, false); } DatasetReader<Value> reader = null; try { reader = valueView.newReader(); Assert.assertEquals(Sets.newHashSet(defaultValue), Sets.newHashSet((Iterable<Value>) reader)); } finally { Closeables.close(reader, false); } }
private static <E> void writeToView(View<E> view, E... entities) { DatasetWriter<E> writer = null; try { writer = view.newWriter(); for (E entity : entities) { writer.write(entity); } writer.close(); } finally { if (writer != null) { writer.close(); } } }
private static void writeTestRecords(View<TestRecord> view) { DatasetWriter<TestRecord> writer = null; try { writer = view.newWriter(); for (int i = 0; i < 10; i += 1) { TestRecord record = new TestRecord(); record.id = i; record.data = "test/-" + i; writer.write(record); } } finally { if (writer != null) { writer.close(); } } } }
private static void writeTestRecords(View<TestRecord> view) { DatasetWriter<TestRecord> writer = null; try { writer = view.newWriter(); for (int i = 0; i < 10; i += 1) { TestRecord record = new TestRecord(); record.id = i; record.data = "test-" + i; writer.write(record); } } finally { if (writer != null) { writer.close(); } } } }
@Override public void process(InputStream in) throws IOException { try (DataFileStream<Record> stream = new DataFileStream<>( in, AvroUtil.newDatumReader(schema, Record.class))) { IncompatibleSchemaException.check( SchemaValidationUtil.canRead(stream.getSchema(), schema), "Incompatible file schema %s, expected %s", stream.getSchema(), schema); long written = 0L; try (DatasetWriter<Record> writer = target.newWriter()) { for (Record record : stream) { writer.write(record); written += 1; } } finally { session.adjustCounter("Stored records", written, true /* cannot roll back the write */); } } } });
@Test(expected = IllegalArgumentException.class) public void testInvalidLimitedWriter() { final View<TestEntity> range = ds .fromAfter(NAMES[0], "1").to(NAMES[0], "5") .fromAfter(NAMES[1], "1").to(NAMES[1], "5"); range.newWriter().write(newTestEntity("6", "6")); }
@Test public void testLimitedWriter() { final View<TestEntity> range = ds .fromAfter(NAMES[0], "1").to(NAMES[0], "5") .fromAfter(NAMES[1], "1").to(NAMES[1], "5"); DatasetWriter<TestEntity> writer = range.newWriter(); try { writer.write(newTestEntity("3", "3")); writer.write(newTestEntity("5", "5")); } finally { writer.close(); } }
public static void writeTestUsers(View<GenericData.Record> view, int count, int start, String... fields) { DatasetWriter<GenericData.Record> writer = null; try { writer = view.newWriter(); for (int i = start; i < count + start; i++) { GenericRecordBuilder recordBuilder = new GenericRecordBuilder(view.getDataset().getDescriptor ().getSchema()).set("username", "test-" + i); for (String field : fields) { recordBuilder.set(field, field + "-" + i); } writer.write(recordBuilder.build()); } if (writer instanceof Flushable) { ((Flushable) writer).flush(); } } finally { if (writer != null) { writer.close(); } } }