/** {@inheritDoc} */ @Override public void close() throws IOException { reader.close(); }
public void close(Reporter reporter) throws IOException { flush(); } };
@Test public void testTrevniEvolvedRead() throws IOException { AvroColumnWriter<GenericRecord> acw = new AvroColumnWriter<>(writer, new ColumnFileMetaData()); acw.write(writtenRecord); File serializedTrevni = File.createTempFile("trevni", null); acw.writeTo(serializedTrevni); AvroColumnReader.Params params = new Params(serializedTrevni); params.setSchema(evolved); AvroColumnReader<GenericRecord> acr = new AvroColumnReader<>(params); GenericRecord readRecord = acr.next(); Assert.assertEquals(evolvedRecord, readRecord); Assert.assertFalse(acr.hasNext()); }
private void checkRead(Schema s, List<Object> data) throws Exception { AvroColumnReader<Object> reader = new AvroColumnReader<>(new AvroColumnReader.Params(FILE) .setSchema(s)); try { for (Object datum : data) assertEquals(datum, reader.next()); } finally { reader.close(); } }
/** {@inheritDoc} */ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { final FileSplit file = (FileSplit)inputSplit; context.setStatus(file.toString()); final AvroColumnReader.Params params = new AvroColumnReader.Params(new HadoopInput(file.getPath(), context.getConfiguration())); params.setModel(ReflectData.get()); if (AvroJob.getInputKeySchema(context.getConfiguration()) != null) { params.setSchema(AvroJob.getInputKeySchema(context.getConfiguration())); } reader = new AvroColumnReader<>(params); rows = reader.getRowCount(); }
public void validateCountsFileGenericRecord() throws Exception { AvroColumnReader<GenericRecord > reader = new AvroColumnReader<> ( new AvroColumnReader.Params(countFiles).setModel(SpecificData.get())); int numWords = 0; for (GenericRecord wc : reader) { assertEquals((String)wc.get("key"), COUNTS.get(wc.get("key")), (Long)wc.get("value")); //assertEquals(wc.getKey(), COUNTS.get(wc.getKey()), wc.getValue()); numWords++; } reader.close(); assertEquals(COUNTS.size(), numWords); }
private void checkWrite(Schema schema) throws IOException { AvroColumnWriter<Object> writer = new AvroColumnWriter<>(schema, new ColumnFileMetaData()); int count = 0; for (Object datum : new RandomData(schema, COUNT, SEED)) { //System.out.println("datum="+datum); writer.write(datum); } writer.writeTo(FILE); }
public void write(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { writer.write(wrapper.datum()); if (writer.sizeEstimate() >= blockSize) // block full flush(); } public void close(Reporter reporter) throws IOException {
private void check(Schema s, ColumnMetaData... expected) throws Exception { ColumnMetaData[] shredded = new AvroColumnator(s).getColumns(); assertEquals(expected.length, shredded.length); for (int i = 0; i < expected.length; i++) assertEquals(expected[i].toString(), shredded[i].toString()); checkWrite(s); checkRead(s); }
/** {@inheritDoc} */ @Override public void write(AvroKey<T> key, NullWritable value) throws IOException, InterruptedException { writer.write(key.datum()); if (writer.sizeEstimate() >= blockSize) // block full flush(); }
/** Add a row to the file. */ public void write(D value) throws IOException { writer.startRow(); int count = write(value, schema, 0); assert(count == writer.getColumnCount()); writer.endRow(); }
@Test public void runTestsInOrder() throws Exception { testOutputFormat(); testInputFormat(); }
@Test public void testDefaultValue() throws Exception { String s = "{\"type\":\"record\",\"name\":\"R\",\"fields\":[" +SIMPLE_FIELDS+"," +"{\"name\":\"z\",\"type\":\"int\"," +"\"default\":1,\""+RandomData.USE_DEFAULT+"\":true}" +"]}"; checkWrite(Schema.parse(SIMPLE_RECORD)); checkRead(Schema.parse(s)); }
public AvroColumnator(Schema schema) { this.schema = schema; columnize(null, schema, null, false); }
private String p(String parent, Schema child, String sep) { if (child.getType() == Schema.Type.UNION) return parent; return p(parent, child.getFullName(), sep); }
private void checkRead(Schema schema) throws IOException { AvroColumnReader<Object> reader = new AvroColumnReader<>(new AvroColumnReader.Params(FILE) .setSchema(schema)); for (Object expected : new RandomData(schema, COUNT, SEED)) assertEquals(expected, reader.next()); reader.close(); }
public void validateCountsFile() throws Exception { AvroColumnReader<Pair<String,Long>> reader = new AvroColumnReader<>( new AvroColumnReader.Params(countFiles).setModel(SpecificData.get())); int numWords = 0; for (Pair<String,Long> wc : reader) { assertEquals(wc.key(), COUNTS.get(wc.key()), wc.value()); numWords++; } reader.close(); assertEquals(COUNTS.size(), numWords); }
public void close() throws IOException { reader.close(); }