private void closeQuietly(DataFileStream<GenericRecord> stream) { try { if (stream != null) { stream.close(); } } catch (IOException ignored) { } } }
@Override public void close() throws IOException { _avroReader.close(); } }
@Override public void close() throws IOException { dataFileStream.close(); in.close(); }
@Override public void rewind() throws IOException { _avroReader.close(); _avroReader = AvroUtils.getAvroReader(_dataFile); }
private void publish(JsonNode message) throws IOException { if (!keepIndexing) { avroDataStream.close(); avroDataStream = null; return; } KeyedMessage<String, byte[]> data = new KeyedMessage<String, byte[]>("airlineStatsEvents", message.toString().getBytes("UTF-8")); producer.send(data); }
public void readFields(byte[] bytes, Schema writerSchema, Schema readerSchema) throws IOException { fileSchema = writerSchema; record = new GenericData.Record(writerSchema); GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(); gdr.setExpected(readerSchema); ByteArrayInputStream is = new ByteArrayInputStream(bytes); DataFileStream<GenericRecord> dfr = new DataFileStream<GenericRecord>(is, gdr); record = dfr.next(record); dfr.close(); }
public AvroRecordReader(File dataFile, Schema schema) throws IOException { _dataFile = dataFile; _schema = schema; _avroReader = AvroUtils.getAvroReader(dataFile); try { validateSchema(); } catch (Exception e) { _avroReader.close(); throw e; } }
reader.close(); } catch (Exception e) { e.printStackTrace();
private int getFirstIntDatum(File file) throws Exception { DataFileStream<GenericRecord> reader = new DataFileStream<>(new FileInputStream(file), new GenericDatumReader<>()); int result = (Integer) reader.next().get(0); System.out.println(result); reader.close(); return result; }
private int numRowsInFile(File output) throws Exception { DataFileStream<GenericRecord> reader = new DataFileStream<>( new FileInputStream(output), new GenericDatumReader<>()); Iterator<GenericRecord> rows = reader.iterator(); int rowcount = 0; while(rows.hasNext()) { ++rowcount; rows.next(); } reader.close(); return rowcount; }
private GenericData.Record readRecord(Schema schema, byte[] data) throws Exception { ByteArrayInputStream byteStream = new ByteArrayInputStream(data); GenericDatumReader<GenericData.Record> datumReader = new GenericDatumReader<>(schema); DataFileStream<GenericData.Record> reader = new DataFileStream<>(byteStream, datumReader); try { return reader.next(); } finally { reader.close(); } } }
avroDataStream.close(); } catch (IOException e) { logger.error(e.getMessage());
@Override public void init(Map<String, String> props, Schema indexingSchema, String topicName) throws Exception { // Load Avro schema DataFileStream<GenericRecord> reader = AvroUtils.getAvroReader(avroFile); _avroSchema = reader.getSchema(); reader.close(); _rowGenerator = new AvroRecordToPinotRowGenerator(indexingSchema); _reader = new GenericDatumReader<>(_avroSchema); }
out.flush(); } finally { streamReader.close();
@Test public void testStdOut() throws Exception { TestUtil.resetRandomSeed(); run(Arrays.asList(new String[] { "-", "--count", COUNT, "--schema-file", SCHEMA_FILE.toString(), "--seed", Long.toString(SEED) })); byte[] file = out.toByteArray(); DataFileStream<Object> reader = new DataFileStream<Object>(new ByteArrayInputStream(file), new GenericDatumReader<>()); Iterator<Object> found = reader.iterator(); for (Object expected : new RandomData(schemaParser.parse(SCHEMA_FILE), Integer.parseInt(COUNT), SEED)) assertEquals(expected, found.next()); reader.close(); } }
writer.writeTo(outs); outs.close(); reader.close(); return 0;
dataStream.close(); return schema;
public static Schema extractSchemaFromAvroWithoutTime(File avroFile) throws IOException { DataFileStream<GenericRecord> dataStream = new DataFileStream<GenericRecord>(new FileInputStream(avroFile), new GenericDatumReader<GenericRecord>()); Schema schema = new Schema(); for (final Field field : dataStream.getSchema().getFields()) { try { getColumnType(field); } catch (Exception e) { LOGGER.warn("Caught exception while converting Avro field {} of type {}, field will not be in schema.", field.name(), field.schema().getType()); continue; } final String columnName = field.name(); final String pinotType = field.getProp("pinotType"); final FieldSpec fieldSpec; if (pinotType != null && "METRIC".equals(pinotType)) { fieldSpec = new MetricFieldSpec(); } else { fieldSpec = new DimensionFieldSpec(); } fieldSpec.setName(columnName); fieldSpec.setDataType(getColumnType(dataStream.getSchema().getField(columnName))); fieldSpec.setSingleValueField(isSingleValueField(dataStream.getSchema().getField(columnName))); schema.addField(fieldSpec); } dataStream.close(); return schema; }
@Test public void testWrite() throws IOException { Schema writerSchema = Schema.create(Schema.Type.INT); GenericData dataModel = new ReflectData(); CodecFactory compressionCodec = CodecFactory.nullCodec(); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); TaskAttemptContext context = createMock(TaskAttemptContext.class); replay(context); // Write an avro container file with two records: 1 and 2. AvroKeyRecordWriter<Integer> recordWriter = new AvroKeyRecordWriter<>( writerSchema, dataModel, compressionCodec, outputStream); recordWriter.write(new AvroKey<>(1), NullWritable.get()); recordWriter.write(new AvroKey<>(2), NullWritable.get()); recordWriter.close(context); verify(context); // Verify that the file was written as expected. InputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); Schema readerSchema = Schema.create(Schema.Type.INT); DatumReader<Integer> datumReader = new SpecificDatumReader<>(readerSchema); DataFileStream<Integer> dataFileReader = new DataFileStream<>(inputStream, datumReader); assertTrue(dataFileReader.hasNext()); // Record 1. assertEquals(1, dataFileReader.next().intValue()); assertTrue(dataFileReader.hasNext()); // Record 2. assertEquals(2, dataFileReader.next().intValue()); assertFalse(dataFileReader.hasNext()); // No more records. dataFileReader.close(); }