private int numRowsInFile(File output) throws Exception { DataFileStream<GenericRecord> reader = new DataFileStream<>( new FileInputStream(output), new GenericDatumReader<>()); Iterator<GenericRecord> rows = reader.iterator(); int rowcount = 0; while(rows.hasNext()) { ++rowcount; rows.next(); } reader.close(); return rowcount; }
@Test public void testStdOut() throws Exception { TestUtil.resetRandomSeed(); run(Arrays.asList(new String[] { "-", "--count", COUNT, "--schema-file", SCHEMA_FILE.toString(), "--seed", Long.toString(SEED) })); byte[] file = out.toByteArray(); DataFileStream<Object> reader = new DataFileStream<Object>(new ByteArrayInputStream(file), new GenericDatumReader<>()); Iterator<Object> found = reader.iterator(); for (Object expected : new RandomData(schemaParser.parse(SCHEMA_FILE), Integer.parseInt(COUNT), SEED)) assertEquals(expected, found.next()); reader.close(); } }
/** * Returns an iterator over the nodes in the file. You must call open first. * * @return A node iterator. */ @Override public Iterator<Node> iterator() { Preconditions.checkState(isOpened()); return mFileReader.iterator(); }
private VariantFileMetadata readMetaData(Configuration conf, URI inputMetaFile) throws IOException { Path from = new Path(inputMetaFile); FileSystem fs = FileSystem.get(conf); DatumReader<VariantFileMetadata> userDatumReader = new SpecificDatumReader<>(VariantFileMetadata.class); VariantFileMetadata variantFileMetadata; if (inputMetaFile.toString().endsWith("json") || inputMetaFile.toString().endsWith("json.gz")) { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true); try (InputStream ids = inputMetaFile.toString().endsWith("json.gz") ? new GZIPInputStream(fs.open(from)) : fs.open(from)) { variantFileMetadata = objectMapper.readValue(ids, VariantFileMetadata.class); } } else { try (FSDataInputStream ids = fs.open(from); DataFileStream<VariantFileMetadata> dataFileReader = new DataFileStream<>(ids, userDatumReader)) { Iterator<VariantFileMetadata> iter = dataFileReader.iterator(); if (!iter.hasNext()) { throw new IllegalStateException(String.format("No Meta data object found in %s !!!", inputMetaFile)); } variantFileMetadata = iter.next(); if (iter.hasNext()) { LOGGER.warn(String.format("More than 1 entry found in metadata file %s", inputMetaFile)); } } } return variantFileMetadata; }