@Override public boolean readTo(T model) throws IOException { RecordReader<Object> reader = prepareReader(model); if (reader == null) { return false; } rowRest--; reader.read(); advanceCounter(); return true; }
public boolean nextKeyValue() throws IOException, InterruptedException { boolean recordFound = false; while (!recordFound) { // no more records left if (current >= total) { return false; } try { checkRead(); currentValue = recordReader.read(); current ++; if (recordReader.shouldSkipCurrentRecord()) { // this record is being filtered via the filter2 package if (DEBUG) LOG.debug("skipping record"); continue; } if (currentValue == null) { // only happens with FilteredRecordReader at end of block current = totalCountLoadedSoFar; if (DEBUG) LOG.debug("filtered record reader reached end of block"); continue; } recordFound = true; if (DEBUG) LOG.debug("read value: " + currentValue); } catch (RuntimeException e) { throw new ParquetDecodingException(format("Can not read value at %d in block %d in file %s", current, currentBlock, file), e); } } return true; }
public void read(RecordReader<Group> recordReader, MessageType schema, List<Group> records) { records.add(recordReader.read()); }
public boolean nextKeyValue() throws IOException, InterruptedException { if (current < total) { try { checkRead(); currentValue = recordReader.read(); if (DEBUG) LOG.debug("read value: " + currentValue); current ++; } catch (RuntimeException e) { throw new ParquetDecodingException(format("Can not read value at %d in block %d in file %s", current, currentBlock, file), e); } return true; } return false; } }
private static void read(RecordReader<Object> recordReader, int count, MessageType schema) { Object[] records = new Object[count]; System.gc(); System.out.println("<<<"); long t0 = System.currentTimeMillis(); for (int i = 0; i < records.length; i++) { records[i] = recordReader.read(); } long t1 = System.currentTimeMillis(); System.out.println(">>>"); long t = t1-t0; float err = (float)100 * 2 / t; // (+/- 1 ms) System.out.printf("read %,9d recs in %,5d ms at %,9d rec/s err: %3.2f%%\n", count , t, t == 0 ? 0 : count * 1000 / t, err); if (!records[0].equals("end()")) { throw new RuntimeException(""+records[0]); } }
@Test public void testPushParser() { MemPageStore memPageStore = new MemPageStore(); MemColumnWriteStore columns = new MemColumnWriteStore(memPageStore, 800); MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema); new GroupWriter(columnIO.getRecordWriter(columns), schema).write(r1); columns.flush(); final Deque<String> expectations = new ArrayDeque<String>(); for (String string : expectedEventsForR1) { expectations.add(string); } RecordReader<Void> recordReader = columnIO.getRecordReader(memPageStore, new ExpectationValidatingConverter(expectations, schema)); recordReader.read(); }