@Override public D readRecord(D reuse) throws DataRecordException, IOException { return actualExtractor.readRecord(reuse); }
/** * Read an {@link RecordEnvelope}. By default, just wrap {@link #readRecord(Object)} in a {@link RecordEnvelope}. */ @SuppressWarnings(value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE", justification = "Findbugs believes readRecord(null) is non-null. This is not true.") default RecordEnvelope<D> readRecordEnvelope() throws DataRecordException, IOException { D record = readRecord(null); return record == null ? null : new RecordEnvelope<>(record); }
private int getNumRecords(Extractor<?, ?> extractor) throws DataRecordException, IOException { int numRecords = 0; while (extractor.readRecord(null) != null) { numRecords++; } return numRecords; }
@Override public D readRecord(@Deprecated D reuse) throws DataRecordException, IOException { try (Closer closer = Closer.create()) { if (closer.register(this.limiter.acquirePermits(1)) != null) { return this.extractor.readRecord(reuse); } submitLimiterStopMetadataEvents(); return null; } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Interrupted while trying to acquire the next permit", ie); } }
@Test public void testSourceExtractor() throws DataRecordException, IOException { SourceState state = new SourceState(); state.setProp(HelloWorldSource.NUM_HELLOS_FULL_KEY, 10); HelloWorldSource source = new HelloWorldSource(); List<WorkUnit> wus = source.getWorkunits(state); Assert.assertEquals(wus.size(), 10); for (int i = 0; i < wus.size(); ++i) { WorkUnit wu = wus.get(i); Assert.assertEquals(wu.getPropAsInt(HelloWorldSource.HELLO_ID_FULL_KEY), i + 1); WorkUnitState wuState = new WorkUnitState(wu, state); Extractor<String, String> extr = source.getExtractor(wuState); Assert.assertEquals(extr.getExpectedRecordCount(), 1); Assert.assertEquals(extr.readRecord(null), "Hello world "+ (i+1) + " !"); } }
@Test (enabled=false) public void testComputeTime() throws DataRecordException, IOException { final int MEM_ALLOC_BYTES = 100; final int NUM_WORK_UNITS = 1; final int COMPUTE_TIME_MICRO = 10000; final int NUM_RECORDS = 500; SourceState state = new SourceState(); state.setProp(StressTestingSource.NUM_WORK_UNITS_KEY, NUM_WORK_UNITS); state.setProp(StressTestingSource.MEM_ALLOC_BYTES_KEY, MEM_ALLOC_BYTES); state.setProp(StressTestingSource.COMPUTE_TIME_MICRO_KEY, COMPUTE_TIME_MICRO); state.setProp(StressTestingSource.NUM_RECORDS_KEY, NUM_RECORDS); StressTestingSource source = new StressTestingSource(); List<WorkUnit> wus = source.getWorkunits(state); Assert.assertEquals(wus.size(), NUM_WORK_UNITS); WorkUnit wu = wus.get(0); WorkUnitState wuState = new WorkUnitState(wu, state); Extractor<String, byte[]> extractor = source.getExtractor(wuState); byte[] record; long startTimeNano = System.nanoTime(); while ((record = extractor.readRecord(null)) != null) { Assert.assertEquals(record.length, 100); } long endTimeNano = System.nanoTime(); long timeSpentMicro = (endTimeNano - startTimeNano)/(1000); // check that there is less than 5 second difference between expected and actual time spent Assert.assertTrue(Math.abs(timeSpentMicro - (COMPUTE_TIME_MICRO * NUM_RECORDS)) < (5000000), "Time spent " + timeSpentMicro); }
@Test (enabled=false) public void testSleepTime() throws DataRecordException, IOException { final int MEM_ALLOC_BYTES = 100; final int NUM_WORK_UNITS = 1; final int SLEEP_TIME_MICRO = 10000; final int NUM_RECORDS = 500; SourceState state = new SourceState(); state.setProp(StressTestingSource.NUM_WORK_UNITS_KEY, NUM_WORK_UNITS); state.setProp(StressTestingSource.MEM_ALLOC_BYTES_KEY, MEM_ALLOC_BYTES); state.setProp(StressTestingSource.SLEEP_TIME_MICRO_KEY, SLEEP_TIME_MICRO); state.setProp(StressTestingSource.NUM_RECORDS_KEY, NUM_RECORDS); StressTestingSource source = new StressTestingSource(); List<WorkUnit> wus = source.getWorkunits(state); Assert.assertEquals(wus.size(), NUM_WORK_UNITS); WorkUnit wu = wus.get(0); WorkUnitState wuState = new WorkUnitState(wu, state); Extractor<String, byte[]> extractor = source.getExtractor(wuState); byte[] record; long startTimeNano = System.nanoTime(); while ((record = extractor.readRecord(null)) != null) { Assert.assertEquals(record.length, 100); } long endTimeNano = System.nanoTime(); long timeSpentMicro = (endTimeNano - startTimeNano)/(1000); // check that there is less than 2 second difference between expected and actual time spent Assert.assertTrue(Math.abs(timeSpentMicro - (SLEEP_TIME_MICRO * NUM_RECORDS)) < (2000000), "Time spent " + timeSpentMicro); }
@Test public void testSourceExtractor() throws DataRecordException, IOException { final int MEM_ALLOC_BYTES = 100; final int NUM_WORK_UNITS = 10; final int COMPUTE_TIME_MICRO = 10; final int NUM_RECORDS = 10000; SourceState state = new SourceState(); state.setProp(StressTestingSource.NUM_WORK_UNITS_KEY, NUM_WORK_UNITS); state.setProp(StressTestingSource.MEM_ALLOC_BYTES_KEY, MEM_ALLOC_BYTES); state.setProp(StressTestingSource.COMPUTE_TIME_MICRO_KEY, COMPUTE_TIME_MICRO); state.setProp(StressTestingSource.NUM_RECORDS_KEY, NUM_RECORDS); StressTestingSource source = new StressTestingSource(); List<WorkUnit> wus = source.getWorkunits(state); Assert.assertEquals(wus.size(), NUM_WORK_UNITS); for (int i = 0; i < wus.size(); ++i) { WorkUnit wu = wus.get(i); WorkUnitState wuState = new WorkUnitState(wu, state); Extractor<String, byte[]> extractor = source.getExtractor(wuState); Assert.assertEquals(extractor.getExpectedRecordCount(), NUM_RECORDS); Assert.assertEquals(extractor.readRecord(null).length, 100); } }
while ((record = extractor.readRecord(null)) != null) { Assert.assertEquals(record.length, 100);
Assert.assertEquals(extractor.readRecord(null).toString(), json);
@Override public D readRecord(D reuse) throws DataRecordException, IOException { return actualExtractor.readRecord(reuse); }
/** * Read an {@link RecordEnvelope}. By default, just wrap {@link #readRecord(Object)} in a {@link RecordEnvelope}. */ @SuppressWarnings(value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE", justification = "Findbugs believes readRecord(null) is non-null. This is not true.") default RecordEnvelope<D> readRecordEnvelope() throws DataRecordException, IOException { D record = readRecord(null); return record == null ? null : new RecordEnvelope<>(record); }
@Override public D readRecord(@Deprecated D reuse) throws DataRecordException, IOException { try (Closer closer = Closer.create()) { if (closer.register(this.limiter.acquirePermits(1)) != null) { return this.extractor.readRecord(reuse); } submitLimiterStopMetadataEvents(); return null; } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Interrupted while trying to acquire the next permit", ie); } }