/** * Returns true if there are more records, false otherwise. */ public boolean hasNext() throws IOException, RecordServiceException { return records_.hasNext(); }
/** * Closes the underlying task. Must be called for every GenericRecords object * created. Invalid to call other APIs after this. Idempotent. */ @Override public void close() { records_.close(); } }
public GenericRecords(Records records) { records_ = records ; schema_ = records_.getSchema(); avroSchema_ = SchemaUtils.convertSchema(schema_); }
/** * Advances to the next record. Return false if there are no more records. */ public boolean nextRecord() throws IOException { try { if (!reader_.records().hasNext()) return false; } catch (RecordServiceException e) { // TODO: is this the most proper way to deal with this in MR? throw new IOException("Could not fetch record.", e); } currentRSRecord_ = reader_.records().next(); return true; }
protected Records(RecordServiceWorkerClient worker, RecordServiceWorkerClient.TaskState handle) throws IOException, RecordServiceException { worker_ = worker; handle_ = handle; closeWorker_ = false; record_ = new Record(handle.getSchema()); nextBatch(); if (record_.schema_.isCountStar) { // This is a count(*). We will read the one and only value that is the number of // records. The iterator interface will return count(*) number of NULLs. Preconditions.checkState(record_.schema_.cols.size() == 1); Preconditions.checkState(record_.schema_.cols.get(0).type.typeId == Type.BIGINT); Preconditions.checkState(record_.numRecords_ == 1); record_.numRecords_ = record_.nextLong(0); } hasNext_ = hasNext(); }
/** * Returns and advances to the next record. Throws exception if * there are no more records. */ @Override public T next() throws IOException, RecordServiceException { T record; try { record = class_.newInstance(); } catch (InstantiationException e) { throw new RuntimeException("Could not create new record instance.", e); } catch (IllegalAccessException e) { throw new RuntimeException("Could not create new record instance.", e); } // add columns for record. RecordUtil.buildRecord( schema_, records_.next(), avroSchema_, record, recordIndexToRsIndex_); return record; }
@Override public float getProgress() throws IOException { if (reader_ == null) return 1; return reader_.records().progress(); } }
@Override public void close() throws IOException { if (reader_ != null) { Preconditions.checkNotNull(reporter_); try { setCounters(reporter_, reader_.records().getStatus().stats); } catch (RecordServiceException e) { LOG.debug("Could not populate counters: " + e); } finally { reader_.close(); reader_ = null; } } }
/** * Returns true if there are more records. */ public boolean hasNext() throws IOException, RecordServiceException { Preconditions.checkNotNull(fetchResult_); while (record_.recordIdx_ == record_.numRecords_ - 1) { if (fetchResult_.done) { hasNext_ = false; return false; } nextBatch(); } return true; }
/** * Utility function to launch task 'taskId' in plan at the first host. * Local hosts are preferred over remote hosts. * * TODO: move this into a test/sample directory? Real frameworks have * to deal with much more stuff. */ public static Records execTask(PlanRequestResult plan, int taskId) throws RecordServiceException, IOException { if (taskId >= plan.tasks.size() || taskId < 0) { throw new RuntimeException("Invalid task id."); } Task task = plan.tasks.get(taskId); NetworkAddress host; if (task.localHosts.isEmpty()) { if (plan.hosts.isEmpty()) { throw new RuntimeException("No hosts are provided to run this task."); } host = plan.hosts.get(0); } else { host = task.localHosts.get(0); } Records records = new RecordServiceWorkerClient.Builder() .connect(host.hostname, host.port).execAndFetch(task); records.setCloseWorker(true); return records; } }
/** * Executes the task asynchronously, returning a Rows object that can be * used to fetch results. */ public Records execAndFetch(Task task) throws RecordServiceException, IOException { validateIsConnected(); TaskState result = execTaskInternal(task, 0); Records records = null; try { records = new Records(this, result); return records; } finally { if (records == null) closeTask(result); } }
/** * Advances to the next record, populating key,value with the results. * This is the hot path. */ @Override public boolean next(WritableComparable<?> key, RecordServiceRecord value) throws IOException { try { if (!reader_.records().hasNext()) return false; } catch (RecordServiceException e) { // TODO: is this the most proper way to deal with this in MR? throw new IOException("Could not fetch record.", e); } value.reset(reader_.records().next()); return true; }
/** * Returns the next key / value record. Throws exception if there are no more records. */ public KeyValuePair<K, V> next() throws IOException, RecordServiceException { K key; V value; try { key = keyClass_.newInstance(); value = valueClass_.newInstance(); } catch (InstantiationException e) { throw new RuntimeException("Could not create new key/value record instance.", e); } catch (IllegalAccessException e) { throw new RuntimeException("Could not create new key/value record instance.", e); } Records.Record rsRecord = records_.next(); // Add columns for the key record. RecordUtil.buildRecord(schema_, rsRecord, keySchema_, key, keyIndexToRsIndex_); // Add columns for the value record. RecordUtil.buildRecord(schema_, rsRecord, valueSchema_, value, valueIndexToRsIndex_); return new KeyValuePair<K, V>(key, value); }
@Override public float getProgress() throws IOException { if (reader_ == null) return 1; return reader_.records().progress(); } }
@Override public void close() throws IOException { if (reader_ != null) { try { RecordServiceInputFormatBase.setCounters( context_, reader_.records().getStatus().stats); } catch (RecordServiceException e) { LOG.debug("Could not populate counters: " + e); } finally { reader_.close(); reader_ = null; } } }
@Override public boolean next(LongWritable key, Text value) throws IOException { try { if (!reader_.records().hasNext()) return false; Records.Record record = reader_.records().next(); if (record.isNull(0)) { value.set(EMPTY); } else { ByteArray data = record.nextByteArray(0); value.set(data.byteBuffer().array(), data.offset(), data.len()); } key.set(recordNum_++); return true; } catch (RecordServiceException e) { throw new IOException("Could not get next record.", e); } }
/** * Returns true if there are more records, false otherwise. */ @Override public boolean hasNext() throws IOException, RecordServiceException { return records_.hasNext(); }
@SuppressWarnings("unchecked") public SpecificRecords(Schema readerSchema, Records records, RecordUtil.ResolveBy resolveBy) { avroSchema_ = readerSchema; class_ = new SpecificData().getClass(avroSchema_); records_ = records; schema_ = records_.getSchema(); recordIndexToRsIndex_ = RecordUtil.resolveSchema(avroSchema_, schema_, resolveBy); }
/** * Closes the underlying task. Must be called for every GenericRecords object * created. Invalid to call other APIs after this. Idempotent. */ @Override public void close() { records_.close(); } }
Records.Record rsRecord = records_.next(); Record record = new Record(avroSchema_); for (int i = 0; i < schema_.cols.size(); ++i) {