protected PTFPartition(Configuration cfg, AbstractSerDe serDe, StructObjectInspector inputOI, StructObjectInspector outputOI, boolean createElemContainer) throws HiveException { this.serDe = serDe; this.inputOI = inputOI; this.outputOI = outputOI; if ( createElemContainer ) { int containerNumRows = HiveConf.getIntVar(cfg, ConfVars.HIVEJOINCACHESIZE); elems = new PTFRowContainer<List<Object>>(containerNumRows, cfg, null); elems.setSerDe(serDe, outputOI); elems.setTableDesc(PTFRowContainer.createTableDesc(inputOI)); } else { elems = null; } }
public void append(Object o) throws HiveException { if ( elems.rowCount() == Integer.MAX_VALUE ) { throw new HiveException(String.format("Cannot add more than %d elements to a PTFPartition", Integer.MAX_VALUE)); } @SuppressWarnings("unchecked") List<Object> l = (List<Object>) ObjectInspectorUtils.copyToStandardObject(o, inputOI, ObjectInspectorCopyOption.WRITABLE); elems.addRow(l); }
public Object getAt(int i) throws HiveException { return elems.getAt(i); }
private void readBlock(int blockNum) throws HiveException { currentReadBlockStartRow = getBlockSize() * blockNum; if ( blockNum == numBlocks() - 1 ) { setWriteBlockAsReadBlock(); return; resetCurrentReadBlockToFirstReadBlock(); endSplit = blockInfos.get(blockNum + 1).startingSplit; } else { endSplit = getLastActualSplit(); for(int i = startSplit; i <= endSplit && readIntoOffset < getBlockSize(); i++ ) { org.apache.hadoop.mapred.RecordReader rr = setReaderAtSplit(i); if ( i == startSplit ) { ((PTFSequenceFileRecordReader)rr).seek(bI.startOffset); nextBlock(readIntoOffset); readIntoOffset = getCurrentReadBlockSize(); clearRows(); LOG.error(e.toString(), e); if ( e instanceof HiveException ) {
row.add(new Text("def " + i)); row.add(new Text(value)); rc.addRow(row); assertEquals(sz, rc.rowCount()); i = 0; row = new ArrayList<Object>(); row = rc.first(); while(row != null ) { assertEquals("abc " + i, row.get(1).toString()); i++; row = rc.next(); row = rc.first(); for(i = sz - 1; i >= 0; i-- ) { row = rc.getAt(i); assertEquals("abc " + i, row.get(1).toString()); row = rc.getAt(j); assertEquals("abc " + j, row.get(1).toString()); row = rc.getAt(j); assertEquals("abc " + j, row.get(1).toString()); for(int k = j + 1; k < j + (blockSize/4) && k < sz; k++) { row = rc.next(); assertEquals("def " + k, row.get(4).toString());
@Override public void addRow(Row t) throws HiveException { if ( willSpill() ) { setupWriter(); PTFRecordWriter rw = (PTFRecordWriter) getRecordWriter(); BlockInfo blkInfo = new BlockInfo(); try { blkInfo.startOffset = rw.outStream.getLength(); blockInfos.add(blkInfo); } catch(IOException e) { clearRows(); LOG.error(e.toString(), e); throw new HiveException(e); } } super.addRow(t); }
public PTFPartitionIterator<Object> iterator() throws HiveException { elems.first(); return new PItr(0, size()); }
public int size() { return elems.rowCount(); }
public void close() { try { elems.close(); } catch (HiveException e) { LOG.error(e.toString(), e); } }
public void reset() throws HiveException { elems.clearRows(); }
ObjectInspectorUtils.getStandardObjectInspector(serde.getObjectInspector())); result.setTableDesc( PTFRowContainer.createTableDesc((StructObjectInspector) serde.getObjectInspector())); TimestampWritableV2 key = new TimestampWritableV2(Timestamp.ofEpochMilli(10)); result.setKeyObject(Lists.newArrayList(key));
private void readBlock(int blockNum) throws HiveException { currentReadBlockStartRow = getBlockSize() * blockNum; if ( blockNum == numBlocks() - 1 ) { setWriteBlockAsReadBlock(); return; resetCurrentReadBlockToFirstReadBlock(); endSplit = blockInfos.get(blockNum + 1).startingSplit; } else { endSplit = getLastActualSplit(); for(int i = startSplit; i <= endSplit && readIntoOffset < getBlockSize(); i++ ) { org.apache.hadoop.mapred.RecordReader rr = setReaderAtSplit(i); if ( i == startSplit ) { ((PTFSequenceFileRecordReader)rr).seek(bI.startOffset); nextBlock(readIntoOffset); readIntoOffset = getCurrentReadBlockSize(); clearRows(); LOG.error(e.toString(), e); if ( e instanceof HiveException ) {
@Override public void addRow(Row t) throws HiveException { if ( willSpill() ) { setupWriter(); PTFRecordWriter rw = (PTFRecordWriter) getRecordWriter(); BlockInfo blkInfo = new BlockInfo(); try { blkInfo.startOffset = rw.outStream.getLength(); blockInfos.add(blkInfo); } catch(IOException e) { clearRows(); LOG.error(e.toString(), e); throw new HiveException(e); } } super.addRow(t); }
public PTFPartitionIterator<Object> iterator() throws HiveException { elems.first(); return new PItr(0, size()); }
public int size() { return elems.rowCount(); }
public void close() { try { elems.close(); } catch (HiveException e) { LOG.error(e.toString(), e); } }
public void reset() throws HiveException { elems.clearRows(); }
private void readBlock(int blockNum) throws HiveException { currentReadBlockStartRow = getBlockSize() * blockNum; if ( blockNum == numBlocks() - 1 ) { setWriteBlockAsReadBlock(); return; resetCurrentReadBlockToFirstReadBlock(); int readIntoOffset = 0; for(int i = startSplit; i <= endSplit; i++ ) { org.apache.hadoop.mapred.RecordReader rr = setReaderAtSplit(i); if ( i == startSplit ) { ((PTFSequenceFileRecordReader)rr).seek(bI.startOffset); nextBlock(readIntoOffset); readIntoOffset = getCurrentReadBlockSize(); clearRows(); LOG.error(e.toString(), e); if ( e instanceof HiveException ) {
protected PTFPartition(Configuration cfg, AbstractSerDe serDe, StructObjectInspector inputOI, StructObjectInspector outputOI, boolean createElemContainer) throws HiveException { this.serDe = serDe; this.inputOI = inputOI; this.outputOI = outputOI; if ( createElemContainer ) { int containerNumRows = HiveConf.getIntVar(cfg, ConfVars.HIVEJOINCACHESIZE); elems = new PTFRowContainer<List<Object>>(containerNumRows, cfg, null); elems.setSerDe(serDe, outputOI); elems.setTableDesc(PTFRowContainer.createTableDesc(inputOI)); } else { elems = null; } }
@Override public void addRow(Row t) throws HiveException { if ( willSpill() ) { setupWriter(); PTFRecordWriter rw = (PTFRecordWriter) getRecordWriter(); BlockInfo blkInfo = new BlockInfo(); try { blkInfo.startOffset = rw.outStream.getLength(); blockInfos.add(blkInfo); } catch(IOException e) { clearRows(); LOG.error(e.toString(), e); throw new HiveException(e); } } super.addRow(t); }