protected void spillBatchRepeated(VectorizedRowBatch batch, VectorMapJoinHashTableResult hashTableResult) throws HiveException, IOException { if (bigTableVectorSerializeRow == null) { setupSpillSerDe(batch); } int[] selected = batch.selected; boolean selectedInUse = batch.selectedInUse; for (int logical = 0; logical < batch.size; logical++) { int batchIndex = (selectedInUse ? selected[logical] : logical); spillSerializeRow(batch, batchIndex, hashTableResult.spillPartitionId()); } }
protected void spillHashMapBatch(VectorizedRowBatch batch, VectorMapJoinHashTableResult[] hashTableResults, int[] spills, int[] spillHashTableResultIndices, int spillCount) throws HiveException, IOException { if (bigTableVectorSerializeRow == null) { setupSpillSerDe(batch); } for (int i = 0; i < spillCount; i++) { int batchIndex = spills[i]; int hashTableResultIndex = spillHashTableResultIndices[i]; VectorMapJoinHashTableResult hashTableResult = hashTableResults[hashTableResultIndex]; spillSerializeRow(batch, batchIndex, hashTableResult.spillPartitionId()); } }
private void spillSerializeRow(VectorizedRowBatch batch, int batchIndex, VectorMapJoinHashTableResult hashTableResult) throws IOException { int partitionId = hashTableResult.spillPartitionId(); HybridHashTableContainer ht = (HybridHashTableContainer) mapJoinTables[posSingleVectorMapJoinSmallTable]; HashPartition hp = ht.getHashPartitions()[partitionId]; VectorMapJoinRowBytesContainer rowBytesContainer = hp.getMatchfileRowBytesContainer(); Output output = rowBytesContainer.getOuputForRowBytes(); // int offset = output.getLength(); bigTableVectorSerializeRow.setOutputAppend(output); bigTableVectorSerializeRow.serializeWrite(batch, batchIndex); // int length = output.getLength() - offset; rowBytesContainer.finishRow(); // LOG.debug("spillSerializeRow spilled batchIndex " + batchIndex + ", length " + length); }
private void spillSerializeRow(VectorizedRowBatch batch, int batchIndex, VectorMapJoinHashTableResult hashTableResult) throws IOException { int partitionId = hashTableResult.spillPartitionId(); HybridHashTableContainer ht = (HybridHashTableContainer) mapJoinTables[posSingleVectorMapJoinSmallTable]; HashPartition hp = ht.getHashPartitions()[partitionId]; VectorMapJoinRowBytesContainer rowBytesContainer = hp.getMatchfileRowBytesContainer(); Output output = rowBytesContainer.getOuputForRowBytes(); // int offset = output.getLength(); bigTableVectorSerializeRow.setOutputAppend(output); bigTableVectorSerializeRow.serializeWrite(batch, batchIndex); // int length = output.getLength() - offset; rowBytesContainer.finishRow(); // LOG.debug("spillSerializeRow spilled batchIndex " + batchIndex + ", length " + length); }