/** * Generate the inner big table only join output results for one vectorized row batch with * a repeated key. * * @param batch * The big table batch with any matching and any non matching rows both as * selected in use. * @param hashMultiSetResult * The hash multi-set results for the batch. */ protected int generateHashMultiSetResultRepeatedAll(VectorizedRowBatch batch, VectorMapJoinHashMultiSetResult hashMultiSetResult) throws HiveException { long count = hashMultiSetResult.count(); if (batch.selectedInUse) { // The selected array is already filled in as we want it. } else { int[] selected = batch.selected; for (int i = 0; i < batch.size; i++) { selected[i] = i; } batch.selectedInUse = true; } do { forwardBigTableBatch(batch); count--; } while (count > 0); // We forwarded the batch in this method. return 0; }
@Override public void forget() { haveCount = false; bytesBytesMultiHashMapResult.forget(); super.forget(); } }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("(" + super.toString() + ", "); sb.append("count " + count + ")"); return sb.toString(); } }
@Override public JoinUtil.JoinResult contains(byte[] keyBytes, int keyStart, int keyLength, VectorMapJoinHashMultiSetResult hashMultiSetResult) { VectorMapJoinFastHashMultiSet.HashMultiSetResult optimizedHashMultiSetResult = (VectorMapJoinFastHashMultiSet.HashMultiSetResult) hashMultiSetResult; optimizedHashMultiSetResult.forget(); long hashCode = HashCodeUtil.murmurHash(keyBytes, keyStart, keyLength); long count = findReadSlot(keyBytes, keyStart, keyLength, hashCode, hashMultiSetResult.getReadPos()); JoinUtil.JoinResult joinResult; if (count == -1) { joinResult = JoinUtil.JoinResult.NOMATCH; } else { optimizedHashMultiSetResult.set(count); joinResult = JoinUtil.JoinResult.MATCH; } optimizedHashMultiSetResult.setJoinResult(joinResult); return joinResult; }
/** * Generate the inner big table only join output results for one vectorized row batch with * a repeated key. * * @param batch * The big table batch with any matching and any non matching rows both as * selected in use. * @param hashMultiSetResult * The hash multi-set results for the batch. */ protected int generateHashMultiSetResultRepeatedAll(VectorizedRowBatch batch, VectorMapJoinHashMultiSetResult hashMultiSetResult) throws HiveException { long count = hashMultiSetResult.count(); if (batch.selectedInUse) { // The selected array is already filled in as we want it. } else { int[] selected = batch.selected; for (int i = 0; i < batch.size; i++) { selected[i] = i; } batch.selectedInUse = true; } do { forwardBigTableBatch(batch); count--; } while (count > 0); // We forwarded the batch in this method. return 0; }
@Override public void forget() { haveCount = false; bytesBytesMultiHashMapResult.forget(); super.forget(); } }
equalKeySeriesValueCounts[equalKeySeriesCount] = hashMultiSetResults[hashMultiSetResultCount].count(); equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount; equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
@Override public void forget() { haveCount = false; bytesBytesMultiHashMapResult.forget(); super.forget(); } }
equalKeySeriesValueCounts[equalKeySeriesCount] = hashMultiSetResults[hashMultiSetResultCount].count(); equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount; equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
public void verify(VectorMapJoinFastBytesHashMultiSet map) { int mapSize = map.size(); if (mapSize != count) { TestCase.fail("map.size() does not match expected count"); } for (int index = 0; index < count; index++) { FastBytesHashMultiSetElement element = array[index]; byte[] key = element.getKey(); int multiSetCount = element.getMultiSetCount(); VectorMapJoinHashMultiSetResult hashMultiSetResult = map.createHashMultiSetResult(); JoinUtil.JoinResult joinResult = map.contains(key, 0, key.length, hashMultiSetResult); if (joinResult != JoinUtil.JoinResult.MATCH) { assertTrue(false); } assertEquals(hashMultiSetResult.count(), multiSetCount); } } }
public void verify(VectorMapJoinFastLongHashMultiSet map) { int mapSize = map.size(); if (mapSize != count) { TestCase.fail("map.size() does not match expected count"); } for (int index = 0; index < count; index++) { FastLongHashMultiSetElement element = array[index]; long key = element.getKey(); int multiSetCount = element.getMultiSetCount(); VectorMapJoinHashMultiSetResult hashMultiSetResult = map.createHashMultiSetResult(); JoinUtil.JoinResult joinResult = map.contains(key, hashMultiSetResult); if (joinResult != JoinUtil.JoinResult.MATCH) { assertTrue(false); } assertEquals(hashMultiSetResult.count(), multiSetCount); } } }
@Test public void testGetNonExistent() throws Exception { random = new Random(98222); VectorMapJoinFastMultiKeyHashMultiSet map = new VectorMapJoinFastMultiKeyHashMultiSet( false,CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastBytesHashMultiSet verifyTable = new VerifyFastBytesHashMultiSet(); byte[] key1 = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key1); map.testPutRow(key1); verifyTable.add(key1); verifyTable.verify(map); byte[] key2 = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key2); VectorMapJoinHashMultiSetResult hashMultiSetResult = map.createHashMultiSetResult(); JoinUtil.JoinResult joinResult = map.contains(key2, 0, key2.length, hashMultiSetResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH); map.testPutRow(key2); verifyTable.add(key2); verifyTable.verify(map); byte[] key3 = new byte[random.nextInt(MAX_KEY_LENGTH)]; random.nextBytes(key3); hashMultiSetResult = map.createHashMultiSetResult(); joinResult = map.contains(key3, 0, key3.length, hashMultiSetResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH); assertEquals(hashMultiSetResult.count(), 0); }
equalKeySeriesValueCounts[equalKeySeriesCount] = hashMultiSetResults[hashMultiSetResultCount].count(); equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount; equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
equalKeySeriesValueCounts[equalKeySeriesCount] = hashMultiSetResults[hashMultiSetResultCount].count(); equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount; equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
equalKeySeriesValueCounts[equalKeySeriesCount] = hashMultiSetResults[hashMultiSetResultCount].count(); equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount; equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
@Test public void testGetNonExistent() throws Exception { random = new Random(4000); VectorMapJoinFastLongHashMultiSet map = new VectorMapJoinFastLongHashMultiSet( false, false, HashTableKeyType.LONG, CAPACITY, LOAD_FACTOR, WB_SIZE, -1); VerifyFastLongHashMultiSet verifyTable = new VerifyFastLongHashMultiSet(); long key1 = random.nextLong(); map.testPutRow(key1); verifyTable.add(key1); verifyTable.verify(map); long key2 = key1 += 1; VectorMapJoinHashMultiSetResult hashMultiSetResult = map.createHashMultiSetResult(); JoinUtil.JoinResult joinResult = map.contains(key2, hashMultiSetResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH); assertEquals(hashMultiSetResult.count(), 0); map.testPutRow(key2); verifyTable.add(key2); verifyTable.verify(map); long key3 = key2 += 1; hashMultiSetResult = map.createHashMultiSetResult(); joinResult = map.contains(key3, hashMultiSetResult); assertTrue(joinResult == JoinUtil.JoinResult.NOMATCH); assertEquals(hashMultiSetResult.count(), 0); }
equalKeySeriesValueCounts[equalKeySeriesCount] = hashMultiSetResults[hashMultiSetResultCount].count(); equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount; equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
/** * Generate the inner big table only join output results for one vectorized row batch with * a repeated key. * * @param batch * The big table batch with any matching and any non matching rows both as * selected in use. * @param hashMultiSetResult * The hash multi-set results for the batch. */ protected int generateHashMultiSetResultRepeatedAll(VectorizedRowBatch batch, VectorMapJoinHashMultiSetResult hashMultiSetResult) throws HiveException { long count = hashMultiSetResult.count(); if (batch.selectedInUse) { // The selected array is already filled in as we want it. } else { int[] selected = batch.selected; for (int i = 0; i < batch.size; i++) { selected[i] = i; } batch.selectedInUse = true; } do { forwardBigTableBatch(batch); count--; } while (count > 0); // We forwarded the batch in this method. return 0; }
equalKeySeriesValueCounts[equalKeySeriesCount] = hashMultiSetResults[hashMultiSetResultCount].count(); equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount; equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
equalKeySeriesValueCounts[equalKeySeriesCount] = hashMultiSetResults[hashMultiSetResultCount].count(); equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount; equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;