public MapJoinOperatorExplainVectorization(MapJoinDesc mapJoinDesc, VectorMapJoinDesc vectorMapJoinDesc) { // VectorMapJoinOperator is not native vectorized. super( vectorMapJoinDesc, vectorMapJoinDesc.getHashTableImplementationType() != HashTableImplementationType.NONE); this.mapJoinDesc = mapJoinDesc; this.vectorMapJoinDesc = vectorMapJoinDesc; vectorMapJoinInfo = (vectorMapJoinDesc != null ? vectorMapJoinDesc.getVectorMapJoinInfo() : null); }
@Explain(vectorization = Vectorization.EXPRESSION, displayName = "hashTableImplementationType", explainLevels = { Level.DEFAULT, Level.EXTENDED }) public String hashTableImplementationType() { if (!isNative) { return null; } return vectorMapJoinDesc.getHashTableImplementationType().name(); }
private void setUpHashTable() { HashTableImplementationType hashTableImplementationType = vectorDesc.getHashTableImplementationType(); switch (vectorDesc.getHashTableImplementationType()) { case OPTIMIZED: { // Create our vector map join optimized hash table variation *above* the // map join table container. vectorMapJoinHashTable = VectorMapJoinOptimizedCreateHashTable.createHashTable(conf, mapJoinTables[posSingleVectorMapJoinSmallTable]); } break; case FAST: { // Get our vector map join fast hash table variation from the // vector map join table container. VectorMapJoinTableContainer vectorMapJoinTableContainer = (VectorMapJoinTableContainer) mapJoinTables[posSingleVectorMapJoinSmallTable]; vectorMapJoinHashTable = vectorMapJoinTableContainer.vectorMapJoinHashTable(); } break; default: throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name()); } LOG.info("Using " + vectorMapJoinHashTable.getClass().getSimpleName() + " from " + this.getClass().getSimpleName()); }
/** * This override lets us substitute our own fast vectorized hash table loader. */ @Override protected HashTableLoader getHashTableLoader(Configuration hconf) { HashTableImplementationType hashTableImplementationType = vectorDesc.getHashTableImplementationType(); HashTableLoader hashTableLoader; switch (vectorDesc.getHashTableImplementationType()) { case OPTIMIZED: // Use the Tez hash table loader. hashTableLoader = HashTableLoaderFactory.getLoader(hconf); break; case FAST: // Use our specialized hash table loader. hashTableLoader = HiveConf.getVar( hconf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark") ? HashTableLoaderFactory.getLoader(hconf) : new VectorMapJoinFastHashTableLoader(); break; default: throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name()); } return hashTableLoader; }
@Override public void init(ExecMapperContext context, MapredContext mrContext, Configuration hconf, MapJoinOperator joinOp) { this.context = context; this.hconf = hconf; this.joinOp = joinOp; this.desc = joinOp.getConf(); if (desc.getVectorMode() && HiveConf.getBoolVar( hconf, HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_FAST_HASHTABLE_ENABLED)) { if (joinOp instanceof VectorizationOperator) { VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) ((VectorizationOperator) joinOp).getVectorDesc(); useFastContainer = vectorDesc != null && vectorDesc.getHashTableImplementationType() == VectorMapJoinDesc.HashTableImplementationType.FAST; } } }
switch (vectorDesc.getHashTableImplementationType()) { case OPTIMIZED: mapJoinTableContainer = break; default: throw new RuntimeException("Unexpected hash table implementation type " + vectorDesc.getHashTableImplementationType());
private VectorMapJoinFastHashTable createHashTable(int newThreshold) { HashTableImplementationType hashTableImplementationType = vectorDesc.getHashTableImplementationType(); HashTableKind hashTableKind = vectorDesc.getHashTableKind(); HashTableKeyType hashTableKeyType = vectorDesc.getHashTableKeyType();