public MapJoinOperatorExplainVectorization(MapJoinDesc mapJoinDesc, VectorDesc vectorDesc) { // VectorMapJoinOperator is not native vectorized. super(vectorDesc, ((VectorMapJoinDesc) vectorDesc).hashTableImplementationType() != HashTableImplementationType.NONE); this.mapJoinDesc = mapJoinDesc; vectorMapJoinDesc = (VectorMapJoinDesc) vectorDesc; vectorMapJoinInfo = vectorMapJoinDesc.getVectorMapJoinInfo(); }
@Override protected void completeInitializationOp(Object[] os) throws HiveException { // setup mapJoinTables and serdes super.completeInitializationOp(os); VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED: { // Create our vector map join optimized hash table variation *above* the // map join table container. vectorMapJoinHashTable = VectorMapJoinOptimizedCreateHashTable.createHashTable(conf, mapJoinTables[posSingleVectorMapJoinSmallTable]); } break; case FAST: { // Get our vector map join fast hash table variation from the // vector map join table container. VectorMapJoinTableContainer vectorMapJoinTableContainer = (VectorMapJoinTableContainer) mapJoinTables[posSingleVectorMapJoinSmallTable]; vectorMapJoinHashTable = vectorMapJoinTableContainer.vectorMapJoinHashTable(); } break; default: throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name()); } LOG.info("Using " + vectorMapJoinHashTable.getClass().getSimpleName() + " from " + this.getClass().getSimpleName()); }
/** * This override lets us substitute our own fast vectorized hash table loader. */ @Override protected HashTableLoader getHashTableLoader(Configuration hconf) { VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableLoader hashTableLoader; switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED: // Use the Tez hash table loader. hashTableLoader = HashTableLoaderFactory.getLoader(hconf); break; case FAST: // Use our specialized hash table loader. hashTableLoader = HiveConf.getVar( hconf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark") ? HashTableLoaderFactory.getLoader(hconf) : new VectorMapJoinFastHashTableLoader(); break; default: throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name()); } return hashTableLoader; }
@Override public void init(ExecMapperContext context, MapredContext mrContext, Configuration hconf, MapJoinOperator joinOp) { this.context = context; this.hconf = hconf; this.joinOp = joinOp; this.desc = joinOp.getConf(); if (desc.getVectorMode() && HiveConf.getBoolVar( hconf, HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_FAST_HASHTABLE_ENABLED)) { VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); useFastContainer = vectorDesc != null && vectorDesc.hashTableImplementationType() == VectorMapJoinDesc.HashTableImplementationType.FAST; } }
HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableKind hashTableKind = vectorDesc.hashTableKind(); HashTableKeyType hashTableKeyType = vectorDesc.hashTableKeyType();
/** * This override lets us substitute our own fast vectorized hash table loader. */ @Override protected HashTableLoader getHashTableLoader(Configuration hconf) { VectorMapJoinDesc vectorDesc = conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableLoader hashTableLoader; switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED: // Use the Tez hash table loader. hashTableLoader = HashTableLoaderFactory.getLoader(hconf); break; case FAST: // Use our specialized hash table loader. hashTableLoader = new VectorMapJoinFastHashTableLoader(); break; default: throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name()); } return hashTableLoader; }
HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableLoader hashTableLoader; switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED:
HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableKind hashTableKind = vectorDesc.hashTableKind(); HashTableKeyType hashTableKeyType = vectorDesc.hashTableKeyType();