@Explain(vectorization = Vectorization.OPERATOR, displayName = "SMB Map Join Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED }) public SMBJoinOperatorExplainVectorization getSMBJoinVectorization() { VectorSMBJoinDesc vectorSMBJoinDesc = (VectorSMBJoinDesc) getVectorDesc(); if (vectorSMBJoinDesc == null || !(this instanceof SMBJoinDesc)) { return null; } return new SMBJoinOperatorExplainVectorization((SMBJoinDesc) this, vectorSMBJoinDesc); }
@Explain(vectorization = Vectorization.OPERATOR, displayName = "Map Join Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED }) public MapJoinOperatorExplainVectorization getMapJoinVectorization() { VectorMapJoinDesc vectorMapJoinDesc = (VectorMapJoinDesc) getVectorDesc(); if (vectorMapJoinDesc == null || this instanceof SMBJoinDesc) { return null; } return new MapJoinOperatorExplainVectorization(this, vectorMapJoinDesc); }
@Override protected void completeInitializationOp(Object[] os) throws HiveException { // setup mapJoinTables and serdes super.completeInitializationOp(os); VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED: { // Create our vector map join optimized hash table variation *above* the // map join table container. vectorMapJoinHashTable = VectorMapJoinOptimizedCreateHashTable.createHashTable(conf, mapJoinTables[posSingleVectorMapJoinSmallTable]); } break; case FAST: { // Get our vector map join fast hash table variation from the // vector map join table container. VectorMapJoinTableContainer vectorMapJoinTableContainer = (VectorMapJoinTableContainer) mapJoinTables[posSingleVectorMapJoinSmallTable]; vectorMapJoinHashTable = vectorMapJoinTableContainer.vectorMapJoinHashTable(); } break; default: throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name()); } LOG.info("Using " + vectorMapJoinHashTable.getClass().getSimpleName() + " from " + this.getClass().getSimpleName()); }
@Override public void init(ExecMapperContext context, MapredContext mrContext, Configuration hconf, MapJoinOperator joinOp) { this.context = context; this.hconf = hconf; this.joinOp = joinOp; this.desc = joinOp.getConf(); if (desc.getVectorMode() && HiveConf.getBoolVar( hconf, HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_FAST_HASHTABLE_ENABLED)) { VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); useFastContainer = vectorDesc != null && vectorDesc.hashTableImplementationType() == VectorMapJoinDesc.HashTableImplementationType.FAST; } }
/** * This override lets us substitute our own fast vectorized hash table loader. */ @Override protected HashTableLoader getHashTableLoader(Configuration hconf) { VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableLoader hashTableLoader; switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED: // Use the Tez hash table loader. hashTableLoader = HashTableLoaderFactory.getLoader(hconf); break; case FAST: // Use our specialized hash table loader. hashTableLoader = HiveConf.getVar( hconf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark") ? HashTableLoaderFactory.getLoader(hconf) : new VectorMapJoinFastHashTableLoader(); break; default: throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name()); } return hashTableLoader; }
Class<? extends Operator<?>> opClass = null; VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc();
vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); vectorMapJoinInfo = vectorDesc.getVectorMapJoinInfo(); Preconditions.checkState(vectorMapJoinInfo != null);
private VectorMapJoinFastHashTable createHashTable(int newThreshold) { VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.getHashTableImplementationType(); HashTableKind hashTableKind = vectorDesc.getHashTableKind();
VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); HashTableKind hashTableKind = vectorDesc.getHashTableKind(); HashTableKeyType hashTableKeyType = vectorDesc.getHashTableKeyType();
VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); HashTableKind hashTableKind = vectorDesc.hashTableKind(); HashTableKeyType hashTableKeyType = vectorDesc.hashTableKeyType();
private VectorMapJoinFastHashTable createHashTable(int newThreshold) { VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) desc.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableKind hashTableKind = vectorDesc.hashTableKind();
/** * This override lets us substitute our own fast vectorized hash table loader. */ @Override protected HashTableLoader getHashTableLoader(Configuration hconf) { VectorMapJoinDesc vectorDesc = conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableLoader hashTableLoader; switch (vectorDesc.hashTableImplementationType()) { case OPTIMIZED: // Use the Tez hash table loader. hashTableLoader = HashTableLoaderFactory.getLoader(hconf); break; case FAST: // Use our specialized hash table loader. hashTableLoader = new VectorMapJoinFastHashTableLoader(); break; default: throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name()); } return hashTableLoader; }
VectorMapJoinDesc vectorDesc = conf.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableLoader hashTableLoader;
HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_MINMAX_ENABLED); VectorMapJoinDesc vectorDesc = desc.getVectorDesc(); vectorDesc.setHashTableImplementationType(hashTableImplementationType); vectorDesc.setHashTableKind(hashTableKind);
private VectorMapJoinFastHashTable createHashTable(int newThreshold) { VectorMapJoinDesc vectorDesc = desc.getVectorDesc(); HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType(); HashTableKind hashTableKind = vectorDesc.hashTableKind();
VectorMapJoinDesc vectorDesc = desc.getVectorDesc(); HashTableKind hashTableKind = vectorDesc.hashTableKind(); HashTableKeyType hashTableKeyType = vectorDesc.hashTableKeyType();