private boolean validateSparkHashTableSinkOperator(SparkHashTableSinkOperator op) { SparkHashTableSinkDesc desc = op.getConf(); byte tag = desc.getTag(); // it's essentially a MapJoinDesc List<ExprNodeDesc> filterExprs = desc.getFilters().get(tag); List<ExprNodeDesc> keyExprs = desc.getKeys().get(tag); List<ExprNodeDesc> valueExprs = desc.getExprs().get(tag); return validateExprNodeDesc( filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER, /* allowComplex */ true) && validateExprNodeDesc(keyExprs, "Key") && validateExprNodeDesc(valueExprs, "Value"); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); ObjectInspector[] inputOIs = new ObjectInspector[conf.getTagLength()]; byte tag = conf.getTag(); inputOIs[tag] = inputObjInspectors[0]; conf.setTagOrder(new Byte[]{ tag }); numReplication = (short) hconf.getInt(MAPRED_FILE_REPLICATION, DEFAULT_REPLICATION); htsOperator.setConf(conf); htsOperator.initialize(hconf, inputOIs); }
SparkHashTableSinkDesc hashTableSinkDesc = new SparkHashTableSinkDesc(mjDesc); SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator)OperatorFactory.get( tableScan.getCompilationOpContext(), hashTableSinkDesc); if (valueIndex != null) { List<ExprNodeDesc> newValues = new ArrayList<ExprNodeDesc>(); List<ExprNodeDesc> values = hashTableSinkDesc.getExprs().get(tag); for (int index = 0; index < values.size(); index++) { if (valueIndex[index] < 0) { hashTableSinkDesc.getExprs().put(tag, newValues); tableScanParents.add(tableScan); hashTableSinkOp.setParentOperators(tableScanParents); hashTableSinkOp.getConf().setTag(tag);
SparkHashTableSinkDesc hashTableSinkDesc = new SparkHashTableSinkDesc(mjDesc); SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator) OperatorFactory.get(hashTableSinkDesc); if (valueIndex != null) { List<ExprNodeDesc> newValues = new ArrayList<ExprNodeDesc>(); List<ExprNodeDesc> values = hashTableSinkDesc.getExprs().get(tag); for (int index = 0; index < values.size(); index++) { if (valueIndex[index] < 0) { hashTableSinkDesc.getExprs().put(tag, newValues);
@Override protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException { Collection<Future<?>> result = super.initializeOp(hconf); ObjectInspector[] inputOIs = new ObjectInspector[conf.getTagLength()]; inputOIs[tag] = inputObjInspectors[0]; conf.setTagOrder(new Byte[]{ tag }); htsOperator.setConf(conf); htsOperator.initialize(hconf, inputOIs); return result; }
SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator) op; SparkHashTableSinkDesc hashTableSinkDesc = hashTableSinkOp.getConf(); BucketMapJoinContext original = hashTableSinkDesc.getBucketMapjoinContext(); if (original != null && original.getBucketFileNameMapping() == bucketMJCxt.getBucketFileNameMapping()) {
String dumpFilePrefix = conf.getDumpFilePrefix(); Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName); FileSystem fs = path.getFileSystem(htsOperator.getConfiguration());
@Override public void process(Object row, int tag) throws HiveException { // Ignore the tag passed in, which should be 0, not what we want htsOperator.process(row, conf.getTag()); }
SparkHashTableSinkDesc hashTableSinkDesc = new SparkHashTableSinkDesc(mjDesc); SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator)OperatorFactory.get( tableScan.getCompilationOpContext(), hashTableSinkDesc); if (valueIndex != null) { List<ExprNodeDesc> newValues = new ArrayList<ExprNodeDesc>(); List<ExprNodeDesc> values = hashTableSinkDesc.getExprs().get(tag); for (int index = 0; index < values.size(); index++) { if (valueIndex[index] < 0) { hashTableSinkDesc.getExprs().put(tag, newValues); tableScanParents.add(tableScan); hashTableSinkOp.setParentOperators(tableScanParents); hashTableSinkOp.getConf().setTag(tag);
SparkHashTableSinkDesc hashTableSinkDesc = new SparkHashTableSinkDesc(mjDesc); SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator) OperatorFactory.get(hashTableSinkDesc); if (valueIndex != null) { List<ExprNodeDesc> newValues = new ArrayList<ExprNodeDesc>(); List<ExprNodeDesc> values = hashTableSinkDesc.getExprs().get(tag); for (int index = 0; index < values.size(); index++) { if (valueIndex[index] < 0) { hashTableSinkDesc.getExprs().put(tag, newValues);
SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator) op; SparkHashTableSinkDesc hashTableSinkDesc = hashTableSinkOp.getConf(); BucketMapJoinContext original = hashTableSinkDesc.getBucketMapjoinContext(); if (original != null && original.getBucketFileNameMapping() == bucketMJCxt.getBucketFileNameMapping()) {
String dumpFilePrefix = conf.getDumpFilePrefix(); Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName); FileSystem fs = path.getFileSystem(htsOperator.getConfiguration());
@Override public void process(Object row, int tag) throws HiveException { // Ignore the tag passed in, which should be 0, not what we want htsOperator.process(row, conf.getTag()); }
SparkHashTableSinkDesc hashTableSinkDesc = new SparkHashTableSinkDesc(mjDesc); SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator) OperatorFactory.get( mapJoinOp.getCompilationOpContext(), hashTableSinkDesc); if (valueIndex != null) { List<ExprNodeDesc> newValues = new ArrayList<ExprNodeDesc>(); List<ExprNodeDesc> values = hashTableSinkDesc.getExprs().get(tag); for (int index = 0; index < values.size(); index++) { if (valueIndex[index] < 0) { hashTableSinkDesc.getExprs().put(tag, newValues); hashTableSinkOp.getConf().setTag(tag); return true;
private boolean validateSparkHashTableSinkOperator(SparkHashTableSinkOperator op) { SparkHashTableSinkDesc desc = op.getConf(); byte tag = desc.getTag(); // it's essentially a MapJoinDesc List<ExprNodeDesc> filterExprs = desc.getFilters().get(tag); List<ExprNodeDesc> keyExprs = desc.getKeys().get(tag); List<ExprNodeDesc> valueExprs = desc.getExprs().get(tag); return validateExprNodeDesc(filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER) && validateExprNodeDesc(keyExprs, "Key") && validateExprNodeDesc(valueExprs, "Value"); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); ObjectInspector[] inputOIs = new ObjectInspector[conf.getTagLength()]; byte tag = conf.getTag(); inputOIs[tag] = inputObjInspectors[0]; conf.setTagOrder(new Byte[]{ tag }); int dfsMaxReplication = hconf.getInt(DFS_REPLICATION_MAX, minReplication); // minReplication value should not cross the value of dfs.replication.max minReplication = Math.min(minReplication, dfsMaxReplication); htsOperator.setConf(conf); htsOperator.initialize(hconf, inputOIs); }
SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator) op; SparkHashTableSinkDesc hashTableSinkDesc = hashTableSinkOp.getConf(); BucketMapJoinContext original = hashTableSinkDesc.getBucketMapjoinContext(); if (original != null && original.getBucketFileNameMapping() == bucketMJCxt.getBucketFileNameMapping()) {
String dumpFilePrefix = conf.getDumpFilePrefix(); Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName); FileSystem fs = path.getFileSystem(htsOperator.getConfiguration());
@Override public void closeOp(boolean abort) throws HiveException { try { MapJoinPersistableTableContainer[] mapJoinTables = htsOperator.mapJoinTables; byte tag = conf.getTag(); if (mapJoinTables == null || mapJoinTables.length < tag || mapJoinTables[tag] == null) { LOG.debug("mapJoinTable is null"); } else if (abort) { if (LOG.isDebugEnabled()) { LOG.debug("Aborting, skip dumping side-table for tag: " + tag); } } else { String method = PerfLogger.SPARK_FLUSH_HASHTABLE + getName(); perfLogger.PerfLogBegin(CLASS_NAME, method); try { flushToFile(mapJoinTables[tag], tag); } finally { perfLogger.PerfLogEnd(CLASS_NAME, method); } } super.closeOp(abort); } catch (HiveException e) { throw e; } catch (Exception e) { throw new HiveException(e); } }
SparkHashTableSinkDesc hashTableSinkDesc = new SparkHashTableSinkDesc(mjDesc); SparkHashTableSinkOperator hashTableSinkOp = (SparkHashTableSinkOperator) OperatorFactory.get( mapJoinOp.getCompilationOpContext(), hashTableSinkDesc); if (valueIndex != null) { List<ExprNodeDesc> newValues = new ArrayList<ExprNodeDesc>(); List<ExprNodeDesc> values = hashTableSinkDesc.getExprs().get(tag); for (int index = 0; index < values.size(); index++) { if (valueIndex[index] < 0) { hashTableSinkDesc.getExprs().put(tag, newValues); hashTableSinkOp.getConf().setTag(tag); return true;