@SuppressWarnings("nls") private Operator genLimitPlan(String dest, QB qb, Operator input, int offset, int limit) throws SemanticException { // A map-only job can be optimized - instead of converting it to a // map-reduce job, we can have another map // job to do the same to avoid the cost of sorting in the map-reduce phase. // A better approach would be to // write into a local file and then have a map-only job. // Add the limit operator to get the value fields RowResolver inputRR = opParseCtx.get(input).getRowResolver(); LimitDesc limitDesc = new LimitDesc(offset, limit); globalLimitCtx.setLastReduceLimitDesc(limitDesc); Operator limitMap = putOpInsertMap(OperatorFactory.getAndMakeChild( limitDesc, new RowSchema(inputRR.getColumnInfos()), input), inputRR); if (LOG.isDebugEnabled()) { LOG.debug("Created LimitOperator Plan for clause: " + dest + " row schema: " + inputRR.toString()); } return limitMap; }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); limit = conf.getLimit(); leastRow = conf.getLeastRows(); offset = (conf.getOffset() == null) ? 0 : conf.getOffset(); currCount = 0; isMap = hconf.getBoolean("mapred.task.is.map", true); }
Integer tempOffset = tempGlobalLimitDesc.getOffset(); globalLimitCtx.enableOpt(tempGlobalLimitDesc.getLimit(), (tempOffset == null) ? 0 : tempOffset); Integer tempOffset = tempGlobalLimitDesc.getOffset(); globalLimitCtx.enableOpt(tempGlobalLimitDesc.getLimit(), (tempOffset == null) ? 0 : tempOffset);
@Override protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException { Collection<Future<?>> result = super.initializeOp(hconf); limit = conf.getLimit(); leastRow = conf.getLeastRows(); currCount = 0; isMap = hconf.getBoolean("mapred.task.is.map", true); return result; }
globalLimitCtx.getLastReduceLimitDesc().setLeastRows(globalLimitCtx.getGlobalLimit());
@Explain(vectorization = Vectorization.OPERATOR, displayName = "Limit Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED }) public LimitOperatorExplainVectorization getLimitVectorization() { VectorLimitDesc vectorLimitDesc = (VectorLimitDesc) getVectorDesc(); if (vectorLimitDesc == null) { return null; } return new LimitOperatorExplainVectorization(this, vectorLimitDesc); }
globalLimitCtx.getLastReduceLimitDesc().setLeastRows(globalLimitCtx.getGlobalLimit()); List<ExecDriver> mrTasks = Utilities.getMRTasks(rootTasks); for (ExecDriver tsk : mrTasks) {
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { if(!(((LimitOperator)nd).getConf().getLimit() == 0)) { return null; } LOG.info("Found Limit 0 TableScan. " + nd); ((WalkerCtx)procCtx).convertMetadataOnly(); return null; }
@SuppressWarnings("nls") private Operator genLimitPlan(String dest, QB qb, Operator input, int offset, int limit) throws SemanticException { // A map-only job can be optimized - instead of converting it to a // map-reduce job, we can have another map // job to do the same to avoid the cost of sorting in the map-reduce phase. // A better approach would be to // write into a local file and then have a map-only job. // Add the limit operator to get the value fields RowResolver inputRR = opParseCtx.get(input).getRowResolver(); LimitDesc limitDesc = new LimitDesc(offset, limit); globalLimitCtx.setLastReduceLimitDesc(limitDesc); Operator limitMap = putOpInsertMap(OperatorFactory.getAndMakeChild( limitDesc, new RowSchema(inputRR.getColumnInfos()), input), inputRR); if (LOG.isDebugEnabled()) { LOG.debug("Created LimitOperator Plan for clause: " + dest + " row schema: " + inputRR.toString()); } return limitMap; }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); limit = conf.getLimit(); leastRow = conf.getLeastRows(); offset = (conf.getOffset() == null) ? 0 : conf.getOffset(); currCount = 0; isMap = hconf.getBoolean("mapred.task.is.map", true); }
Integer tempOffset = tempGlobalLimitDesc.getOffset(); globalLimitCtx.enableOpt(tempGlobalLimitDesc.getLimit(), (tempOffset == null) ? 0 : tempOffset); Integer tempOffset = tempGlobalLimitDesc.getOffset(); globalLimitCtx.enableOpt(tempGlobalLimitDesc.getLimit(), (tempOffset == null) ? 0 : tempOffset);
globalLimitCtx.getLastReduceLimitDesc().setLeastRows(globalLimitCtx.getGlobalLimit()); List<ExecDriver> mrTasks = Utilities.getMRTasks(rootTasks); for (ExecDriver tsk : mrTasks) {
int limit = RexLiteral.intValue(sortRel.fetch); int offset = sortRel.offset == null ? 0 : RexLiteral.intValue(sortRel.offset); LimitDesc limitDesc = new LimitDesc(offset,limit); ArrayList<ColumnInfo> cinfoLst = createColInfos(resultOp); resultOp = OperatorFactory.getAndMakeChild(limitDesc, new RowSchema(cinfoLst), resultOp);
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LimitOperator limitOp = (LimitOperator)nd; if (!(limitOp.getConf().getLimit() == 0)) { return null; } Set<TableScanOperator> tsOps = ((WalkerCtx) procCtx).getMayBeMetadataOnlyTableScans(); if (tsOps != null) { for (Iterator<TableScanOperator> tsOp = tsOps.iterator(); tsOp.hasNext();) { if (!isNullOpPresentInAllBranches(tsOp.next(), limitOp)) { tsOp.remove(); } } } LOG.debug("Found Limit 0 TableScan. {}", nd); ((WalkerCtx)procCtx).convertMetadataOnly(); return null; } }
private void validateVectorLimitOperator(int limit, int batchSize, int expectedBatchSize) throws HiveException { @SuppressWarnings("unchecked") FakeVectorRowBatchFromObjectIterables frboi = new FakeVectorRowBatchFromObjectIterables( batchSize, new String[] {"tinyint", "double"}, Arrays.asList(new Object[] {1, 2, 3, 4}), Arrays.asList(new Object[] {323.0, 34.5, null, 89.3})); // Get next batch VectorizedRowBatch vrb = frboi.produceNextBatch(); // Create limit desc with limit value LimitDesc ld = new LimitDesc(limit); VectorLimitDesc vectorDesc = new VectorLimitDesc(); VectorLimitOperator lo = new VectorLimitOperator( new CompilationOpContext(), ld, null, vectorDesc); lo.initialize(new Configuration(), null); // Process the batch lo.process(vrb, 0); // Verify batch size Assert.assertEquals(vrb.size, expectedBatchSize); } }
private void handleTopLimit(Operator<?> rootOp) { if (rootOp instanceof LimitOperator) { // this can happen only on top most limit, not while visiting Limit Operator // since that can be within subquery. this.semanticAnalyzer.getQB().getParseInfo().setOuterQueryLimit(((LimitOperator) rootOp).getConf().getLimit()); } }