@Test(expected = IllegalArgumentException.class) public void testMergePathWithInvalidMoveWorkThrowsException() { final Path condInputPath = new Path("s3a://bucket/scratch/-ext-10000"); final MoveWork mockWork = mock(MoveWork.class); final LineageState lineageState = new LineageState(); when(mockWork.getLoadMultiFilesWork()).thenReturn(new LoadMultiFilesDesc()); GenMapRedUtils.mergeMovePaths(condInputPath, mockWork, lineageState); }
public HookContext(QueryPlan queryPlan, QueryState queryState, Map<String, ContentSummary> inputPathToContentSummary, String userName, String ipAddress, String hiveInstanceAddress, String operationId, String sessionId, String threadId, boolean isHiveServerQuery, PerfLogger perfLogger, QueryInfo queryInfo) throws Exception { this.queryPlan = queryPlan; this.queryState = queryState; this.conf = queryState.getConf(); this.inputPathToContentSummary = inputPathToContentSummary; completeTaskList = new ArrayList<TaskRunner>(); inputs = queryPlan.getInputs(); outputs = queryPlan.getOutputs(); ugi = Utils.getUGI(); linfo = queryState.getLineageState().getLineageInfo(); depMap = queryState.getLineageState().getIndex(); this.userName = userName; this.ipAddress = ipAddress; this.hiveInstanceAddress = hiveInstanceAddress; this.operationId = operationId; this.sessionId = sessionId; this.threadId = threadId; this.isHiveServerQuery = isHiveServerQuery; this.perfLogger = perfLogger; this.queryInfo = queryInfo; }
queryState.getLineageState().setLineage(tbd.getSourcePath(), dc, table.getCols());
(SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo() : null), Utils.getUGI()); SessionState.get().getLineageState().clear(); SessionState.get().getHiveHistory().logPlanProgress(plan); } catch (Exception e) {
(SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo() : null), Utils.getUGI());
private void handleLineage(LoadTableDesc ltd, Operator output) throws SemanticException { if (ltd != null) { queryState.getLineageState() .mapDirToOp(ltd.getSourcePath(), output); } else if ( queryState.getCommandType().equals(HiveOperation.CREATETABLE_AS_SELECT.getOperationName())) { Path tlocation = null; String tName = Utilities.getDbTableName(tableDesc.getTableName())[1]; try { Warehouse wh = new Warehouse(conf); tlocation = wh.getDefaultTablePath(db.getDatabase(tableDesc.getDatabaseName()), tName, tableDesc.isExternal()); } catch (MetaException|HiveException e) { throw new SemanticException(e); } queryState.getLineageState() .mapDirToOp(tlocation, output); } }
/** * Release some resources after a query is executed * while keeping the result around. */ private void releaseResources() { releasePlan(); releaseDriverContext(); if (SessionState.get() != null) { SessionState.get().getLineageState().clear(); } }
Index index = pctx.getQueryState().getLineageState().getIndex(); if (index == null) { index = new Index();
.mapDirToFop(ltd.getSourceDir(), (FileSinkOperator)output);
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // Create the lineage context LineageCtx lCtx = new LineageCtx(pctx); Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", "TS%"), OpProcFactory.getTSProc()); opRules.put(new RuleRegExp("R2", "SCR%"), OpProcFactory.getTransformProc()); opRules.put(new RuleRegExp("R3", "UDTF%"), OpProcFactory.getTransformProc()); opRules.put(new RuleRegExp("R4", "SEL%"), OpProcFactory.getSelProc()); opRules.put(new RuleRegExp("R5", "GBY%"), OpProcFactory.getGroupByProc()); opRules.put(new RuleRegExp("R6", "UNION%"), OpProcFactory.getUnionProc()); opRules.put(new RuleRegExp("R7", "JOIN%|MAPJOIN%"), OpProcFactory.getJoinProc()); opRules.put(new RuleRegExp("R8", "RS%"), OpProcFactory.getReduceSinkProc()); opRules.put(new RuleRegExp("R9", "LVJ%"), OpProcFactory.getLateralViewJoinProc()); // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(OpProcFactory.getDefaultProc(), opRules, lCtx); GraphWalker ogw = new PreOrderWalker(disp); // Create a list of topop nodes ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); // Transfer the index from the lineage context to the session state. if (SessionState.get() != null) { SessionState.get().getLineageState().setIndex(lCtx.getIndex()); } return pctx; }
public HookContext(QueryPlan queryPlan, HiveConf conf) throws Exception{ this.queryPlan = queryPlan; this.conf = conf; completeTaskList = new ArrayList<TaskRunner>(); inputs = queryPlan.getInputs(); outputs = queryPlan.getOutputs(); ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); linfo= null; if(SessionState.get() != null){ linfo = SessionState.get().getLineageState().getLineageInfo(); } }
.mapDirToOp(new Path(createVwDesc.getViewName()), sinkOp);
public int close() { lDrvState.stateLock.lock(); try { releaseDriverContext(); if (lDrvState.driverState == DriverState.COMPILING || lDrvState.driverState == DriverState.EXECUTING || lDrvState.driverState == DriverState.INTERRUPT) { lDrvState.driverState = DriverState.INTERRUPT; return 0; } releasePlan(); releaseFetchTask(); releaseResStream(); releaseContext(); lDrvState.driverState = DriverState.CLOSED; } finally { lDrvState.stateLock.unlock(); } if (SessionState.get() != null) { SessionState.get().getLineageState().clear(); } return 0; }
SessionState.get().getLineageState().getIndex() : new Index();
.mapDirToFop(ltd.getSourcePath(), (FileSinkOperator) output);
SessionState.get().getLineageState().setIndex(lCtx.getIndex());
break; queryState.getLineageState().setLineage(tbd.getSourcePath(), dc, tableCols);
public SessionState(HiveConf conf, String userName) { this.sessionConf = conf; this.userName = userName; this.registry = new Registry(false); if (LOG.isDebugEnabled()) { LOG.debug("SessionState user: " + userName); } isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT); ls = new LineageState(); resourceMaps = new ResourceMaps(); // Must be deterministic order map for consistent q-test output across Java versions overriddenConfigurations = new LinkedHashMap<String, String>(); overriddenConfigurations.putAll(HiveConf.getConfSystemProperties()); // if there isn't already a session name, go ahead and create it. if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) { conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId()); } // Using system classloader as the parent. Using thread context // classloader as parent can pollute the session. See HIVE-11878 parentLoader = SessionState.class.getClassLoader(); // Make sure that each session has its own UDFClassloader. For details see {@link UDFClassLoader} final ClassLoader currentLoader = Utilities.createUDFClassLoader((URLClassLoader) parentLoader, new String[]{}); this.sessionConf.setClassLoader(currentLoader); resourceDownloader = new ResourceDownloader(conf, HiveConf.getVar(conf, ConfVars.DOWNLOADED_RESOURCES_DIR)); }
public HookContext(QueryPlan queryPlan, QueryState queryState, Map<String, ContentSummary> inputPathToContentSummary, String userName, String ipAddress, String hiveInstanceAddress, String operationId, String sessionId, String threadId, boolean isHiveServerQuery, PerfLogger perfLogger) throws Exception { this.queryPlan = queryPlan; this.queryState = queryState; this.conf = queryState.getConf(); this.inputPathToContentSummary = inputPathToContentSummary; completeTaskList = new ArrayList<TaskRunner>(); inputs = queryPlan.getInputs(); outputs = queryPlan.getOutputs(); ugi = Utils.getUGI(); linfo= null; depMap = null; if(SessionState.get() != null){ linfo = SessionState.get().getLineageState().getLineageInfo(); depMap = SessionState.get().getLineageState().getIndex(); } this.userName = userName; this.ipAddress = ipAddress; this.hiveInstanceAddress = hiveInstanceAddress; this.operationId = operationId; this.sessionId = sessionId; this.threadId = threadId; this.isHiveServerQuery = isHiveServerQuery; this.perfLogger = perfLogger; }
public HookContext(QueryPlan queryPlan, HiveConf conf, Map<String, ContentSummary> inputPathToContentSummary, String userName, String ipAddress, String operationId) throws Exception { this.queryPlan = queryPlan; this.conf = conf; this.inputPathToContentSummary = inputPathToContentSummary; completeTaskList = new ArrayList<TaskRunner>(); inputs = queryPlan.getInputs(); outputs = queryPlan.getOutputs(); ugi = Utils.getUGI(); linfo= null; if(SessionState.get() != null){ linfo = SessionState.get().getLineageState().getLineageInfo(); } this.userName = userName; this.ipAddress = ipAddress; this.operationId = operationId; }