public Object getFieldValue(_Fields field) { switch (field) { case QUERY_ID: return getQueryId(); case QUERY_TYPE: return getQueryType(); case QUERY_ATTRIBUTES: return getQueryAttributes(); case QUERY_COUNTERS: return getQueryCounters(); case STAGE_GRAPH: return getStageGraph(); case STAGE_LIST: return getStageList(); case DONE: return isDone(); case STARTED: return isStarted(); } throw new IllegalStateException(); }
@Override public void read(org.apache.thrift.protocol.TProtocol prot, QueryPlan struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { { org.apache.thrift.protocol.TList _list133 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); struct.queries = new ArrayList<Query>(_list133.size); Query _elem134; for (int _i135 = 0; _i135 < _list133.size; ++_i135) { _elem134 = new Query(); _elem134.read(iprot); struct.queries.add(_elem134); } } struct.setQueriesIsSet(true); } if (incoming.get(1)) { struct.done = iprot.readBool(); struct.setDoneIsSet(true); } if (incoming.get(2)) { struct.started = iprot.readBool(); struct.setStartedIsSet(true); } } }
@Override public boolean equals(Object that) { if (that == null) return false; if (that instanceof Query) return this.equals((Query)that); return false; }
query.setStarted(started.contains(query.getQueryId())); query.setDone(done.contains(query.getQueryId())); if (query.getStageList() != null) { for (org.apache.hadoop.hive.ql.plan.api.Stage stage : query .getStageList()) { if (stage.getStageId() == null) { continue;
query.setStageGraph(new org.apache.hadoop.hive.ql.plan.api.Graph()); query.getStageGraph().setNodeType(NodeType.STAGE); stage.setStageId(task.getId()); stage.setStageType(task.getType()); query.addToStageList(stage); query.getStageGraph().addToAdjacencyList(childEntry); query.getStageGraph().addToAdjacencyList(listEntry); } else if (task.getChildTasks() != null) { org.apache.hadoop.hive.ql.plan.api.Adjacency entry = query.getStageGraph().addToAdjacencyList(entry);
public QueryPlan(String queryString, BaseSemanticAnalyzer sem, Long startTime, String queryId, HiveOperation operation, Schema resultSchema) { this.queryString = queryString; rootTasks = new ArrayList<Task<? extends Serializable>>(sem.getAllRootTasks()); reducerTimeStatsPerJobList = new ArrayList<ReducerTimeStatsPerJob>(); fetchTask = sem.getFetchTask(); // Note that inputs and outputs can be changed when the query gets executed inputs = sem.getAllInputs(); outputs = sem.getAllOutputs(); linfo = sem.getLineageInfo(); tableAccessInfo = sem.getTableAccessInfo(); columnAccessInfo = sem.getColumnAccessInfo(); idToTableNameMap = new HashMap<String, String>(sem.getIdToTableNameMap()); this.queryId = queryId == null ? makeQueryId() : queryId; query = new org.apache.hadoop.hive.ql.plan.api.Query(); query.setQueryId(this.queryId); query.putToQueryAttributes("queryString", this.queryString); queryProperties = sem.getQueryProperties(); queryStartTime = startTime; this.operation = operation; this.autoCommitValue = sem.getAutoCommitValue(); this.resultSchema = resultSchema; }
public org.apache.hadoop.hive.ql.plan.api.Query getQueryPlan() throws IOException { if (query.getStageGraph() == null) { populateQueryPlan(); } extractCounters(); updateCountersInQueryPlan(); return query; }
public Query deepCopy() { return new Query(this); }
protected AtlasEntity getHiveProcessEntity(List<AtlasEntity> inputs, List<AtlasEntity> outputs) throws Exception { AtlasEntity ret = new AtlasEntity(HIVE_TYPE_PROCESS); HookContext hookContext = getHiveContext(); String queryStr = hookContext.getQueryPlan().getQueryStr(); if (queryStr != null) { queryStr = queryStr.toLowerCase().trim(); } ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, getQualifiedName(inputs, outputs)); ret.setAttribute(ATTRIBUTE_INPUTS, getObjectIds(inputs)); ret.setAttribute(ATTRIBUTE_OUTPUTS, getObjectIds(outputs)); ret.setAttribute(ATTRIBUTE_NAME, queryStr); ret.setAttribute(ATTRIBUTE_OPERATION_TYPE, hookContext.getOperationName()); ret.setAttribute(ATTRIBUTE_START_TIME, hookContext.getQueryPlan().getQueryStartTime()); ret.setAttribute(ATTRIBUTE_END_TIME, System.currentTimeMillis()); ret.setAttribute(ATTRIBUTE_USER_NAME, getUserName()); ret.setAttribute(ATTRIBUTE_QUERY_TEXT, queryStr); ret.setAttribute(ATTRIBUTE_QUERY_ID, hookContext.getQueryPlan().getQuery().getQueryId()); ret.setAttribute(ATTRIBUTE_QUERY_PLAN, "Not Supported"); ret.setAttribute(ATTRIBUTE_RECENT_QUERIES, Collections.singletonList(queryStr)); return ret; }
event.setQueryStr(hookContext.getQueryPlan().getQueryStr()); event.setQueryStartTime(hookContext.getQueryPlan().getQueryStartTime()); event.setQueryType(hookContext.getQueryPlan().getQueryPlan().getQueryType()); event.setLineageInfo(hookContext.getLinfo());
query.setStarted(started.contains(query.getQueryId())); query.setDone(done.contains(query.getQueryId())); if (query.getStageList() != null) { for (org.apache.hadoop.hive.ql.plan.api.Stage stage : query .getStageList()) { if (stage.getStageId() == null) { continue;
query.setStageGraph(new org.apache.hadoop.hive.ql.plan.api.Graph()); query.getStageGraph().setNodeType(NodeType.STAGE); stage.setStageId(task.getId()); stage.setStageType(task.getType()); query.addToStageList(stage); query.getStageGraph().addToAdjacencyList(childEntry); query.getStageGraph().addToAdjacencyList(listEntry); } else if (task.getChildTasks() != null) { org.apache.hadoop.hive.ql.plan.api.Adjacency entry = query.getStageGraph().addToAdjacencyList(entry);
public QueryPlan(String queryString, BaseSemanticAnalyzer sem, Long startTime, String queryId, HiveOperation operation, Schema resultSchema) { this.queryString = queryString; rootTasks = new ArrayList<Task<? extends Serializable>>(sem.getAllRootTasks()); reducerTimeStatsPerJobList = new ArrayList<ReducerTimeStatsPerJob>(); fetchTask = sem.getFetchTask(); // Note that inputs and outputs can be changed when the query gets executed inputs = sem.getAllInputs(); outputs = sem.getAllOutputs(); linfo = sem.getLineageInfo(); tableAccessInfo = sem.getTableAccessInfo(); columnAccessInfo = sem.getColumnAccessInfo(); idToTableNameMap = new HashMap<String, String>(sem.getIdToTableNameMap()); this.queryId = queryId == null ? makeQueryId() : queryId; query = new org.apache.hadoop.hive.ql.plan.api.Query(); query.setQueryId(this.queryId); query.putToQueryAttributes("queryString", this.queryString); queryProperties = sem.getQueryProperties(); queryStartTime = startTime; this.operation = operation; this.autoCommitValue = sem.getAutoCommitValue(); this.resultSchema = resultSchema; // TODO: all this ACID stuff should be in some sub-object this.acidResourcesInQuery = sem.hasTransactionalInQuery(); this.acidSinks = sem.getAcidFileSinks(); this.acidDdlDesc = sem.getAcidDdlDesc(); this.acidAnalyzeTable = sem.getAcidAnalyzeTable(); this.cboInfo = sem.getCboInfo(); }
public static void setWorkflowAdjacencies(Configuration conf, QueryPlan plan) { try { Graph stageGraph = plan.getQueryPlan().getStageGraph(); if (stageGraph == null) { return; } List<Adjacency> adjList = stageGraph.getAdjacencyList(); if (adjList == null) { return; } for (Adjacency adj : adjList) { List<String> children = adj.getChildren(); if (children == null || children.isEmpty()) { return; } conf.setStrings("mapreduce.workflow.adjacency."+adj.getNode(), children.toArray(new String[children.size()])); } } catch (IOException e) { } }
public Query deepCopy() { return new Query(this); }
public Object getFieldValue(_Fields field) { switch (field) { case QUERY_ID: return getQueryId(); case QUERY_TYPE: return getQueryType(); case QUERY_ATTRIBUTES: return getQueryAttributes(); case QUERY_COUNTERS: return getQueryCounters(); case STAGE_GRAPH: return getStageGraph(); case STAGE_LIST: return getStageList(); case DONE: return isDone(); case STARTED: return isStarted(); } throw new IllegalStateException(); }
query.setStarted(started.contains(query.getQueryId())); query.setDone(done.contains(query.getQueryId())); if (query.getStageList() != null) { for (org.apache.hadoop.hive.ql.plan.api.Stage stage : query .getStageList()) { stage.setStarted(started.contains(stage.getStageId())); stage.setStageCounters(counters.get(stage.getStageId()));
query.setStageGraph(new org.apache.hadoop.hive.ql.plan.api.Graph()); query.getStageGraph().setNodeType(NodeType.STAGE); stage.setStageId(task.getId()); stage.setStageType(task.getType()); query.addToStageList(stage); query.getStageGraph().addToAdjacencyList(childEntry); query.getStageGraph().addToAdjacencyList(listEntry); } else if (task.getChildTasks() != null) { org.apache.hadoop.hive.ql.plan.api.Adjacency entry = query.getStageGraph().addToAdjacencyList(entry);
public QueryPlan(String queryString, BaseSemanticAnalyzer sem) { this.queryString = queryString; rootTasks = new ArrayList<Task<? extends Serializable>>(); rootTasks.addAll(sem.getRootTasks()); fetchTask = sem.getFetchTask(); // Note that inputs and outputs can be changed when the query gets executed inputs = sem.getInputs(); outputs = sem.getOutputs(); linfo = sem.getLineageInfo(); idToTableNameMap = new HashMap<String, String>(sem.getIdToTableNameMap()); queryId = makeQueryId(); query = new org.apache.hadoop.hive.ql.plan.api.Query(); query.setQueryId(queryId); query.putToQueryAttributes("queryString", this.queryString); counters = new HashMap<String, HashMap<String, Long>>(); done = new HashSet<String>(); started = new HashSet<String>(); }