DAG dag = DAG.create(dagName); LOG.debug("DagInfo: " + dagInfo); dag.setDAGInfo(dagInfo); dag.setCredentials(conf.getCredentials()); setAccessControlsForCurrentUser(dag, queryPlan.getQueryId(), conf); vertexArray[i++] = workToVertex.get(v); VertexGroup group = dag.createVertexGroup(w.getName(), vertexArray); workToVertex.get(v), work.getEdgeProperty(w, v), v, work); dag.addEdge(e); wx.setConf(TEZ_MEMORY_RESERVE_FRACTION, Double.toString(frac)); dag.addVertex(wx); utils.addCredentials(w, dag); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_CREATE_VERTEX + w.getName()); dag.addEdge(e);
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
/** * Adds any necessary resources that must be localized in each vertex to the DAG. */ void addExtraResourcesToDag(TezSessionState session, DAG dag, String[] inputOutputJars, Map<String,LocalResource> inputOutputLocalResources) throws Exception { if (!session.hasResources(inputOutputJars)) { if (null != inputOutputLocalResources) { dag.addTaskLocalFiles(inputOutputLocalResources); } } }
@Test public void testCredentialsNotOverwritten() throws Exception { final UserGroupInformation testUser = UserGroupInformation.createUserForTesting("test_user", new String[0]); final DagUtils dagUtils = DagUtils.getInstance(); Credentials originalCredentials = new Credentials(); final Text testTokenAlias = new Text("my_test_token"); @SuppressWarnings("unchecked") Token<? extends TokenIdentifier> testToken = mock(Token.class); originalCredentials.addToken(testTokenAlias, testToken); Credentials testUserCredentials = new Credentials(); testUser.addCredentials(testUserCredentials); final BaseWork work = mock(BaseWork.class); final DAG dag = DAG.create("test_credentials_dag"); dag.setCredentials(originalCredentials); testUser.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { dagUtils.addCredentials(work, dag); return null; } }); Token<? extends TokenIdentifier> actualToken = dag.getCredentials().getToken(testTokenAlias); assertEquals(testToken, actualToken); }
private DAG createDAG(int numGenTasks, int totalSourceDataSize, int numFetcherTasks) { int bytesPerSource = totalSourceDataSize / numGenTasks; LOG.info("DataPerSourceTask(bytes)=" + bytesPerSource); ByteBuffer payload = ByteBuffer.allocate(4); payload.putInt(0, bytesPerSource); Vertex broadcastVertex = Vertex.create("DataGen", ProcessorDescriptor.create(InputGenProcessor.class.getName()) .setUserPayload(UserPayload.create(payload)), numGenTasks); Vertex fetchVertex = Vertex.create("FetchVertex", ProcessorDescriptor.create(InputFetchProcessor.class.getName()), numFetcherTasks); UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(NullWritable.class .getName(), IntWritable.class.getName()).setCompression(false, null, null).build(); DAG dag = DAG.create("BroadcastLoadGen"); dag.addVertex(broadcastVertex).addVertex(fetchVertex).addEdge( Edge.create(broadcastVertex, fetchVertex, edgeConf.createDefaultBroadcastEdgeProperty())); return dag; }
DAG dag = DAG.create( getStepDisplayName( initializedConfig.getInt( "cascading.display.id.truncate", Util.ID_LENGTH ) ) ); dag.addTaskLocalFiles( allLocalResources ); dag.addVertex( vertex ); dag.addEdge( Edge.create( sourceVertex, targetVertex, edgeProperty ) ); VertexGroup vertexGroup = dag.createVertexGroup( edgeTargetFlowNode.getID(), sourceVertices.toArray( new Vertex[ sourceVertices.size() ] ) ); dag.addEdge( GroupInputEdge.create( vertexGroup, targetVertex, edgeProperty, inputDescriptor ) );
vertices.add(finalReduceVertex); DAG dag = DAG.create("OrderedWordCount" + dagIndex); for (int i = 0; i < vertices.size(); ++i) { dag.addVertex(vertices.get(i)); HashPartitioner.class.getName()).setFromConfiguration(iReduceStageConf) .configureInput().useLegacyInput().done().build(); dag.addEdge( Edge.create(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"), edgeConf1.createDefaultEdgeProperty())); HashPartitioner.class.getName()).setFromConfiguration(finalReduceConf) .configureInput().useLegacyInput().done().build(); dag.addEdge( Edge.create(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"), edgeConf2.createDefaultEdgeProperty()));
DAG dag = DAG.create(work.getName()); dag.setCredentials(job.getCredentials()); work.getVertexType(mapWork), DagUtils.createTezLrMap(appJarLr, null)); String vertexName = wx.getName(); dag.addVertex(wx); utils.addCredentials(mapWork, dag);
private DAG createDAG(TezConfiguration conf, int numTasks, int maxSleepTimeMillis, int payloadSize, String mode) throws IOException { Map<String, LocalResource> localResourceMap = new HashMap<String, LocalResource>(); UserPayload payload = createUserPayload(conf, maxSleepTimeMillis, payloadSize, mode, localResourceMap); Vertex vertex = Vertex.create("RPCLoadVertex", ProcessorDescriptor.create(RPCSleepProcessor.class.getName()).setUserPayload( payload), numTasks).addTaskLocalFiles(localResourceMap); return DAG.create("RPCLoadGen").addVertex(vertex); }
if (!tezOp.isVertexGroup()) { to = newVertex(tezOp); dag.addVertex(to); } else { dag.addEdge(edge); } else { Vertex from = dag.getVertex(pred.getOperatorKey().toString()); if (tezOp.isVertexGroup()) { groupMembers[i] = from; EdgeProperty prop = newEdge(pred, tezOp, false); Edge edge = Edge.create(from, to, prop); dag.addEdge(edge); VertexGroup vertexGroup = dag.createVertexGroup(groupName, groupMembers); tezOp.getVertexGroupInfo().setVertexGroup(vertexGroup); POStore store = tezOp.getVertexGroupInfo().getStore(); OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), dag.getCredentials()));
@VisibleForTesting DAG createDag(TezConfiguration tezConf, Path lhs, Path rhs, int numPartitions) throws IOException { DAG dag = DAG.create(getDagName()); if (getDefaultExecutionContext() != null) { dag.setExecutionContext(getDefaultExecutionContext()); Edge e2 = Edge.create(rhsVertex, joinValidateVertex, edgeConf.createDefaultEdgeProperty()); dag.addVertex(lhsVertex).addVertex(rhsVertex).addVertex(joinValidateVertex).addEdge(e1) .addEdge(e2); return dag;
public DAG buildDAG(TezPlanContainerNode tezPlanNode, Map<String, LocalResource> localResources) throws IOException, YarnException { DAG tezDag = DAG.create(tezPlanNode.getOperatorKey().toString()); tezDag.setCredentials(tezPlanNode.getTezOperPlan().getCredentials()); TezDagBuilder dagBuilder = new TezDagBuilder(pigContext, tezPlanNode.getTezOperPlan(), tezDag, localResources); dagBuilder.visit(); dagBuilder.avoidContainerReuseIfInputSplitInDisk(); disableDAGRecovery = dagBuilder.shouldDisableDAGRecovery(); return tezDag; }
@Test public void testSubmit() throws Exception { DAG dag = DAG.create("test"); task.submit(conf, dag, Ref.from(sessionState)); // validate close/reopen verify(sessionState, times(1)).reopen(); verify(session, times(2)).submitDAG(any(DAG.class)); }
private long hiveInputRecordsFromOtherVertices(String vertexName) { List<Vertex> inputVerticesList = dag.getVertex(vertexName).getInputVertices(); long result = 0; for (Vertex inputVertex : inputVerticesList) { long inputVertexRecords = hiveInputRecordsFromTezCounters(vertexName, inputVertex.getName()); if (inputVertexRecords < 0) { inputVertexRecords = hiveInputRecordsFromHiveCounters(inputVertex.getName()); } result += inputVertexRecords; } return result; }
try { tezClient = TezSessionManager.getClient(conf, requestAMResources, dag.getCredentials(), tezJobConf); log.info("Submitting DAG " + dag.getName()); dagClient = tezClient.submitDAG(dag); appId = tezClient.getAppMasterApplicationId(); log.info("Submitted DAG " + dag.getName() + ". Application id: " + appId); } catch (Exception e) { if (tezClient != null) {
( "UnknownApp_" + System.currentTimeMillis() + dagIndex ) : ( tezSession.getAppMasterApplicationId().toString() + "_" + dagIndex); dag.setCallerContext(CallerContext.create("Tez", callerId, callerType, "TestOrderedWordCount Job")); LOG.info("Pre-warming Session"); PreWarmVertex preWarmVertex = PreWarmVertex.create("PreWarm", preWarmNumContainers, dag .getVertex("initialmap").getTaskResource()); preWarmVertex.addTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles()); preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment()); preWarmVertex.setTaskLaunchCmdOpts(dag.getVertex("initialmap").getTaskLaunchCmdOpts());
"Tez Example DAG: " + dag.getName()); ApplicationId appId = tezClientInternal.getAppMasterApplicationId(); if (hadoopShim == null) { callerContext.setCallerIdAndType(appId.toString(), "TezExampleApplication"); dag.setCallerContext(callerContext);
ApplicationSubmissionContext appContext = TezClientUtils .createApplicationSubmissionContext( appId, dag, dag.getName(), amConfig, tezJarResources, credentials, usingTezArchiveDeploy, apiVersionInfo, servicePluginsDescriptor, javaOptsChecker); String callerContextStr = ""; if (dag.getCallerContext() != null) { callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString(); + ", dagName=" + dag.getName() + callerContextStr);
private static void setAccessControlsForCurrentUser(DAG dag, String queryId, Configuration conf) throws IOException { String user = SessionState.getUserFromAuthenticator(); UserGroupInformation loginUserUgi = UserGroupInformation.getLoginUser(); String loginUser = loginUserUgi == null ? null : loginUserUgi.getShortUserName(); boolean addHs2User = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVETEZHS2USERACCESS); // Temporarily re-using the TEZ AM View ACLs property for individual dag access control. // Hive may want to setup it's own parameters if it wants to control per dag access. // Setting the tez-property per dag should work for now. String viewStr = Utilities.getAclStringWithHiveModification(conf, TezConfiguration.TEZ_AM_VIEW_ACLS, addHs2User, user, loginUser); String modifyStr = Utilities.getAclStringWithHiveModification(conf, TezConfiguration.TEZ_AM_MODIFY_ACLS, addHs2User, user, loginUser); if (LOG.isDebugEnabled()) { LOG.debug("Setting Tez DAG access for queryId={} with viewAclString={}, modifyStr={}", queryId, viewStr, modifyStr); } // set permissions for current user on DAG DAGAccessControls ac = new DAGAccessControls(viewStr, modifyStr); dag.setAccessControls(ac); }