@Test public void testSubmit() throws Exception { DAG dag = DAG.create("test"); task.submit(conf, dag, Ref.from(sessionState)); // validate close/reopen verify(sessionState, times(1)).reopen(); verify(session, times(2)).submitDAG(any(DAG.class)); }
DAG dag = DAG.create(dagName);
DAG dag = DAG.create(work.getName()); dag.setCredentials(job.getCredentials());
DAG dag = DAG.create(dagName);
public InputSplit[] getSplits(JobConf job, int numSplits, TezWork work, Schema schema) throws IOException { DAG dag = DAG.create(work.getName()); dag.setCredentials(job.getCredentials());
DAG dag = DAG.create( getStepDisplayName( initializedConfig.getInt( "cascading.display.id.truncate", Util.ID_LENGTH ) ) );
@Test public void testCredentialsNotOverwritten() throws Exception { final UserGroupInformation testUser = UserGroupInformation.createUserForTesting("test_user", new String[0]); final DagUtils dagUtils = DagUtils.getInstance(); Credentials originalCredentials = new Credentials(); final Text testTokenAlias = new Text("my_test_token"); @SuppressWarnings("unchecked") Token<? extends TokenIdentifier> testToken = mock(Token.class); originalCredentials.addToken(testTokenAlias, testToken); Credentials testUserCredentials = new Credentials(); testUser.addCredentials(testUserCredentials); final BaseWork work = mock(BaseWork.class); final DAG dag = DAG.create("test_credentials_dag"); dag.setCredentials(originalCredentials); testUser.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { dagUtils.addCredentials(work, dag); return null; } }); Token<? extends TokenIdentifier> actualToken = dag.getCredentials().getToken(testTokenAlias); assertEquals(testToken, actualToken); }
private DAG createDAG(FileSystem fs, TezConfiguration tezConf, Map<String, LocalResource> localResources, Path stagingDir, String inputPath, String outputPath) throws IOException { DAG dag = DAG.create("UnionExample");
private DAG createDAG(TezConfiguration conf, int numTasks, int maxSleepTimeMillis, int payloadSize, String mode) throws IOException { Map<String, LocalResource> localResourceMap = new HashMap<String, LocalResource>(); UserPayload payload = createUserPayload(conf, maxSleepTimeMillis, payloadSize, mode, localResourceMap); Vertex vertex = Vertex.create("RPCLoadVertex", ProcessorDescriptor.create(RPCSleepProcessor.class.getName()).setUserPayload( payload), numTasks).addTaskLocalFiles(localResourceMap); return DAG.create("RPCLoadGen").addVertex(vertex); }
private DAG createDag(TezConfiguration tezConf, Path largeOutPath, Path smallOutPath, Path expectedOutputPath, int numTasks, long largeOutSize, long smallOutSize) throws IOException { long largeOutSizePerTask = largeOutSize / numTasks; long smallOutSizePerTask = smallOutSize / numTasks; DAG dag = DAG.create("JoinDataGen"); Vertex genDataVertex = Vertex.create("datagen", ProcessorDescriptor.create( GenDataProcessor.class.getName()).setUserPayload( UserPayload.create(ByteBuffer.wrap(GenDataProcessor.createConfiguration(largeOutSizePerTask, smallOutSizePerTask)))), numTasks); genDataVertex.addDataSink(STREAM_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, largeOutPath.toUri().toString()).build()); genDataVertex.addDataSink(HASH_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, smallOutPath.toUri().toString()).build()); genDataVertex.addDataSink(EXPECTED_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, expectedOutputPath.toUri().toString()).build()); dag.addVertex(genDataVertex); return dag; }
private DAG createDAG(int numGenTasks, int totalSourceDataSize, int numFetcherTasks) { int bytesPerSource = totalSourceDataSize / numGenTasks; LOG.info("DataPerSourceTask(bytes)=" + bytesPerSource); ByteBuffer payload = ByteBuffer.allocate(4); payload.putInt(0, bytesPerSource); Vertex broadcastVertex = Vertex.create("DataGen", ProcessorDescriptor.create(InputGenProcessor.class.getName()) .setUserPayload(UserPayload.create(payload)), numGenTasks); Vertex fetchVertex = Vertex.create("FetchVertex", ProcessorDescriptor.create(InputFetchProcessor.class.getName()), numFetcherTasks); UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(NullWritable.class .getName(), IntWritable.class.getName()).setCompression(false, null, null).build(); DAG dag = DAG.create("BroadcastLoadGen"); dag.addVertex(broadcastVertex).addVertex(fetchVertex).addEdge( Edge.create(broadcastVertex, fetchVertex, edgeConf.createDefaultBroadcastEdgeProperty())); return dag; }
DAG dag = DAG.create(jobName);
DAG dag = org.apache.tez.dag.api.DAG.create(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX + "_" + preWarmDAGCounter++); dag.addVertex(preWarmVertex);
EdgeProperty edgeProperty = edgeConf.createDefaultCustomEdgeProperty(edgeManagerDescriptor); return DAG.create("CrossProduct").addVertex(v1).addVertex(v2).addVertex(v3) .addEdge(Edge.create(v1, v3, edgeProperty)).addEdge(Edge.create(v2, v3, edgeProperty));
DAG dag = DAG.create(dagName); dag.addVertex(tokenizerVertex) .addVertex(summationVertex)
DAG dag = DAG.create("WordCount"); dag.addVertex(tokenizerVertex) .addVertex(summationVertex)
public DAG buildDAG(TezPlanContainerNode tezPlanNode, Map<String, LocalResource> localResources) throws IOException, YarnException { DAG tezDag = DAG.create(tezPlanNode.getOperatorKey().toString()); tezDag.setCredentials(tezPlanNode.getTezOperPlan().getCredentials()); TezDagBuilder dagBuilder = new TezDagBuilder(pigContext, tezPlanNode.getTezOperPlan(), tezDag, localResources); dagBuilder.visit(); dagBuilder.avoidContainerReuseIfInputSplitInDisk(); disableDAGRecovery = dagBuilder.shouldDisableDAGRecovery(); return tezDag; }
broadcastEdgeProperty = broadcastEdgeConf.createDefaultBroadcastEdgeProperty(); return DAG.create("CartesianProduct") .addVertex(v1).addVertex(v2).addVertex(v3).addVertex(v4) .addEdge(Edge.create(v1, v4, cpEdgeProperty))
.setFromConfiguration(tezConf).build(); DAG dag = DAG.create("BroadcastAndOneToOneExample"); dag.addVertex(inputVertex) .addVertex(broadcastVertex)
@VisibleForTesting DAG createDag(TezConfiguration tezConf, Path lhs, Path rhs, int numPartitions) throws IOException { DAG dag = DAG.create(getDagName()); if (getDefaultExecutionContext() != null) { dag.setExecutionContext(getDefaultExecutionContext());