public ApplicationId createExtClientAppId() { // Note that we cannot allow users to provide app ID, since providing somebody else's appId // would give one LLAP token (and splits) for that app ID. If we could verify it somehow // (YARN token? nothing we can do in an UDF), we could get it from client already running on // YARN. As such, the clients running on YARN will have two app IDs to be aware of. return ApplicationId.newInstance(startTime, appIdCounter.incrementAndGet()); }
/** * This function converts an application in form of a String into a ApplicationId. * * @param appIDStr The application id in form of a string * @return the application id as an instance of ApplicationId class. */ private static ApplicationId getApplicationIDFromString(String appIDStr) { String[] parts = appIDStr.split("_"); if (parts.length < 3) { throw new IllegalStateException("the application id found is not valid. application id: " + appIDStr); } long timestamp = Long.parseLong(parts[1]); int id = Integer.parseInt(parts[2]); return ApplicationId.newInstance(timestamp, id); }
private static String getBaseLocation(String jobIdString, int dagId, String user) { // $x/$user/appcache/$appId/${dagId}/output/$mapId // TODO: Once Shuffle is out of NM, this can use MR APIs to convert // between App and Job String parts[] = jobIdString.split("_"); Preconditions.checkArgument(parts.length == 3, "Invalid jobId. Expecting 3 parts"); final ApplicationId appID = ApplicationId.newInstance(Long.parseLong(parts[1]), Integer.parseInt(parts[2])); final String baseStr = USERCACHE_CONSTANT + "/" + user + "/" + APPCACHE_CONSTANT + "/" + ConverterUtils.toString(appID) + "/" + dagId + "/output" + "/"; return baseStr; }
@Override public void readFields(DataInput in) throws IOException { long appIdTs = in.readLong(); int appIdId = in.readInt(); fakeAppId = ApplicationId.newInstance(appIdTs, appIdId); token = new Token<>(); token.readFields(in); creationTime = in.readLong(); vertexParallelism = in.readInt(); int vertexSpecBytes = in.readInt(); if (vertexSpecBytes > 0) { vertexSpec = new byte[vertexSpecBytes]; in.readFully(vertexSpec); } int vertexSpecSignBytes = in.readInt(); if (vertexSpecSignBytes > 0) { vertexSpecSignature = new byte[vertexSpecSignBytes]; in.readFully(vertexSpecSignature); } }
public ContainerFactory(ApplicationAttemptId appAttemptId, long appIdLong) { this.nextId = new AtomicLong(1); ApplicationId appId = ApplicationId.newInstance(appIdLong, appAttemptId.getApplicationId().getId()); this.customAppAttemptId = ApplicationAttemptId.newInstance(appId, appAttemptId.getAttemptId()); }
private static Container mockContainer(String host, int port, int containerId, Resource resource) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance( ApplicationId.newInstance(System.currentTimeMillis(), 1), 1 ), containerId ); when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(resource); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }
ApplicationId id = ApplicationId.newInstance( System.nanoTime(), (int)(System.nanoTime() % 100000)); Token<JobTokenIdentifier> token = createAmsToken(id);
targetFileSystem, targetDir, ApplicationId.newInstance(0, 0), remotePaths, localResources,
/** * Tests that the cluster retrieval of a finished YARN application fails. */ @Test(expected = ClusterRetrieveException.class) public void testClusterClientRetrievalOfFinishedYarnApplication() throws Exception { final ApplicationId applicationId = ApplicationId.newInstance(System.currentTimeMillis(), 42); final ApplicationReport applicationReport = createApplicationReport( applicationId, YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED); final YarnClient yarnClient = new TestingYarnClient(Collections.singletonMap(applicationId, applicationReport)); final YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnClient.init(yarnConfiguration); yarnClient.start(); final TestingAbstractYarnClusterDescriptor clusterDescriptor = new TestingAbstractYarnClusterDescriptor( new Configuration(), yarnConfiguration, temporaryFolder.newFolder().getAbsolutePath(), yarnClient, false); try { clusterDescriptor.retrieve(applicationId); } finally { clusterDescriptor.close(); } }
private SubmitWorkRequestProto createRequest(int fragmentNumber, int numSelfAndUpstreamTasks, int numSelfAndUpstreamComplete, int dagStartTime, int attemptStartTime, int withinDagPriority, String dagName) throws IOException { ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35); return LlapDaemonTestUtils.buildSubmitProtoRequest(fragmentNumber, appId.toString(), dagId.getId(), vId.getId(), dagName, dagStartTime, attemptStartTime, numSelfAndUpstreamTasks, numSelfAndUpstreamComplete, withinDagPriority, new Credentials()); }
ContainerId.newInstance( ApplicationAttemptId.newInstance( ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), i));
output1Bb.putInt(0, 400); ApplicationId appId = ApplicationId.newInstance(1000, 100); TezDAGID tezDagId = TezDAGID.getInstance(appId, 300); TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
List<OutputSpec> outputSpecList = Lists.newArrayList(outputSpec1, outputSpec2); ApplicationId appId = ApplicationId.newInstance(1000, 100); TezDAGID tezDagId = TezDAGID.getInstance(appId, 300); TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400);
long currentAttemptStartTime, int withinDagPriority, String dagName, boolean isGuaranteed) { ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35);
public static ApplicationId toAppID(String prefix, String s, Iterator<String> it) { if (!it.hasNext() || !it.next().equals(prefix)) { throwParseException(sjoin(prefix, ID), s); } shouldHaveNext(prefix, s, it); ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), Integer.parseInt(it.next())); return appId; }
private ApplicationAttemptId createAppAttemptId( long clusterTimeStamp, int id, int attemptId) { ApplicationId appId = ApplicationId.newInstance(clusterTimeStamp, id); return ApplicationAttemptId.newInstance(appId, attemptId); }
public static ContainerId newContainerId(int appId, int appAttemptId, long timestamp, long containerId) { ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, appAttemptId); return ContainerId.newContainerId(applicationAttemptId, containerId); } }
private static ContainerId createContainerId(int id) { return ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), id); }
private TestV2TimelineClient createTimelineClient(YarnConfiguration config) { ApplicationId id = ApplicationId.newInstance(0, 0); TestV2TimelineClient tc = new TestV2TimelineClient(id); tc.init(config); tc.start(); return tc; }
@Before public void setup() { conf.set(YarnConfiguration.RM_CLUSTER_ID, "subclusterId"); callback = new CountingCallback(); attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); uam = new TestableUnmanagedApplicationManager(conf, attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true); }