/** * Get the profile id for the given program, given arguments for a run. All non-workflow program types will use the * native profile. Workflow program types will use whatever profile is specified in its arguments, or the native * profile if none is specified. * * @param programId program to get the profile for * @param args arguments for a program run * @return the profile id for the program run */ public static ProfileId getProfileIdForProgram(ProgramId programId, Map<String, String> args) { if (programId.getType() == ProgramType.WORKFLOW) { return getProfileIdFromArgs(programId.getNamespaceId(), args).orElse(ProfileId.NATIVE); } return ProfileId.NATIVE; }
public ProgramDescriptor(ProgramId programId, ApplicationSpecification appSpec) { this(programId, appSpec, Artifacts.toArtifactId(programId.getNamespaceId(), appSpec.getArtifactId())); }
public ProgramDescriptor(ProgramId programId, ApplicationSpecification appSpec) { this(programId, appSpec, Artifacts.toArtifactId(programId.getNamespaceId(), appSpec.getArtifactId())); }
private static WorkerSpecification getWorkerSpecOrFail(ProgramId id, ApplicationSpecification appSpec) { WorkerSpecification workerSpecification = appSpec.getWorkers().get(id.getProgram()); if (workerSpecification == null) { throw new NoSuchElementException("no such worker @ namespace id: " + id.getNamespaceId() + ", app id: " + id.getApplication() + ", worker id: " + id.getProgram()); } return workerSpecification; }
private static WorkerSpecification getWorkerSpecOrFail(ProgramId id, ApplicationSpecification appSpec) { WorkerSpecification workerSpecification = appSpec.getWorkers().get(id.getProgram()); if (workerSpecification == null) { throw new NoSuchElementException("no such worker @ namespace id: " + id.getNamespaceId() + ", app id: " + id.getApplication() + ", worker id: " + id.getProgram()); } return workerSpecification; }
/** * Get the profile id for the given program, given arguments for a run. All non-workflow program types will use the * native profile. Workflow program types will use whatever profile is specified in its arguments, or the native * profile if none is specified. * * @param programId program to get the profile for * @param args arguments for a program run * @return the profile id for the program run */ public static ProfileId getProfileIdForProgram(ProgramId programId, Map<String, String> args) { if (isProfileAllowed(programId.getType())) { return getProfileIdFromArgs(programId.getNamespaceId(), args).orElse(ProfileId.NATIVE); } return ProfileId.NATIVE; }
@Override public Void call() throws Exception { String namespace = String.format("%s.%s", programId.getApplication(), programId.getProgram()); NamespaceId namespaceId = programId.getNamespaceId(); for (Map.Entry<String, Collection<Long>> entry : streamGroups.asMap().entrySet()) { streamConsumerFactory.dropAll(namespaceId.stream(entry.getKey()), namespace, entry.getValue()); } queueAdmin.dropAllForFlow(programId.getParent().flow(programId.getEntityName())); return null; } });
/** * Constructor. This should be called from the service handler via * the {@link SparkHttpServiceContext#getPluginContext()} method. */ public DefaultSparkHttpServicePluginContext(SparkRuntimeContext runtimeContext) throws IOException { this.runtimeContext = runtimeContext; this.pluginInstantiator = createPluginsInstantiator(runtimeContext); this.pluginConfigurer = new DefaultPluginConfigurer(runtimeContext.getArtifactId(), runtimeContext.getProgram().getId().getNamespaceId(), pluginInstantiator, runtimeContext.getPluginFinder()); this.extraPlugins = null; }
/** * Constructor. This should be called from the service handler via * the {@link SparkHttpServiceContext#getPluginContext()} method. */ public DefaultSparkHttpServicePluginContext(SparkRuntimeContext runtimeContext) throws IOException { this.runtimeContext = runtimeContext; this.pluginInstantiator = createPluginsInstantiator(runtimeContext); this.pluginConfigurer = new DefaultPluginConfigurer(runtimeContext.getArtifactId(), runtimeContext.getProgram().getId().getNamespaceId(), pluginInstantiator, runtimeContext.getPluginFinder()); this.extraPlugins = null; }
/** * Constructor. This should be called from the service handler via * the {@link SparkHttpServiceContext#getPluginContext()} method. */ public DefaultSparkHttpServicePluginContext(SparkRuntimeContext runtimeContext) throws IOException { this.runtimeContext = runtimeContext; this.pluginInstantiator = createPluginsInstantiator(runtimeContext); this.pluginConfigurer = new DefaultPluginConfigurer(runtimeContext.getArtifactId(), runtimeContext.getProgram().getId().getNamespaceId(), pluginInstantiator, runtimeContext.getPluginFinder()); this.extraPlugins = null; }
@Override public FlowSpecification setFlowletInstances(ProgramId id, String flowletId, int count) { Preconditions.checkArgument(count > 0, "Cannot change number of flowlet instances to %s", count); LOG.trace("Setting flowlet instances: namespace: {}, application: {}, flow: {}, flowlet: {}, " + "new instances count: {}", id.getNamespace(), id.getApplication(), id.getProgram(), flowletId, count); FlowSpecification flowSpec = Transactionals.execute(transactional, context -> { AppMetadataStore metaStore = getAppMetadataStore(context); ApplicationSpecification appSpec = getAppSpecOrFail(metaStore, id); ApplicationSpecification newAppSpec = updateFlowletInstancesInAppSpec(appSpec, id, flowletId, count); metaStore.updateAppSpec(id.getNamespace(), id.getApplication(), id.getVersion(), newAppSpec); return appSpec.getFlows().get(id.getProgram()); }); LOG.trace("Set flowlet instances: namespace: {}, application: {}, flow: {}, flowlet: {}, instances now: {}", id.getNamespaceId(), id.getApplication(), id.getProgram(), flowletId, count); return flowSpec; }
/** * Creates a temporary directory through the {@link LocationFactory} provided to this class. */ private Location createTempLocationDirectory() throws IOException { ProgramId programId = context.getProgram().getId(); String tempLocationName = String.format("%s/%s.%s.%s.%s.%s", cConf.get(Constants.AppFabric.TEMP_DIR), programId.getType().name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram(), context.getRunId().getId()); Location location = locationFactory.get(programId.getNamespaceId()).append(tempLocationName); location.mkdirs(); return location; }
/** * Creates a temporary directory through the {@link LocationFactory} provided to this class. */ private Location createTempLocationDirectory() throws IOException { ProgramId programId = context.getProgram().getId(); String tempLocationName = String.format("%s/%s.%s.%s.%s.%s", cConf.get(Constants.AppFabric.TEMP_DIR), programId.getType().name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram(), context.getRunId().getId()); Location location = locationFactory.get(programId.getNamespaceId()).append(tempLocationName); location.mkdirs(); return location; }
/** * Sets the number of instances that a worker will run on. * * @param instances number of instances for the worker to run on * @throws IOException if a network error occurred * @throws NotFoundException if the application or worker could not be found * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server */ public void setWorkerInstances(ProgramId worker, int instances) throws IOException, NotFoundException, UnauthenticatedException, UnauthorizedException { URL url = config.resolveNamespacedURLV3(worker.getNamespaceId(), String.format("apps/%s/workers/%s/instances", worker.getApplication(), worker.getProgram())); HttpRequest request = HttpRequest.put(url).withBody(GSON.toJson(new Instances(instances))).build(); HttpResponse response = restClient.execute(request, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new NotFoundException(worker); } }
/** * Stops a program. * * @param programId the program to stop * @throws IOException if a network error occurred * @throws ProgramNotFoundException if the program with specified name could not be found * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server */ public void stop(ProgramId programId) throws IOException, ProgramNotFoundException, UnauthenticatedException, UnauthorizedException { String path = String.format("apps/%s/versions/%s/%s/%s/stop", programId.getApplication(), programId.getVersion(), programId.getType().getCategoryName(), programId.getProgram()); URL url = config.resolveNamespacedURLV3(programId.getNamespaceId(), path); HttpResponse response = restClient.execute(HttpMethod.POST, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new ProgramNotFoundException(programId); } }
/** * Checks if a user service is available by hitting the availability endpoint. */ private void checkAvailability() throws IOException, UnauthenticatedException, NotFoundException { URL url = clientConfig.resolveNamespacedURLV3(programId.getNamespaceId(), String.format("apps/%s/versions/%s/%s/%s/available", programId.getApplication(), programId.getVersion(), programId.getType().getCategoryName(), programId.getProgram())); HttpResponse response = restClient.execute(HttpMethod.GET, url, clientConfig.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND, HttpURLConnection.HTTP_BAD_REQUEST, HttpURLConnection.HTTP_UNAVAILABLE); if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new NotFoundException(programId); } if (response.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE) { throw new ServiceUnavailableException(programId.toString()); } } }
@Override public void setServiceInstances(ProgramId id, int instances) { Preconditions.checkArgument(instances > 0, "Cannot change number of service instances to %s", instances); Transactionals.execute(transactional, context -> { AppMetadataStore metaStore = getAppMetadataStore(context); ApplicationSpecification appSpec = getAppSpecOrFail(metaStore, id); ServiceSpecification serviceSpec = getServiceSpecOrFail(id, appSpec); // Create a new spec copy from the old one, except with updated instances number serviceSpec = new ServiceSpecification(serviceSpec.getClassName(), serviceSpec.getName(), serviceSpec.getDescription(), serviceSpec.getHandlers(), serviceSpec.getResources(), instances, serviceSpec.getPlugins()); ApplicationSpecification newAppSpec = replaceServiceSpec(appSpec, id.getProgram(), serviceSpec); metaStore.updateAppSpec(id.getNamespace(), id.getApplication(), id.getVersion(), newAppSpec); }); LOG.trace("Setting program instances: namespace: {}, application: {}, service: {}, new instances count: {}", id.getNamespaceId(), id.getApplication(), id.getProgram(), instances); }
@Override public void setServiceInstances(ProgramId id, int instances) { Preconditions.checkArgument(instances > 0, "Cannot change number of service instances to %s", instances); Transactionals.execute(transactional, context -> { AppMetadataStore metaStore = getAppMetadataStore(context); ApplicationSpecification appSpec = getAppSpecOrFail(metaStore, id); ServiceSpecification serviceSpec = getServiceSpecOrFail(id, appSpec); // Create a new spec copy from the old one, except with updated instances number serviceSpec = new ServiceSpecification(serviceSpec.getClassName(), serviceSpec.getName(), serviceSpec.getDescription(), serviceSpec.getHandlers(), serviceSpec.getResources(), instances, serviceSpec.getPlugins()); ApplicationSpecification newAppSpec = replaceServiceSpec(appSpec, id.getProgram(), serviceSpec); metaStore.updateAppSpec(id.getNamespace(), id.getApplication(), id.getVersion(), newAppSpec); }); LOG.trace("Setting program instances: namespace: {}, application: {}, service: {}, new instances count: {}", id.getNamespaceId(), id.getApplication(), id.getProgram(), instances); }
@Override public void setWorkerInstances(ProgramId id, int instances) { Preconditions.checkArgument(instances > 0, "Cannot change number of worker instances to %s", instances); Transactionals.execute(transactional, context -> { AppMetadataStore metaStore = getAppMetadataStore(context); ApplicationSpecification appSpec = getAppSpecOrFail(metaStore, id); WorkerSpecification workerSpec = getWorkerSpecOrFail(id, appSpec); WorkerSpecification newSpecification = new WorkerSpecification(workerSpec.getClassName(), workerSpec.getName(), workerSpec.getDescription(), workerSpec.getProperties(), workerSpec.getDatasets(), workerSpec.getResources(), instances, workerSpec.getPlugins()); ApplicationSpecification newAppSpec = replaceWorkerInAppSpec(appSpec, id, newSpecification); metaStore.updateAppSpec(id.getNamespace(), id.getApplication(), id.getVersion(), newAppSpec); }); LOG.trace("Setting program instances: namespace: {}, application: {}, worker: {}, new instances count: {}", id.getNamespaceId(), id.getApplication(), id.getProgram(), instances); }
@Test public void testConcurrentStopStart() { // Two programs that start/stop at same time // Should have two run history. ProgramId programId = new ProgramId("account1", "concurrentApp", ProgramType.MAPREDUCE, "concurrentMR"); long now = System.currentTimeMillis(); long nowSecs = TimeUnit.MILLISECONDS.toSeconds(now); RunId run1 = RunIds.generate(now - 10000); ArtifactId artifactId = programId.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); setStartAndRunning(programId.run(run1.getId()), artifactId); RunId run2 = RunIds.generate(now - 10000); setStartAndRunning(programId.run(run2.getId()), artifactId); store.setStop(programId.run(run1.getId()), nowSecs, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); store.setStop(programId.run(run2.getId()), nowSecs, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId)); Map<ProgramRunId, RunRecordMeta> historymap = store.getRuns(programId, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE); Assert.assertEquals(2, historymap.size()); }