public PluginNotExistsException(ArtifactId artifactId, String type, String name) { super("plugin", String.format("%s:%s:%s:%s:%s", artifactId.getNamespace(), type, name, artifactId.getArtifact(), artifactId.getVersion())); } }
public PluginNotExistsException(ArtifactId artifactId, String type, String name) { super("plugin", String.format("%s:%s:%s:%s:%s", artifactId.getNamespace(), type, name, artifactId.getArtifact(), artifactId.getVersion())); } }
protected static void setupStreamingArtifacts(ArtifactId artifactId, Class<?> appClass) throws Exception { // add the app artifact addAppArtifact(artifactId, appClass, StreamingSource.class.getPackage().getName(), Transform.class.getPackage().getName(), SparkCompute.class.getPackage().getName(), PipelineConfigurable.class.getPackage().getName()); // add plugins artifact addPluginArtifact(new ArtifactId(artifactId.getNamespace(), artifactId.getArtifact() + "-mocks", "1.0.0"), artifactId, STREAMING_MOCK_PLUGINS, co.cask.cdap.etl.mock.spark.streaming.MockSource.class, co.cask.cdap.etl.mock.batch.MockSink.class, co.cask.cdap.etl.mock.spark.streaming.MockSink.class, DoubleTransform.class, AllErrorTransform.class, IdentityTransform.class, IntValueFilterTransform.class, StringValueFilterTransform.class, StringValueFilterCompute.class, Window.class, NullFieldSplitterTransform.class, NullAlertTransform.class); }
protected static void setupBatchArtifacts(ArtifactId artifactId, Class<?> appClass) throws Exception { // add the app artifact addAppArtifact(artifactId, appClass, BatchSource.class.getPackage().getName(), Action.class.getPackage().getName(), Condition.class.getPackage().getName(), PipelineConfigurable.class.getPackage().getName(), "org.apache.avro.mapred", "org.apache.avro", "org.apache.avro.generic", "org.apache.avro.io"); // add plugins artifact addPluginArtifact(new ArtifactId(artifactId.getNamespace(), artifactId.getArtifact() + "-mocks", "1.0.0"), artifactId, BATCH_MOCK_PLUGINS, co.cask.cdap.etl.mock.batch.MockSource.class, co.cask.cdap.etl.mock.batch.MockSink.class, MockExternalSource.class, MockExternalSink.class, DoubleTransform.class, AllErrorTransform.class, IdentityTransform.class, IntValueFilterTransform.class, StringValueFilterTransform.class, FieldCountAggregator.class, IdentityAggregator.class, FieldsPrefixTransform.class, StringValueFilterCompute.class, NodeStatesAction.class, LookupTransform.class, NullFieldSplitterTransform.class, NullAlertTransform.class, IncapableSource.class, IncapableSink.class); }
boolean isUsableBy(co.cask.cdap.proto.id.ArtifactId artifactId) { if (usableBy == null) { return true; } return usableBy.getNamespace().equals(artifactId.getNamespace()) && usableBy.getName().equals(artifactId.getArtifact()) && usableBy.versionIsInRange(new ArtifactVersion(artifactId.getVersion())); } }
boolean isUsableBy(co.cask.cdap.proto.id.ArtifactId artifactId) { if (usableBy == null) { return true; } return usableBy.getNamespace().equals(artifactId.getNamespace()) && usableBy.getName().equals(artifactId.getArtifact()) && usableBy.versionIsInRange(new ArtifactVersion(artifactId.getVersion())); } }
private Map<String, String> getArtifactProperties(ArtifactId artifact) throws URISyntaxException, IOException { URL endpoint = getEndPoint(String.format("%s/namespaces/%s/artifacts/%s/versions/%s/properties", Constants.Gateway.API_VERSION_3, artifact.getNamespace(), artifact.getArtifact(), artifact.getVersion())).toURL(); return getResults(endpoint, MAP_STRING_STRING_TYPE); }
@Override public ArtifactManager addPluginArtifact(ArtifactId artifactId, Set<ArtifactRange> parents, @Nullable Set<PluginClass> additionalPlugins, Class<?> pluginClass, Class<?>... pluginClasses) throws Exception { Manifest manifest = createManifest(pluginClass, pluginClasses); final Location appJar = PluginJarHelper.createPluginJar(locationFactory, manifest, pluginClass, pluginClasses); artifactClient.add( Ids.namespace(artifactId.getNamespace()), artifactId.getArtifact(), appJar::getInputStream, artifactId.getVersion(), parents, additionalPlugins ); appJar.delete(); return new RemoteArtifactManager(clientConfig, restClient, artifactId); }
private void deleteArtifact(ArtifactId artifact, boolean deleteArtifact, @Nullable String property) throws Exception { String path = String.format("artifacts/%s/versions/%s/", artifact.getArtifact(), artifact.getVersion()); if (!deleteArtifact) { path += property == null ? "properties" : String.format("properties/%s", property); } HttpResponse response = doDelete(getVersionedAPIPath(path, artifact.getNamespace())); Assert.assertEquals(200, response.getResponseCode()); }
/** * @return {@link ArtifactInfo} of the given artifactId in the given scope */ ArtifactInfo getArtifact(ArtifactId artifactId, ArtifactScope scope) throws URISyntaxException, IOException { // get /artifacts/{name}/versions/{version}?scope={scope} URL endpoint = getEndPoint(String.format("%s/namespaces/%s/artifacts/%s/versions/%s%s", Constants.Gateway.API_VERSION_3, artifactId.getNamespace(), artifactId.getArtifact(), artifactId.getVersion(), getScopeQuery(scope))).toURL(); return getResults(endpoint, ArtifactInfo.class); }
private Set<String> getPluginTypes(ArtifactId artifactId, ArtifactScope scope) throws URISyntaxException, IOException { URL endpoint = getEndPoint(String.format("%s/namespaces/%s/artifacts/%s/versions/%s/extensions%s", Constants.Gateway.API_VERSION_3, artifactId.getNamespace(), artifactId.getArtifact(), artifactId.getVersion(), getScopeQuery(scope)) ).toURL(); return getResults(endpoint, PLUGIN_TYPES_TYPE); }
@Override public Map.Entry<ArtifactDescriptor, PluginClass> findPlugin(NamespaceId pluginNamespaceId, ArtifactId parentArtifactId, String pluginType, String pluginName, PluginSelector selector) throws PluginNotExistsException { try { ArtifactRange parentRange = new ArtifactRange(parentArtifactId.getNamespace(), parentArtifactId.getArtifact(), new ArtifactVersion(parentArtifactId.getVersion()), true, new ArtifactVersion(parentArtifactId.getVersion()), true); return artifactRepository.findPlugin(pluginNamespaceId, parentRange, pluginType, pluginName, selector); } catch (IOException | ArtifactNotFoundException e) { // If there is error accessing artifact store or if the parent artifact is missing, just propagate throw Throwables.propagate(e); } } }
private Set<PluginSummary> getPluginSummaries(ArtifactId artifactId, String pluginType, ArtifactScope scope) throws URISyntaxException, IOException { URL endpoint = getEndPoint(String.format("%s/namespaces/%s/artifacts/%s/versions/%s/extensions/%s%s", Constants.Gateway.API_VERSION_3, artifactId.getNamespace(), artifactId.getArtifact(), artifactId.getVersion(), pluginType, getScopeQuery(scope))) .toURL(); return getResults(endpoint, PLUGIN_SUMMARIES_TYPE); }
@Override public Map.Entry<ArtifactDescriptor, PluginClass> findPlugin(NamespaceId pluginNamespaceId, ArtifactId parentArtifactId, String pluginType, String pluginName, PluginSelector selector) throws PluginNotExistsException { try { ArtifactRange parentRange = new ArtifactRange(parentArtifactId.getNamespace(), parentArtifactId.getArtifact(), new ArtifactVersion(parentArtifactId.getVersion()), true, new ArtifactVersion(parentArtifactId.getVersion()), true); return artifactRepository.findPlugin(pluginNamespaceId, parentRange, pluginType, pluginName, selector); } catch (IOException | ArtifactNotFoundException e) { // If there is error accessing artifact store or if the parent artifact is missing, just propagate throw Throwables.propagate(e); } } }
private Set<PluginInfo> getPluginInfos(ArtifactId artifactId, String pluginType, String pluginName, ArtifactScope scope) throws URISyntaxException, IOException { URL endpoint = getEndPoint( String.format("%s/namespaces/%s/artifacts/%s/versions/%s/extensions/%s/plugins/%s%s", Constants.Gateway.API_VERSION_3, artifactId.getNamespace(), artifactId.getArtifact(), artifactId.getVersion(), pluginType, pluginName, getScopeQuery(scope))) .toURL(); return getResults(endpoint, PLUGIN_INFOS_TYPE); }
@Test public void testPluginRequirements() throws Exception { // add a system artifact ArtifactId systemId = NamespaceId.SYSTEM.artifact("app", "1.0.0"); addAppAsSystemArtifacts(); Set<ArtifactRange> parents = Sets.newHashSet(new ArtifactRange( systemId.getNamespace(), systemId.getArtifact(), new ArtifactVersion(systemId.getVersion()), true, new ArtifactVersion(systemId.getVersion()), true)); Manifest manifest = new Manifest(); manifest.getMainAttributes().put(ManifestFields.EXPORT_PACKAGE, InspectionApp.class.getPackage().getName()); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("inspection", "1.0.0"); Assert.assertEquals(HttpResponseStatus.OK.code(), addPluginArtifact(Id.Artifact.fromEntityId(artifactId), InspectionApp.class, manifest, parents) .getResponseCode()); Set<PluginClass> plugins = getArtifact(artifactId).getClasses().getPlugins(); // Only four plugins which does not have transactions as requirement should be visible. Assert.assertEquals(4, plugins.size()); Set<String> actualPluginClassNames = plugins.stream().map(PluginClass::getClassName).collect(Collectors.toSet()); Set<String> expectedPluginClassNames = ImmutableSet.of(InspectionApp.AppPlugin.class.getName(), InspectionApp.EmptyRequirementPlugin.class.getName(), InspectionApp.SingleEmptyRequirementPlugin.class.getName(), InspectionApp.NonTransactionalPlugin.class.getName()); Assert.assertEquals(expectedPluginClassNames, actualPluginClassNames); } }
@Test public void testPluginInvalid() throws Exception { // invalid plugin class json array which does not have a name Set<PluginClass> pluginClasses = Collections.singleton(new PluginClass("type", null, "desc1", "className1", "cnfg", Collections.emptyMap(), Collections.emptySet(), Requirements.EMPTY)); // add a system artifact ArtifactId systemId = NamespaceId.SYSTEM.artifact("app", "1.0.0"); addAppAsSystemArtifacts(); Set<ArtifactRange> parents = Sets.newHashSet(new ArtifactRange( systemId.getNamespace(), systemId.getArtifact(), new ArtifactVersion(systemId.getVersion()), true, new ArtifactVersion(systemId.getVersion()), true)); Manifest manifest = new Manifest(); manifest.getMainAttributes().put(ManifestFields.EXPORT_PACKAGE, InspectionApp.class.getPackage().getName()); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("inspection", "1.0.0"); // should fail with bad request as the plugin class does not have a name Assert.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), addPluginArtifact(Id.Artifact.fromEntityId(artifactId), InspectionApp.class, manifest, parents, GSON.toJson(pluginClasses)).getResponseCode()); // tests that the addArtifact when called with empty plugin classes gets deployed. This simulates the call made // by UI Assert.assertEquals(HttpResponseStatus.OK.code(), addPluginArtifact(Id.Artifact.fromEntityId(artifactId), InspectionApp.class, manifest, parents, "").getResponseCode()); }
@Test public void testPluginRequirements() throws Exception { // add a system artifact ArtifactId systemId = NamespaceId.SYSTEM.artifact("app", "1.0.0"); addAppAsSystemArtifacts(); Set<ArtifactRange> parents = Sets.newHashSet(new ArtifactRange( systemId.getNamespace(), systemId.getArtifact(), new ArtifactVersion(systemId.getVersion()), true, new ArtifactVersion(systemId.getVersion()), true)); Manifest manifest = new Manifest(); manifest.getMainAttributes().put(ManifestFields.EXPORT_PACKAGE, InspectionApp.class.getPackage().getName()); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("inspection", "1.0.0"); Assert.assertEquals(HttpResponseStatus.OK.code(), addPluginArtifact(Id.Artifact.fromEntityId(artifactId), InspectionApp.class, manifest, parents) .getResponseCode()); Set<PluginClass> plugins = getArtifact(artifactId).getClasses().getPlugins(); // check that no plugin is being excluded since the no exclusion is configured in cdap-site.xml Assert.assertEquals(8, plugins.size()); // check that requirement is being stored validateRequirement(plugins, InspectionApp.SingleRequirementPlugin.class.getName(), new Requirements(ImmutableSet.of(Table.TYPE))); }
private co.cask.common.http.HttpResponse callPluginMethod( ArtifactId plugins3Id, String pluginType, String pluginName, String pluginMethod, String body, ArtifactScope scope, int expectedResponseCode) throws URISyntaxException, IOException { URL endpoint = getEndPoint( String.format("%s/namespaces/%s/artifacts/%s/versions/%s/plugintypes/%s/plugins/%s/methods/%s?scope=%s", Constants.Gateway.API_VERSION_3, plugins3Id.getNamespace(), plugins3Id.getArtifact(), plugins3Id.getVersion(), pluginType, pluginName, pluginMethod, scope.name())) .toURL(); HttpRequest request = HttpRequest.post(endpoint).withBody(body).build(); co.cask.common.http.HttpResponse response = HttpRequests.execute(request); Assert.assertEquals(expectedResponseCode, response.getResponseCode()); return response; }
@Override public int hashCode() { if (entityId.getEntityType().equals(EntityType.ARTIFACT)) { ArtifactId artifactId = (ArtifactId) entityId; return Objects.hash(artifactId.getEntityType(), artifactId.getNamespace(), artifactId.getArtifact()); } if (entityId.getEntityType().equals(EntityType.APPLICATION)) { ApplicationId applicationId = (ApplicationId) entityId; return Objects.hash(applicationId.getEntityType(), applicationId.getNamespace(), applicationId.getApplication()); } if (entityId.getEntityType().equals(EntityType.PROGRAM)) { ProgramId programId = (ProgramId) entityId; return Objects.hash(programId.getEntityType(), programId.getNamespace(), programId.getApplication(), programId.getType(), programId.getProgram()); } return Objects.hash(entityId); } }