@Override public Location read(JsonReader in) throws IOException { return Locations.getLocationFromAbsolutePath(runtimeContext.getLocationFactory(), in.nextString()); } })
@Override public Location read(JsonReader in) throws IOException { return Locations.getLocationFromAbsolutePath(runtimeContext.getLocationFactory(), in.nextString()); } })
@Override public Location read(JsonReader in) throws IOException { return Locations.getLocationFromAbsolutePath(runtimeContext.getLocationFactory(), in.nextString()); } })
@Override public Location call() throws Exception { // we stored uri in old format return Locations.getLocationFromAbsolutePath( locationFactory, new URI(Bytes.toString(entry.getValue())).getPath()); } });
@Override public Location call() throws Exception { // we stored uri in old format return Locations.getLocationFromAbsolutePath( locationFactory, new URI(Bytes.toString(entry.getValue())).getPath()); } });
@Override public ClassLoader load(CacheKey key) throws Exception { if (key.uri == null) { return key.parentClassLoader; } Location jarLocation = Locations.getLocationFromAbsolutePath(locationFactory, key.uri.getPath()); File unpackedDir = DirUtils.createTempDir(tmpDir); BundleJarUtil.unJar(jarLocation, unpackedDir); LOG.trace("unpacking dataset jar from {} to {}.", key.uri.toString(), unpackedDir.getAbsolutePath()); return new DirectoryClassLoader(unpackedDir, key.parentClassLoader, "lib"); } }
@Override public ClassLoader load(CacheKey key) throws Exception { if (key.uri == null) { return key.parentClassLoader; } Location jarLocation = Locations.getLocationFromAbsolutePath(locationFactory, key.uri.getPath()); File unpackedDir = DirUtils.createTempDir(tmpDir); BundleJarUtil.unJar(jarLocation, unpackedDir); LOG.trace("unpacking dataset jar from {} to {}.", key.uri.toString(), unpackedDir.getAbsolutePath()); return new DirectoryClassLoader(unpackedDir, key.parentClassLoader, "lib"); } }
@Override public Void call() throws Exception { Locations.getLocationFromAbsolutePath(locationFactory, oldMeta.getLocationPath()).delete(); return null; } });
@Override public Void call() throws Exception { Locations.getLocationFromAbsolutePath(locationFactory, oldMeta.getLocationPath()).delete(); return null; } });
@Override public Location get(NamespaceMeta namespaceMeta) throws IOException { String rootDirectory = namespaceMeta.getConfig().getRootDirectory(); if (Strings.isNullOrEmpty(rootDirectory)) { // if no custom mapping was specified, then use the default namespaces location return getNonCustomMappedLocation(namespaceMeta.getNamespaceId()); } return Locations.getLocationFromAbsolutePath(locationFactory, rootDirectory); }
private <A, R> R collectArtifacts(Row row, @Nullable ArtifactRange range, int limit, Collector<ArtifactDetail, A, R> collector) { ArtifactKey artifactKey = ArtifactKey.parse(row.getRow()); // Includes artifacts with version matching the one from the given range return row.getColumns().entrySet().stream() .map(e -> Maps.immutableEntry(new ArtifactVersion(Bytes.toString(e.getKey())), e.getValue())) .filter(e -> range == null || range.versionIsInRange(e.getKey())) .limit(limit) .map(e -> { ArtifactData data = GSON.fromJson(Bytes.toString(e.getValue()), ArtifactData.class); ArtifactMeta filteredArtifactMeta = filterPlugins(data.meta); ArtifactId artifactId = new ArtifactId(artifactKey.name, e.getKey(), artifactKey.namespace.equals(NamespaceId.SYSTEM.getNamespace()) ? ArtifactScope.SYSTEM : ArtifactScope.USER); Location artifactLocation = Locations.getLocationFromAbsolutePath(locationFactory, data.getLocationPath()); return new ArtifactDetail(new ArtifactDescriptor(artifactId, artifactLocation), filteredArtifactMeta); }) .collect(collector); }
private <A, R> R collectArtifacts(Iterator<StructuredRow> iterator, @Nullable ArtifactRange range, int limit, Collector<ArtifactDetail, A, R> collector) { // Includes artifacts with version matching the one from the given range return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, 0), false) .map(row -> Maps.immutableEntry( new ArtifactVersion(row.getString(StoreDefinition.ArtifactStore.ARTIFACT_VER_FIELD)), row)) .filter(e -> range == null || range.versionIsInRange(e.getKey())) .limit(limit) .map(e -> { StructuredRow row = e.getValue(); ArtifactKey artifactKey = ArtifactKey.fromRow(row); ArtifactData data = GSON.fromJson(row.getString(StoreDefinition.ArtifactStore.ARTIFACT_DATA_FIELD), ArtifactData.class); ArtifactMeta filteredArtifactMeta = filterPlugins(data.meta); ArtifactId artifactId = new ArtifactId(artifactKey.name, e.getKey(), artifactKey.namespace.equals(NamespaceId.SYSTEM.getNamespace()) ? ArtifactScope.SYSTEM : ArtifactScope.USER); Location artifactLocation = Locations.getLocationFromAbsolutePath(locationFactory, data.getLocationPath()); return new ArtifactDetail(new ArtifactDescriptor(artifactId, artifactLocation), filteredArtifactMeta); }) .collect(collector); }
private SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginsInArtifact(StructuredTable artifactDataTable, Id.Artifact artifactId, Predicate<PluginClass> filter) throws ArtifactNotFoundException, IOException { SortedMap<ArtifactDescriptor, Set<PluginClass>> result = new TreeMap<>(); // Make sure the artifact exists ArtifactCell artifactCell = new ArtifactCell(artifactId); Optional<StructuredRow> row = artifactDataTable.read(artifactCell.keys); if (!row.isPresent()) { throw new ArtifactNotFoundException(artifactId.toEntityId()); } // include any plugin classes that are inside the artifact itself and is accepted by the filter ArtifactData artifactData = GSON.fromJson(row.get().getString(StoreDefinition.ArtifactStore.ARTIFACT_DATA_FIELD), ArtifactData.class); Set<PluginClass> plugins = artifactData.meta.getClasses().getPlugins().stream() .filter(filter).collect(Collectors.toCollection(LinkedHashSet::new)); if (!plugins.isEmpty()) { Location location = Locations.getLocationFromAbsolutePath(locationFactory, artifactData.getLocationPath()); ArtifactDescriptor descriptor = new ArtifactDescriptor(artifactId.toArtifactId(), location); result.put(descriptor, plugins); } return result; }
private SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginsInArtifact(Table table, Id.Artifact artifactId, Predicate<PluginClass> filter) throws ArtifactNotFoundException { SortedMap<ArtifactDescriptor, Set<PluginClass>> result = new TreeMap<>(); // Make sure the artifact exists ArtifactCell artifactCell = new ArtifactCell(artifactId); byte[] artifactDataBytes = table.get(artifactCell.rowkey, artifactCell.column); if (artifactDataBytes == null) { throw new ArtifactNotFoundException(artifactId.toEntityId()); } // include any plugin classes that are inside the artifact itself and is accepted by the filter ArtifactData artifactData = GSON.fromJson(Bytes.toString(artifactDataBytes), ArtifactData.class); Set<PluginClass> plugins = artifactData.meta.getClasses().getPlugins().stream() .filter(filter).collect(Collectors.toCollection(LinkedHashSet::new)); if (!plugins.isEmpty()) { Location location = Locations.getLocationFromAbsolutePath(locationFactory, artifactData.getLocationPath()); ArtifactDescriptor descriptor = new ArtifactDescriptor(artifactId.toArtifactId(), location); result.put(descriptor, plugins); } return result; }
@Override public Location get(NamespaceMeta namespaceMeta) throws IOException { String rootDirectory = namespaceMeta.getConfig().getRootDirectory(); if (isReservedNamespace(namespaceMeta.getNamespaceId()) || Strings.isNullOrEmpty(rootDirectory)) { // if no custom mapping was specified, then use the default namespaces location return getNonCustomMappedLocation(namespaceMeta.getNamespaceId()); } return Locations.getLocationFromAbsolutePath(locationFactory, rootDirectory); }
private Map.Entry<ArtifactDescriptor, ApplicationClass> extractApplicationClass(StructuredRow row) { Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.from(row.getString(StoreDefinition.ArtifactStore.ARTIFACT_NAMESPACE_FIELD)), row.getString(StoreDefinition.ArtifactStore.ARTIFACT_NAME_FIELD), row.getString(StoreDefinition.ArtifactStore.ARTIFACT_VER_FIELD)); AppData appData = GSON.fromJson(row.getString(StoreDefinition.ArtifactStore.APP_DATA_FIELD), AppData.class); ArtifactDescriptor artifactDescriptor = new ArtifactDescriptor( artifactId.toArtifactId(), Locations.getLocationFromAbsolutePath(locationFactory, appData.getArtifactLocationPath())); return new AbstractMap.SimpleEntry<>(artifactDescriptor, appData.appClass); }
@Nullable private ImmutablePair<ArtifactDescriptor, PluginData> getPlugin(StructuredRow row, Predicate<co.cask.cdap.proto.id.ArtifactId> range) { // column is the artifact namespace, name, and version. value is the serialized PluginData Id.Namespace artifactNamespace = Id.Namespace.from(row.getString(StoreDefinition.ArtifactStore.ARTIFACT_NAMESPACE_FIELD)); Id.Artifact artifactId = Id.Artifact.from(artifactNamespace, row.getString(StoreDefinition.ArtifactStore.ARTIFACT_NAME_FIELD), row.getString(StoreDefinition.ArtifactStore.ARTIFACT_VER_FIELD)); if (!range.test(artifactId.toEntityId())) { return null; } PluginData pluginData = GSON.fromJson(row.getString(StoreDefinition.ArtifactStore.PLUGIN_DATA_FIELD), PluginData.class); return ImmutablePair.of(new ArtifactDescriptor( artifactId.toArtifactId(), Locations.getLocationFromAbsolutePath(locationFactory, pluginData.getArtifactLocationPath())), pluginData); }
/** * Decode the PluginClass from the table column if it is from an artifact in the given namespace and * extends the given parent artifact. If the plugin's artifact is not in the given namespace, or it does not * extend the given parent artifact, return null. */ private ImmutablePair<ArtifactDescriptor, PluginClass> getPluginEntry(NamespaceId namespace, Id.Artifact parentArtifactId, Map.Entry<byte[], byte[]> column) { // column is the artifact namespace, name, and version. value is the serialized PluginData ArtifactColumn artifactColumn = ArtifactColumn.parse(column.getKey()); Id.Namespace artifactNamespace = artifactColumn.artifactId.getNamespace(); // filter out plugins whose artifacts are not in the system namespace and not in this namespace if (!Id.Namespace.SYSTEM.equals(artifactNamespace) && !artifactNamespace.equals(Id.Namespace.fromEntityId(namespace))) { return null; } PluginData pluginData = GSON.fromJson(Bytes.toString(column.getValue()), PluginData.class); // filter out plugins that don't extend this version of the parent artifact if (pluginData.isUsableBy(parentArtifactId.toEntityId())) { ArtifactDescriptor artifactDescriptor = new ArtifactDescriptor( artifactColumn.artifactId.toArtifactId(), Locations.getLocationFromAbsolutePath(locationFactory, pluginData.getArtifactLocationPath())); return ImmutablePair.of(artifactDescriptor, pluginData.pluginClass); } return null; }
@Test public void absolutePathTests() { // Test HDFS: Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://1.2.3.4:8020/"); LocationFactory locationFactory = new FileContextLocationFactory(conf, TEST_BASE_PATH); Location location1 = locationFactory.create(TEST_PATH); Location location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath()); Assert.assertEquals(location1.toURI(), location2.toURI()); // Test file: conf = new Configuration(); conf.set("fs.defaultFS", "file:///"); locationFactory = new FileContextLocationFactory(conf, TEST_BASE_PATH); location1 = locationFactory.create(TEST_PATH); location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath()); Assert.assertEquals(location1.toURI(), location2.toURI()); // Test LocalLocation locationFactory = new LocalLocationFactory(new File(TEST_BASE_PATH)); location1 = locationFactory.create(TEST_PATH); location2 = Locations.getLocationFromAbsolutePath(locationFactory, location1.toURI().getPath()); Assert.assertEquals(location1.toURI(), location2.toURI()); } }
.append(NamespaceId.DEFAULT.getNamespace()); Assert.assertEquals(expectedLocation, defaultLoc); expectedLocation = Locations.getLocationFromAbsolutePath(locationFactory, "/ns1"); Assert.assertEquals(expectedLocation, ns1Loc);