/** * Creates a new logical graph by union the vertex and edge sets of all graph * contained in the given collection. * * @param collection input collection * @return combined graph */ @Override public LogicalGraph execute(GraphCollection collection) { return collection.getConfig().getLogicalGraphFactory().fromDataSets( collection.getVertices(), collection.getEdges()); }
/** * Builds a {@link LogicalGraph} from the graph referenced by the given * graph variable. * * @param variable graph variable used in GDL script * @return LogicalGraph */ public LogicalGraph getLogicalGraphByVariable(String variable) { GraphHead graphHead = getGraphHeadByVariable(variable); Collection<Vertex> vertices = getVerticesByGraphVariables(variable); Collection<Edge> edges = getEdgesByGraphVariables(variable); return config.getLogicalGraphFactory().fromCollections(graphHead, vertices, edges); }
/** * Creates a new Configuration. * * @param executionEnvironment Flink execution environment * @param logicalGraphLayoutFactory Factory for creating logical graphs * @param graphCollectionLayoutFactory Factory for creating graph collections */ protected GradoopFlinkConfig( ExecutionEnvironment executionEnvironment, LogicalGraphLayoutFactory<GraphHead, Vertex, Edge> logicalGraphLayoutFactory, GraphCollectionLayoutFactory<GraphHead, Vertex, Edge> graphCollectionLayoutFactory) { super(); Objects.requireNonNull(executionEnvironment); Objects.requireNonNull(logicalGraphLayoutFactory); Objects.requireNonNull(graphCollectionLayoutFactory); this.executionEnvironment = executionEnvironment; // init with default layout factories this.logicalGraphFactory = new LogicalGraphFactory(this); this.logicalGraphFactory.setLayoutFactory(logicalGraphLayoutFactory); this.graphCollectionFactory = new GraphCollectionFactory(this); this.graphCollectionFactory.setLayoutFactory(graphCollectionLayoutFactory); }
@Override public LogicalGraph getLogicalGraph() throws IOException { MetaData metaData = MetaData.fromFile(getMetaDataPath(), hdfsConfig); ExecutionEnvironment env = getConfig().getExecutionEnvironment(); VertexFactory vertexFactory = getConfig().getVertexFactory(); EdgeFactory edgeFactory = getConfig().getEdgeFactory(); Map<String, DataSet<Vertex>> vertices = metaData.getVertexLabels().stream() .map(l -> Tuple2.of(l, env.readTextFile(getVertexCSVPath(l)) .map(new CSVLineToVertex(vertexFactory)) .withBroadcastSet(MetaData.fromFile(getMetaDataPath(), getConfig()), BC_METADATA))) .collect(Collectors.toMap(t -> t.f0, t -> t.f1)); Map<String, DataSet<Edge>> edges = metaData.getEdgeLabels().stream() .map(l -> Tuple2.of(l, env.readTextFile(getEdgeCSVPath(l)) .map(new CSVLineToEdge(edgeFactory)) .withBroadcastSet(MetaData.fromFile(getMetaDataPath(), getConfig()), BC_METADATA))) .collect(Collectors.toMap(t -> t.f0, t -> t.f1)); return getConfig().getLogicalGraphFactory().fromIndexedDataSets(vertices, edges); }
@Override public LogicalGraph getLogicalGraph() throws IOException { MetaData metaData = MetaData.fromFile(getMetaDataPath(), hdfsConfig); ExecutionEnvironment env = getConfig().getExecutionEnvironment(); VertexFactory vertexFactory = getConfig().getVertexFactory(); EdgeFactory edgeFactory = getConfig().getEdgeFactory(); Map<String, DataSet<Vertex>> vertices = metaData.getVertexLabels().stream() .map(l -> Tuple2.of(l, env.readTextFile(getVertexCSVPath(l)) .map(new CSVLineToVertex(vertexFactory)) .withBroadcastSet(MetaData.fromFile(getMetaDataPath(), getConfig()), BC_METADATA))) .collect(Collectors.toMap(t -> t.f0, t -> t.f1)); Map<String, DataSet<Edge>> edges = metaData.getEdgeLabels().stream() .map(l -> Tuple2.of(l, env.readTextFile(getEdgeCSVPath(l)) .map(new CSVLineToEdge(edgeFactory)) .withBroadcastSet(MetaData.fromFile(getMetaDataPath(), getConfig()), BC_METADATA))) .collect(Collectors.toMap(t -> t.f0, t -> t.f1)); return getConfig().getLogicalGraphFactory().fromIndexedDataSets(vertices, edges); }
/** * Creates a new logical graph by union the vertex and edge sets of all graph * contained in the given collection. * * @param collection input collection * @return combined graph */ @Override public LogicalGraph execute(GraphCollection collection) { return collection.getConfig().getLogicalGraphFactory().fromDataSets( collection.getVertices(), collection.getEdges()); } }
/** * Builds a {@link LogicalGraph} from the graph referenced by the given * graph variable. * * @param variable graph variable used in GDL script * @return LogicalGraph */ public LogicalGraph getLogicalGraphByVariable(String variable) { GraphHead graphHead = getGraphHeadByVariable(variable); Collection<Vertex> vertices = getVerticesByGraphVariables(variable); Collection<Edge> edges = getEdgesByGraphVariables(variable); return config.getLogicalGraphFactory().fromCollections(graphHead, vertices, edges); }
/** * Creates a new Configuration. * * @param executionEnvironment Flink execution environment * @param logicalGraphLayoutFactory Factory for creating logical graphs * @param graphCollectionLayoutFactory Factory for creating graph collections */ protected GradoopFlinkConfig( ExecutionEnvironment executionEnvironment, LogicalGraphLayoutFactory<GraphHead, Vertex, Edge> logicalGraphLayoutFactory, GraphCollectionLayoutFactory<GraphHead, Vertex, Edge> graphCollectionLayoutFactory) { super(); Objects.requireNonNull(executionEnvironment); Objects.requireNonNull(logicalGraphLayoutFactory); Objects.requireNonNull(graphCollectionLayoutFactory); this.executionEnvironment = executionEnvironment; // init with default layout factories this.logicalGraphFactory = new LogicalGraphFactory(this); this.logicalGraphFactory.setLayoutFactory(logicalGraphLayoutFactory); this.graphCollectionFactory = new GraphCollectionFactory(this); this.graphCollectionFactory.setLayoutFactory(graphCollectionLayoutFactory); }
/** * Import each row of the file as a vertex and create a logical graph from it. * If no column property names are set, read the first line of the file as header and set this * values as column names. * * @return a logical graph with a vertex per csv line and no edges * @throws IOException on failure */ @Override public LogicalGraph getLogicalGraph() throws IOException { DataSet<Vertex> vertices; if (columnNames == null) { vertices = readCSVFile(readHeaderRow(), checkReoccurringHeader); } else { vertices = readCSVFile(columnNames, checkReoccurringHeader); } return config.getLogicalGraphFactory().fromDataSets(vertices); }
/** * Returns a logical graph containing the complete vertex and edge space of * the database. * * @param withGraphContainment true, if vertices and edges shall be updated to * be contained in the logical graph representing * the database * @return logical graph of vertex and edge space */ public LogicalGraph getLogicalGraph(boolean withGraphContainment) { if (withGraphContainment) { return config.getLogicalGraphFactory().fromCollections(getVertices(), getEdges()) .transformGraphHead(new RenameLabel<>(GradoopConstants.DEFAULT_GRAPH_LABEL, GradoopConstants.DB_GRAPH_LABEL)); } else { GraphHead graphHead = config.getGraphHeadFactory() .createGraphHead(GradoopConstants.DB_GRAPH_LABEL); return config.getLogicalGraphFactory().fromCollections(graphHead, getVertices(), getEdges()); } }
@Override public LogicalGraph getLogicalGraph() { DataSet<Vertex> vertices = config.getExecutionEnvironment().readTextFile(jsonPath) .map(new MinimalJsonToVertex(config.getVertexFactory())); return config.getLogicalGraphFactory().fromDataSets(vertices); }
/** * Returns a logical graph containing the complete vertex and edge space of * the database. * * @param withGraphContainment true, if vertices and edges shall be updated to * be contained in the logical graph representing * the database * @return logical graph of vertex and edge space */ public LogicalGraph getLogicalGraph(boolean withGraphContainment) { if (withGraphContainment) { return config.getLogicalGraphFactory().fromCollections(getVertices(), getEdges()) .transformGraphHead(new RenameLabel<>(GradoopConstants.DEFAULT_GRAPH_LABEL, GradoopConstants.DB_GRAPH_LABEL)); } else { GraphHead graphHead = config.getGraphHeadFactory() .createGraphHead(GradoopConstants.DB_GRAPH_LABEL); return config.getLogicalGraphFactory().fromCollections(graphHead, getVertices(), getEdges()); } }
/** * {@inheritDoc} */ @Override public LogicalGraph execute(LogicalGraph graph) { DataSet<GraphHead> newGraphHead = graph .aggregate(new VertexCount(), new EdgeCount()) .getGraphHead() .map(new CalculateDensity(SamplingEvaluationConstants.PROPERTY_KEY_DENSITY)); return graph.getConfig().getLogicalGraphFactory() .fromDataSets(newGraphHead, graph.getVertices(), graph.getEdges()); }
/** * Tests whether list properties are extracted correctly. * * @throws Exception If collect doesn't work as expected. */ @Test public void listPropertyTest() throws Exception { VertexFactory vf = getConfig().getVertexFactory(); Vertex v1 = vf.createVertex("foo"); v1.setProperty("a", PropertyValue.create(Arrays.asList(PropertyValue.create("m"), PropertyValue.create("n")))); Vertex v2 = vf.createVertex("foo"); v2.setProperty("a", PropertyValue.create(Arrays.asList(PropertyValue.create("x"), PropertyValue.create("y"), PropertyValue.create("z")))); LogicalGraph input = getConfig().getLogicalGraphFactory().fromCollections(Arrays.asList(v1, v2), Collections.emptyList()); ExtractPropertyFromVertex ext = new ExtractPropertyFromVertex("foo", "a", "A", "key"); LogicalGraph output = input.callForGraph(ext); List<Vertex> createdVertices = new ArrayList<>(); output.getVertices() .filter(new ByLabel<>("A")) .output(new LocalCollectionOutputFormat<>(createdVertices)); input.getConfig().getExecutionEnvironment().execute(); ArrayList<String> properties = new ArrayList<>(); createdVertices.forEach(v -> properties.add(v.getPropertyValue("key").getString())); Assert.assertTrue(properties.containsAll(Arrays.asList("m", "n", "x", "y", "z"))); }
@Override public LogicalGraph execute(LogicalGraph graph) { DataSet<GraphHead> newGraphHead = graph .aggregate(new VertexCount(), new EdgeCount()) .getGraphHead() .map(new CalculateDensity(SamplingEvaluationConstants.PROPERTY_KEY_DENSITY)); return graph.getConfig().getLogicalGraphFactory() .fromDataSets(newGraphHead, graph.getVertices(), graph.getEdges()); } }
/** * Creates a new logical graph containing the overlapping vertex and edge sets * of the graphs contained in the given collection. Vertex and edge equality * is based on their respective identifiers. * * @param collection input collection * @return graph with overlapping elements from the input collection */ @Override public LogicalGraph execute(GraphCollection collection) { DataSet<GraphHead> graphHeads = collection.getGraphHeads(); DataSet<GradoopId> graphIDs = graphHeads.map(new Id<GraphHead>()); return collection.getConfig().getLogicalGraphFactory().fromDataSets( getVertices(collection.getVertices(), graphIDs), getEdges(collection.getEdges(), graphIDs) ); }
@Override public LogicalGraph executeInGelly(Graph<GradoopId, NullValue, NullValue> graph) throws Exception { DataSet<Tuple3<GradoopId, GradoopId, GradoopId>> triangles = new org.apache.flink.graph.library.TriangleEnumerator<GradoopId, NullValue, NullValue>() .run(graph); DataSet<GraphHead> resultHead = currentGraph.getGraphHead() .map(new WritePropertyToGraphHeadMap( PROPERTY_KEY_TRIANGLES, PropertyValue.create(triangles.count()))); return currentGraph.getConfig().getLogicalGraphFactory().fromDataSets( resultHead, currentGraph.getVertices(), currentGraph.getEdges()); }
@Override public LogicalGraph executeInGelly(Graph<GradoopId, PropertyValue, NullValue> graph) { DataSet<Vertex> labeledVertices = executeInternal(graph) .join(currentGraph.getVertices()) .where(0).equalTo(new Id<>()) .with(new LPVertexJoin(propertyKey)); // return labeled graph return currentGraph.getConfig().getLogicalGraphFactory() .fromDataSets(labeledVertices, currentGraph.getEdges()); }
/** * Creates a new logical graph containing the overlapping vertex and edge sets * of the graphs contained in the given collection. Vertex and edge equality * is based on their respective identifiers. * * @param collection input collection * @return graph with overlapping elements from the input collection */ @Override public LogicalGraph execute(GraphCollection collection) { DataSet<GraphHead> graphHeads = collection.getGraphHeads(); DataSet<GradoopId> graphIDs = graphHeads.map(new Id<GraphHead>()); return collection.getConfig().getLogicalGraphFactory().fromDataSets( getVertices(collection.getVertices(), graphIDs), getEdges(collection.getEdges(), graphIDs) ); } }
/** * {@inheritDoc} */ @Override public LogicalGraph executeInGelly(Graph<GradoopId, PropertyValue, NullValue> graph) { DataSet<Vertex> labeledVertices = executeInternal(graph) .join(currentGraph.getVertices()) .where(0).equalTo(new Id<>()) .with(new LPVertexJoin(propertyKey)); // return labeled graph return currentGraph.getConfig().getLogicalGraphFactory() .fromDataSets(labeledVertices, currentGraph.getEdges()); }