@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { dag.getIncomingEdgesOf(vertex).stream() .forEach(edge -> edge.setPropertyPermanently( DataStoreProperty.of(DataStoreProperty.Value.Pipe))); }); return dag; } }
/** * Static method exposing the constructor. * @param value value of the new execution property. * @return the newly created execution property. */ public static DataStoreProperty of(final Value value) { return new DataStoreProperty(value); }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { dag.getIncomingEdgesOf(vertex).stream() .forEach(edge -> edge.setPropertyPermanently( DataStoreProperty.of(DataStoreProperty.Value.Pipe))); }); return dag; } }
/** * Static method exposing the constructor. * @param value value of the new execution property. * @return the newly created execution property. */ public static DataStoreProperty of(final Value value) { return new DataStoreProperty(value); }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { dag.getIncomingEdgesOf(vertex).stream() .filter(edge -> !edge.getPropertyValue(DataStoreProperty.class).isPresent()) .forEach(edge -> edge.setProperty( DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore))); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { dag.getIncomingEdgesOf(vertex).stream() .filter(edge -> !edge.getPropertyValue(DataStoreProperty.class).isPresent()) .forEach(edge -> edge.setProperty( DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore))); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { // Initialize the DataStore of the DAG with GlusterFileStore. final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> edge.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.GlusterFileStore))); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { // Initialize the DataStore of the DAG with GlusterFileStore. final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); inEdges.forEach(edge -> edge.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.GlusterFileStore))); }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); if (!inEdges.isEmpty()) { inEdges.forEach(edge -> { if (fromTransientToReserved(edge) || fromReservedToTransient(edge)) { edge.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); } else if (CommunicationPatternProperty.Value.OneToOne .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get())) { edge.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.MemoryStore)); } else { edge.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); } }); } }); return dag; }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { final List<IREdge> inEdges = dag.getIncomingEdgesOf(vertex); if (!inEdges.isEmpty()) { inEdges.forEach(edge -> { if (fromTransientToReserved(edge) || fromReservedToTransient(edge)) { edge.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); } else if (CommunicationPatternProperty.Value.OneToOne .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get())) { edge.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.MemoryStore)); } else { edge.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); } }); } }); return dag; }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { // Find the merger vertex inserted by reshaping pass. if (dag.getIncomingEdgesOf(vertex).stream().anyMatch(irEdge -> CommunicationPatternProperty.Value.Shuffle .equals(irEdge.getPropertyValue(CommunicationPatternProperty.class).get()))) { dag.getIncomingEdgesOf(vertex).forEach(edgeToMerger -> { if (CommunicationPatternProperty.Value.Shuffle .equals(edgeToMerger.getPropertyValue(CommunicationPatternProperty.class).get())) { // Pass data through memory to the merger vertex. edgeToMerger.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.SerializedMemoryStore)); } }); dag.getOutgoingEdgesOf(vertex).forEach(edgeFromMerger -> // Merge the input data and write it immediately to the remote disk. edgeFromMerger.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore))); } }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.getVertices().forEach(vertex -> { // Find the merger vertex inserted by reshaping pass. if (dag.getIncomingEdgesOf(vertex).stream().anyMatch(irEdge -> CommunicationPatternProperty.Value.Shuffle .equals(irEdge.getPropertyValue(CommunicationPatternProperty.class).get()))) { dag.getIncomingEdgesOf(vertex).forEach(edgeToMerger -> { if (CommunicationPatternProperty.Value.Shuffle .equals(edgeToMerger.getPropertyValue(CommunicationPatternProperty.class).get())) { // Pass data through memory to the merger vertex. edgeToMerger.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.SerializedMemoryStore)); } }); dag.getOutgoingEdgesOf(vertex).forEach(edgeFromMerger -> // Merge the input data and write it immediately to the remote disk. edgeFromMerger.setPropertyPermanently(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore))); } }); return dag; } }
case Shuffle: map.put(PartitionerProperty.of(PartitionerProperty.Value.HashPartitioner)); map.put(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); break; case BroadCast: map.put(PartitionerProperty.of(PartitionerProperty.Value.IntactPartitioner)); map.put(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); break; case OneToOne: map.put(PartitionerProperty.of(PartitionerProperty.Value.IntactPartitioner)); map.put(DataStoreProperty.of(DataStoreProperty.Value.MemoryStore)); break; default: map.put(PartitionerProperty.of(PartitionerProperty.Value.HashPartitioner)); map.put(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore));
case Shuffle: map.put(PartitionerProperty.of(PartitionerProperty.Value.HashPartitioner)); map.put(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); break; case BroadCast: map.put(PartitionerProperty.of(PartitionerProperty.Value.IntactPartitioner)); map.put(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); break; case OneToOne: map.put(PartitionerProperty.of(PartitionerProperty.Value.IntactPartitioner)); map.put(DataStoreProperty.of(DataStoreProperty.Value.MemoryStore)); break; default: map.put(PartitionerProperty.of(PartitionerProperty.Value.HashPartitioner)); map.put(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore));
final OperatorVertex abv) { final IREdge newEdge = new IREdge(CommunicationPatternProperty.Value.Shuffle, mcv, abv); newEdge.setProperty(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); newEdge.setProperty(DataPersistenceProperty.of(DataPersistenceProperty.Value.Keep)); newEdge.setProperty(DataFlowProperty.of(DataFlowProperty.Value.Push));
final OperatorVertex abv) { final IREdge newEdge = new IREdge(CommunicationPatternProperty.Value.Shuffle, mcv, abv); newEdge.setProperty(DataStoreProperty.of(DataStoreProperty.Value.LocalFileStore)); newEdge.setProperty(DataPersistenceProperty.of(DataPersistenceProperty.Value.Keep)); newEdge.setProperty(DataFlowProperty.of(DataFlowProperty.Value.Pull));