public FulgoraMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) { this.currentMap = new ConcurrentHashMap<>(); this.previousMap = new ConcurrentHashMap<>(); if (null != vertexProgram) { for (final String key : vertexProgram.getMemoryComputeKeys()) { MemoryHelper.validateKey(key); this.memoryKeys.add(key); } } for (final MapReduce mapReduce : mapReducers) { this.memoryKeys.add(mapReduce.getMemoryKey()); } }
public FulgoraMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) { this.currentMap = new ConcurrentHashMap<>(); this.previousMap = new ConcurrentHashMap<>(); if (null != vertexProgram) { for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) { this.memoryKeys.put(key.getKey(), key); } } for (final MapReduce mapReduce : mapReducers) { this.memoryKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } }
public void addMapReduceMemoryKey(final MapReduce mapReduce) { this.memoryComputeKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); }
public static boolean areEqual(final MapReduce a, final Object b) { if (null == a) throw Graph.Exceptions.argumentCanNotBeNull("a"); if (null == b) throw Graph.Exceptions.argumentCanNotBeNull("b"); if (!(b instanceof MapReduce)) return false; return a.getClass().equals(b.getClass()) && a.getMemoryKey().equals(((MapReduce) b).getMemoryKey()); }
public TinkerMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) { this.currentMap = new ConcurrentHashMap<>(); this.previousMap = new ConcurrentHashMap<>(); if (null != vertexProgram) { for (final MemoryComputeKey memoryComputeKey : vertexProgram.getMemoryComputeKeys()) { this.memoryKeys.put(memoryComputeKey.getKey(), memoryComputeKey); } } for (final MapReduce mapReduce : mapReducers) { this.memoryKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } }
/** * The final result can be generated and added to {@link Memory} and accessible via {@link org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult}. * The default simply takes the object from generateFinalResult() and adds it to the Memory given getMemoryKey(). * * @param memory the memory of the {@link GraphComputer} * @param keyValues the key/value pairs emitted from reduce() (or map() in a map only job). */ public default void addResultToMemory(final Memory.Admin memory, final Iterator<KeyValue<RK, RV>> keyValues) { memory.set(this.getMemoryKey(), this.generateFinalResult(keyValues)); }
public SparkMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers, final JavaSparkContext sparkContext) { if (null != vertexProgram) { for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) { this.memoryComputeKeys.put(key.getKey(), key); } } for (final MapReduce mapReduce : mapReducers) { this.memoryComputeKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys.values()) { this.sparkMemory.put( memoryComputeKey.getKey(), sparkContext.accumulator(ObjectWritable.empty(), memoryComputeKey.getKey(), new MemoryAccumulator<>(memoryComputeKey))); } this.broadcast = sparkContext.broadcast(Collections.emptyMap()); }
Path memoryPath = new Path(Constants.getMemoryLocation(newConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION), (reduceSort.isPresent() ? mapReduce.getMemoryKey() + "-temp" : mapReduce.getMemoryKey()))); if (FileSystem.get(newConfiguration).exists(memoryPath)) { FileSystem.get(newConfiguration).delete(memoryPath, true); reduceSortJob.setNumReduceTasks(1); // todo: is this necessary to ensure sorted order? FileInputFormat.setInputPaths(reduceSortJob, memoryPath); final Path sortedMemoryPath = new Path(Constants.getMemoryLocation(newConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION), mapReduce.getMemoryKey())); FileOutputFormat.setOutputPath(reduceSortJob, sortedMemoryPath); reduceSortJob.waitForCompletion(true);
mapReduce.addResultToMemory(finalMemory, outputRDD.writeMemoryRDD(graphComputerConfiguration, mapReduce.getMemoryKey(), reduceRDD));
this.memoryComputeKeys.add(MemoryComputeKey.of(mapReducer.getMapReduce().getMemoryKey(), Operator.assign, false, false));
public void addMapReduceMemoryKey(final MapReduce mapReduce) { this.memoryComputeKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); }
public static boolean areEqual(final MapReduce a, final Object b) { if (null == a) throw Graph.Exceptions.argumentCanNotBeNull("a"); if (null == b) throw Graph.Exceptions.argumentCanNotBeNull("b"); if (!(b instanceof MapReduce)) return false; return a.getClass().equals(b.getClass()) && a.getMemoryKey().equals(((MapReduce) b).getMemoryKey()); }
public TinkerMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) { this.currentMap = new ConcurrentHashMap<>(); this.previousMap = new ConcurrentHashMap<>(); if (null != vertexProgram) { for (final MemoryComputeKey memoryComputeKey : vertexProgram.getMemoryComputeKeys()) { this.memoryKeys.put(memoryComputeKey.getKey(), memoryComputeKey); } } for (final MapReduce mapReduce : mapReducers) { this.memoryKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } }
public TinkerMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) { this.currentMap = new ConcurrentHashMap<>(); this.previousMap = new ConcurrentHashMap<>(); if (null != vertexProgram) { for (final MemoryComputeKey memoryComputeKey : vertexProgram.getMemoryComputeKeys()) { this.memoryKeys.put(memoryComputeKey.getKey(), memoryComputeKey); } } for (final MapReduce mapReduce : mapReducers) { this.memoryKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } }
public FulgoraMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) { this.currentMap = new ConcurrentHashMap<>(); this.previousMap = new ConcurrentHashMap<>(); if (null != vertexProgram) { for (final String key : vertexProgram.getMemoryComputeKeys()) { MemoryHelper.validateKey(key); this.memoryKeys.add(key); } } for (final MapReduce mapReduce : mapReducers) { this.memoryKeys.add(mapReduce.getMemoryKey()); } }
public DuctileMemory(VertexProgram<M> vertexProgram, Set<MapReduce<?, ?, ?, ?, R>> mapReducers) { if (null != vertexProgram) { for (String key : vertexProgram.getMemoryComputeKeys()) { MemoryHelper.validateKey(key); memoryKeys.add(key); } } for (MapReduce<?, ?, ?, ?, R> mapReduce : mapReducers) { memoryKeys.add(mapReduce.getMemoryKey()); } }
/** * The final result can be generated and added to {@link Memory} and accessible via {@link org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult}. * The default simply takes the object from generateFinalResult() and adds it to the Memory given getMemoryKey(). * * @param memory the memory of the {@link GraphComputer} * @param keyValues the key/value pairs emitted from reduce() (or map() in a map only job). */ public default void addResultToMemory(final Memory.Admin memory, final Iterator<KeyValue<RK, RV>> keyValues) { memory.set(this.getMemoryKey(), this.generateFinalResult(keyValues)); }
public SparkMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers, final JavaSparkContext sparkContext) { if (null != vertexProgram) { for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) { this.memoryComputeKeys.put(key.getKey(), key); } } for (final MapReduce mapReduce : mapReducers) { this.memoryComputeKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys.values()) { this.sparkMemory.put( memoryComputeKey.getKey(), sparkContext.accumulator(ObjectWritable.empty(), memoryComputeKey.getKey(), new MemoryAccumulator<>(memoryComputeKey))); } this.broadcast = sparkContext.broadcast(Collections.emptyMap()); }
public GraknSparkMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers, final JavaSparkContext sparkContext) { if (null != vertexProgram) { for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) { this.memoryComputeKeys.put(key.getKey(), key); } } for (final MapReduce mapReduce : mapReducers) { this.memoryComputeKeys.put( mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys.values()) { this.sparkMemory.put( memoryComputeKey.getKey(), sparkContext.accumulator(ObjectWritable.empty(), memoryComputeKey.getKey(), new MemoryAccumulator<>(memoryComputeKey))); } this.broadcast = sparkContext.broadcast(Collections.emptyMap()); }
this.memoryComputeKeys.add(MemoryComputeKey.of(mapReducer.getMapReduce().getMemoryKey(), Operator.assign, false, false));