@Override public Set<MemoryComputeKey> getMemoryComputeKeys() { return new HashSet<>(Collections.singletonList(MemoryComputeKey.of(DEGREE, Operator.assign, true, false))); }
public FulgoraMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) { this.currentMap = new ConcurrentHashMap<>(); this.previousMap = new ConcurrentHashMap<>(); if (null != vertexProgram) { for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) { this.memoryKeys.put(key.getKey(), key); } } for (final MapReduce mapReduce : mapReducers) { this.memoryKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } }
public TinkerWorkerMemory(final TinkerMemory mainMemory) { this.mainMemory = mainMemory; for (final MemoryComputeKey key : this.mainMemory.memoryKeys.values()) { this.reducers.put(key.getKey(), key.clone().getReducer()); } }
@Override public void add(final String key, final Object value) { checkKeyValue(key, value); if (!this.inExecute && ("incr".equals(key) || "and".equals(key) || "or".equals(key))) throw Memory.Exceptions.memoryIsCurrentlyImmutable(); else if (!this.inExecute) throw Memory.Exceptions.memoryAddOnlyDuringVertexProgramExecute(key); this.currentMap.compute(key, (k, v) -> null == v ? value : this.memoryKeys.get(key).getReducer().apply(v, value)); }
@Override public Set<String> keys() { return this.previousMap.keySet().stream().filter(key -> !this.inExecute || this.memoryKeys.get(key).isBroadcast()).collect(Collectors.toSet()); }
protected void complete() { this.iteration.decrementAndGet(); this.previousMap = this.currentMap; this.memoryKeys.values().stream().filter(MemoryComputeKey::isTransient).forEach(computeKey -> this.previousMap.remove(computeKey.getKey())); }
@Override public TraversalVertexProgram clone() { try { final TraversalVertexProgram clone = (TraversalVertexProgram) super.clone(); clone.traversal = this.traversal.clone(); if (!clone.traversal.get().isLocked()) clone.traversal.get().applyStrategies(); clone.traversalMatrix = new TraversalMatrix<>(clone.traversal.get()); clone.memoryComputeKeys = new HashSet<>(); for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys) { clone.memoryComputeKeys.add(memoryComputeKey.clone()); } return clone; } catch (final CloneNotSupportedException e) { throw new IllegalStateException(e.getMessage(), e); } }
@Override public void add(final String key, final Object value) { this.checkKeyValue(key, value); if (this.memoryMap.containsKey(key)) { final Object newValue = this.memoryComputeKeys.get(key).getReducer().apply(this.memoryMap.get(key), value); this.memoryMap.put(key, newValue); } else { this.memoryMap.put(key, value); } }
@Override public <R> R get(final String key) throws IllegalArgumentException { final R r = (R) this.previousMap.get(key); if (null == r) throw Memory.Exceptions.memoryDoesNotExist(key); else if (this.inExecute && !this.memoryKeys.get(key).isBroadcast()) throw Memory.Exceptions.memoryDoesNotExist(key); else return r; }
protected void complete() { this.iteration.decrementAndGet(); this.previousMap = this.currentMap; this.memoryKeys.values().stream().filter(MemoryComputeKey::isTransient).forEach(computeKey -> this.previousMap.remove(computeKey.getKey())); }
@Override public TraversalVertexProgram clone() { try { final TraversalVertexProgram clone = (TraversalVertexProgram) super.clone(); clone.traversal = this.traversal.clone(); if (!clone.traversal.get().isLocked()) clone.traversal.get().applyStrategies(); clone.traversalMatrix = new TraversalMatrix<>(clone.traversal.get()); clone.memoryComputeKeys = new HashSet<>(); for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys) { clone.memoryComputeKeys.add(memoryComputeKey.clone()); } return clone; } catch (final CloneNotSupportedException e) { throw new IllegalStateException(e.getMessage(), e); } }
@Override public Set<MemoryComputeKey> getMemoryComputeKeys() { return new HashSet<>(Arrays.asList( MemoryComputeKey.of("a", Operator.sum, true, false), MemoryComputeKey.of("b", Operator.sum, true, false), MemoryComputeKey.of("c", Operator.and, true, false), MemoryComputeKey.of("d", Operator.or, true, false), MemoryComputeKey.of("e", Operator.and, true, false), MemoryComputeKey.of("f", Operator.assign, true, false))); }
public TinkerMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) { this.currentMap = new ConcurrentHashMap<>(); this.previousMap = new ConcurrentHashMap<>(); if (null != vertexProgram) { for (final MemoryComputeKey memoryComputeKey : vertexProgram.getMemoryComputeKeys()) { this.memoryKeys.put(memoryComputeKey.getKey(), memoryComputeKey); } } for (final MapReduce mapReduce : mapReducers) { this.memoryKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } }
public TinkerWorkerMemory(final TinkerMemory mainMemory) { this.mainMemory = mainMemory; for (final MemoryComputeKey key : this.mainMemory.memoryKeys.values()) { this.reducers.put(key.getKey(), key.clone().getReducer()); } }
@Override public void add(final String key, final Object value) { checkKeyValue(key, value); if (!this.inExecute) throw Memory.Exceptions.memoryAddOnlyDuringVertexProgramExecute(key); this.currentMap.compute(key, (k, v) -> null == v ? value : this.memoryKeys.get(key).getReducer().apply(v, value)); }
@Override public Set<String> keys() { return this.previousMap.keySet().stream().filter(key -> !this.inExecute || this.memoryKeys.get(key).isBroadcast()).collect(Collectors.toSet()); }
protected void complete() { this.memoryComputeKeys.values().stream().filter(MemoryComputeKey::isTransient).forEach(memoryComputeKey -> this.sparkMemory.remove(memoryComputeKey.getKey())); }
@Override public Set<MemoryComputeKey> getMemoryComputeKeys() { return new HashSet<>(Arrays.asList( MemoryComputeKey.of("set", Operator.assign, true, false), MemoryComputeKey.of("incr", Operator.sum, true, false), MemoryComputeKey.of("and", Operator.and, true, false), MemoryComputeKey.of("or", Operator.or, true, false))); }
public SparkMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers, final JavaSparkContext sparkContext) { if (null != vertexProgram) { for (final MemoryComputeKey key : vertexProgram.getMemoryComputeKeys()) { this.memoryComputeKeys.put(key.getKey(), key); } } for (final MapReduce mapReduce : mapReducers) { this.memoryComputeKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false)); } for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys.values()) { this.sparkMemory.put( memoryComputeKey.getKey(), sparkContext.accumulator(ObjectWritable.empty(), memoryComputeKey.getKey(), new MemoryAccumulator<>(memoryComputeKey))); } this.broadcast = sparkContext.broadcast(Collections.emptyMap()); }
public TinkerWorkerMemory(final TinkerMemory mainMemory) { this.mainMemory = mainMemory; for (final MemoryComputeKey key : this.mainMemory.memoryKeys.values()) { this.reducers.put(key.getKey(), key.clone().getReducer()); } }