public static <S> List<S> getStepsOfAssignableClassRecursively(final Class<S> stepClass, final Traversal.Admin<?, ?> traversal) { return getStepsOfAssignableClassRecursively(null, stepClass, traversal); }
public static void convAllHasSteps(Traversal.Admin<?, ?> traversal) { // Extract all has steps in traversal @SuppressWarnings("rawtypes") List<HasStep> steps = TraversalHelper .getStepsOfAssignableClassRecursively( HasStep.class, traversal); HugeGraph graph = (HugeGraph) traversal.getGraph().get(); for (HasStep<?> step : steps) { TraversalUtil.convHasStep(graph, step); } }
public static <S> List<S> getStepsOfAssignableClassRecursively(final Scope scope, final Class<S> stepClass, final Traversal.Admin<?, ?> traversal) { final List<S> list = new ArrayList<>(); for (final Step<?, ?> step : traversal.getSteps()) { if (stepClass.isAssignableFrom(step.getClass())) list.add((S) step); if (step instanceof TraversalParent) { if (null == scope || Scope.local.equals(scope)) { for (final Traversal.Admin<?, ?> localChild : ((TraversalParent) step).getLocalChildren()) { list.addAll(TraversalHelper.getStepsOfAssignableClassRecursively(stepClass, localChild)); } } if (null == scope || Scope.global.equals(scope)) { for (final Traversal.Admin<?, ?> globalChild : ((TraversalParent) step).getGlobalChildren()) { list.addAll(TraversalHelper.getStepsOfAssignableClassRecursively(stepClass, globalChild)); } } } } return list; }
public static boolean isLegal(final Traversal.Admin<?, ?> traversal) { final Step<?, ?> startStep = traversal.getStartStep(); final Step<?, ?> endStep = traversal.getEndStep(); // right now this is not supported because of how the SparkStarBarrierInterceptor mutates the traversal prior to local evaluation if (traversal.getStrategies().toList().stream().filter(strategy -> strategy instanceof SubgraphStrategy).findAny().isPresent()) return false; if (!startStep.getClass().equals(GraphStep.class) || ((GraphStep) startStep).returnsEdge()) return false; if (!endStep.getClass().equals(CountGlobalStep.class) && !endStep.getClass().equals(SumGlobalStep.class) && !endStep.getClass().equals(MeanGlobalStep.class) && !endStep.getClass().equals(MaxGlobalStep.class) && !endStep.getClass().equals(MinGlobalStep.class) && !endStep.getClass().equals(FoldStep.class) && !endStep.getClass().equals(GroupStep.class) && !endStep.getClass().equals(GroupCountStep.class)) // TODO: tree() return false; if (TraversalHelper.getStepsOfAssignableClassRecursively(Scope.global, Barrier.class, traversal).size() != 1) return false; if (traversal.getTraverserRequirements().contains(TraverserRequirement.SACK)) return false; return TraversalHelper.isLocalStarGraph(traversal); } }
@Override public void addGlobalChildOption(final M pickToken, final Traversal.Admin<S, E> traversalOption) { final Object pickTokenKey = PickTokenKey.make(pickToken); if (this.traversalOptions.containsKey(pickTokenKey)) this.traversalOptions.get(pickTokenKey).add(traversalOption); else this.traversalOptions.put(pickTokenKey, new ArrayList<>(Collections.singletonList(traversalOption))); // adding an IdentityStep acts as a placeholder when reducing barriers get in the way - see the // standardAlgorithm() method for more information. if (TraversalHelper.hasStepOfAssignableClass(ReducingBarrierStep.class, traversalOption)) traversalOption.addStep(0, new IdentityStep(traversalOption)); traversalOption.addStep(new EndStep(traversalOption)); if (!this.hasBarrier && !TraversalHelper.getStepsOfAssignableClassRecursively(Barrier.class, traversalOption).isEmpty()) this.hasBarrier = true; this.integrateChild(traversalOption); }
directionLabels.put(Direction.IN, inLabels); directionLabels.put(Direction.BOTH, bothLabels); TraversalHelper.getStepsOfAssignableClassRecursively(VertexStep.class, traversal).forEach(step -> {
@Override protected E supply() { if (null == this.sideEffectCapableSteps) { this.sideEffectCapableSteps = new HashMap<>(); Traversal.Admin<?, ?> parentTraversal = this.getTraversal(); while (!(parentTraversal instanceof EmptyTraversal)) { for (final SideEffectCapable<Object, E> capableStep : TraversalHelper.getStepsOfAssignableClassRecursively(SideEffectCapable.class, parentTraversal)) { if (this.sideEffectKeys.contains(capableStep.getSideEffectKey()) && !this.sideEffectCapableSteps.containsKey(capableStep.getSideEffectKey())) this.sideEffectCapableSteps.put(capableStep.getSideEffectKey(), capableStep); } if (this.sideEffectKeys.size() == this.sideEffectCapableSteps.size()) break; parentTraversal = parentTraversal.getParent().asStep().getTraversal(); } } //////////// if (this.sideEffectKeys.size() == 1) { final String sideEffectKey = this.sideEffectKeys.get(0); final E result = this.getTraversal().getSideEffects().<E>get(sideEffectKey); final SideEffectCapable<Object, E> sideEffectCapable = this.sideEffectCapableSteps.get(sideEffectKey); final E finalResult = null == sideEffectCapable ? result : sideEffectCapable.generateFinalResult(result); this.getTraversal().getSideEffects().set(sideEffectKey, finalResult); return finalResult; } else return (E) this.getMapOfSideEffects(); }
@Override public void workerIterationEnd(final Memory memory) { // store profile metrics in proper ProfileStep metrics if (this.profile) { List<ProfileStep> profileSteps = TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get()); // guess the profile step to store data int profileStepIndex = memory.getIteration(); // if we guess wrongly write timing into last step profileStepIndex = profileStepIndex >= profileSteps.size() ? profileSteps.size() - 1 : profileStepIndex; this.iterationMetrics.finish(0); // reset counts this.iterationMetrics.setCount(TraversalMetrics.TRAVERSER_COUNT_ID,0); if (null != MemoryTraversalSideEffects.getMemorySideEffectsPhase(this.traversal.get())) { this.traversal.get().getSideEffects().add(profileSteps.get(profileStepIndex).getId(), this.iterationMetrics); } this.iterationMetrics = null; } }
@Override public void apply(final Traversal.Admin<?, ?> traversal) { if (!TraversalHelper.onGraphComputer(traversal)) return; if (traversal.getParent() instanceof TraversalVertexProgramStep) { if (TraversalHelper.getStepsOfAssignableClassRecursively(GraphStep.class, traversal).size() > 1) throw new VerificationException("Mid-traversal V()/E() is currently not supported on GraphComputer", traversal); if (TraversalHelper.hasStepOfAssignableClassRecursively(ProfileStep.class, traversal) && TraversalHelper.getStepsOfAssignableClass(VertexProgramStep.class, TraversalHelper.getRootTraversal(traversal)).size() > 1) throw new VerificationException("Profiling a multi-VertexProgramStep traversal is currently not supported on GraphComputer", traversal); } // this is a problem because sideEffect.merge() is transient on the OLAP reduction if (TraversalHelper.getRootTraversal(traversal).getTraverserRequirements().contains(TraverserRequirement.ONE_BULK)) throw new VerificationException("One bulk is currently not supported on GraphComputer: " + traversal, traversal); // you can not traverse past the local star graph with localChildren (e.g. by()-modulators). if (!TraversalHelper.isGlobalChild(traversal) && !TraversalHelper.isLocalStarGraph(traversal)) throw new VerificationException("Local traversals may not traverse past the local star-graph on GraphComputer: " + traversal, traversal); for (final Step<?, ?> step : traversal.getSteps()) { if (step instanceof PathProcessor && ((PathProcessor) step).getMaxRequirement() != PathProcessor.ElementRequirement.ID) throw new VerificationException("It is not possible to access more than a path element's id on GraphComputer: " + step + " requires " + ((PathProcessor) step).getMaxRequirement(), traversal); if (UNSUPPORTED_STEPS.stream().filter(c -> c.isAssignableFrom(step.getClass())).findFirst().isPresent()) throw new VerificationException("The following step is currently not supported on GraphComputer: " + step, traversal); } Step<?, ?> nextParentStep = traversal.getParent().asStep(); while (!(nextParentStep instanceof EmptyStep)) { if (nextParentStep instanceof PathProcessor && ((PathProcessor) nextParentStep).getMaxRequirement() != PathProcessor.ElementRequirement.ID) throw new VerificationException("The following path processor step requires more than the element id on GraphComputer: " + nextParentStep + " requires " + ((PathProcessor) nextParentStep).getMaxRequirement(), traversal); nextParentStep = nextParentStep.getNextStep(); } }
for (final MapReducer<?, ?, ?, ?, ?> mapReducer : TraversalHelper.getStepsOfAssignableClassRecursively(MapReducer.class, this.traversal.get())) { this.mapReducers.add(mapReducer.getMapReduce()); this.memoryComputeKeys.add(MemoryComputeKey.of(mapReducer.getMapReduce().getMemoryKey(), Operator.assign, false, false)); for (final MemoryComputing<?> memoryComputing : TraversalHelper.getStepsOfAssignableClassRecursively(MemoryComputing.class, this.traversal.get())) { this.memoryComputeKeys.add(memoryComputing.getMemoryComputeKey()); for (final ProfileStep profileStep : TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get())) { this.traversal.get().getSideEffects().register(profileStep.getId(), new MutableMetricsSupplier(profileStep.getPreviousStep()), ProfileStep.ProfileBiOperator.instance()); this.profile = !TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get()).isEmpty();
public static <S> List<S> getStepsOfAssignableClassRecursively(final Class<S> stepClass, final Traversal.Admin<?, ?> traversal) { return getStepsOfAssignableClassRecursively(null, stepClass, traversal); }
@Override public void apply(final Traversal.Admin<?, ?> traversal) { final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // best guess at what the graph will be as its dynamically determined for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) { final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, EmptyMemory.instance()).getTraversal().get().clone(); if (!computerTraversal.isLocked()) computerTraversal.applyStrategies(); /// boolean doesMessagePass = TraversalHelper.hasStepOfAssignableClassRecursively(Scope.global, MULTI_ITERATION_CLASSES, computerTraversal); if (!doesMessagePass) { for (final VertexStep vertexStep : TraversalHelper.getStepsOfAssignableClassRecursively(Scope.global, VertexStep.class, computerTraversal)) { if (vertexStep.returnsVertex() || !vertexStep.getDirection().equals(Direction.OUT)) { // in-edges require message pass in OLAP doesMessagePass = true; break; } } } if (!doesMessagePass && !MessagePassingReductionStrategy.endsWithElement(computerTraversal.getEndStep()) && !(computerTraversal.getTraverserRequirements().contains(TraverserRequirement.LABELED_PATH) || // TODO: remove this when dynamic detachment is available in 3.3.0 computerTraversal.getTraverserRequirements().contains(TraverserRequirement.PATH))) { // TODO: remove this when dynamic detachment is available in 3.3.0 step.setComputer(step.getComputer() // if no message passing, don't partition the loaded graph .configure(Constants.GREMLIN_SPARK_SKIP_PARTITIONER, true) // if no message passing, don't cache the loaded graph .configure(Constants.GREMLIN_SPARK_SKIP_GRAPH_CACHE, true)); } } }
@Override public void setup(final Memory memory) { // memory is local MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.SETUP); ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).storeSideEffectsInMemory(); memory.set(VOTE_TO_HALT, true); memory.set(MUTATED_MEMORY_KEYS, new HashSet<>()); memory.set(COMPLETED_BARRIERS, new HashSet<>()); // if halted traversers are being sent from a previous VertexProgram in an OLAP chain (non-distributed traversers), get them into the flow if (!this.haltedTraversers.isEmpty()) { final TraverserSet<Object> toProcessTraversers = new TraverserSet<>(); IteratorUtils.removeOnNext(this.haltedTraversers.iterator()).forEachRemaining(traverser -> { traverser.setStepId(this.traversal.get().getStartStep().getId()); toProcessTraversers.add(traverser); }); assert this.haltedTraversers.isEmpty(); final IndexedTraverserSet<Object,Vertex> remoteActiveTraversers = new IndexedTraverserSet.VertexIndexedTraverserSet(); MasterExecutor.processTraversers(this.traversal, this.traversalMatrix, toProcessTraversers, remoteActiveTraversers, this.haltedTraversers, this.haltedTraverserStrategy); memory.set(HALTED_TRAVERSERS, this.haltedTraversers); memory.set(ACTIVE_TRAVERSERS, remoteActiveTraversers); } else { memory.set(HALTED_TRAVERSERS, new TraverserSet<>()); memory.set(ACTIVE_TRAVERSERS, new IndexedTraverserSet.VertexIndexedTraverserSet()); } // local variable will no longer be used so null it for GC this.haltedTraversers = null; // does the traversal need profile information this.profile = !TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get()).isEmpty(); }
for (final ProfileSideEffectStep profileStep : TraversalHelper.getStepsOfAssignableClassRecursively(ProfileSideEffectStep.class, this.traversal.get())) { this.traversal.get().getSideEffects().set(profileStep.getSideEffectKey(), profileStep.generateFinalResult(this.traversal.get().getSideEffects().get(profileStep.getSideEffectKey())));
public static boolean hasOneBulkRequirement(Traversal.Admin<?, ?> traversal) { List<Step> steps = TraversalHelper.getStepsOfAssignableClassRecursively(Step.class, traversal); for (Step step : steps) { if (step.getRequirements().contains(TraverserRequirement.ONE_BULK)) { return true; } } return false; }
public static boolean hasOneBulkRequirement(Traversal.Admin<?, ?> traversal) { List<Step> steps = TraversalHelper.getStepsOfAssignableClassRecursively(Step.class, traversal); for (Step step : steps) { if (step.getRequirements().contains(TraverserRequirement.ONE_BULK)) { return true; } } return false; }
public static void convAllHasSteps(Traversal.Admin<?, ?> traversal) { // Extract all has steps in traversal @SuppressWarnings("rawtypes") List<HasStep> steps = TraversalHelper .getStepsOfAssignableClassRecursively( HasStep.class, traversal); HugeGraph graph = (HugeGraph) traversal.getGraph().get(); for (HasStep<?> step : steps) { TraversalUtil.convHasStep(graph, step); } }
@Override public void addGlobalChildOption(final M pickToken, final Traversal.Admin<S, E> traversalOption) { final Object pickTokenKey = PickTokenKey.make(pickToken); if (this.traversalOptions.containsKey(pickTokenKey)) this.traversalOptions.get(pickTokenKey).add(traversalOption); else this.traversalOptions.put(pickTokenKey, new ArrayList<>(Collections.singletonList(traversalOption))); // adding an IdentityStep acts as a placeholder when reducing barriers get in the way - see the // standardAlgorithm() method for more information. if (TraversalHelper.hasStepOfAssignableClass(ReducingBarrierStep.class, traversalOption)) traversalOption.addStep(0, new IdentityStep(traversalOption)); traversalOption.addStep(new EndStep(traversalOption)); if (!this.hasBarrier && !TraversalHelper.getStepsOfAssignableClassRecursively(Barrier.class, traversalOption).isEmpty()) this.hasBarrier = true; this.integrateChild(traversalOption); }
SqlgWhereStartStep(final Traversal.Admin traversal, final String selectKey) { super(traversal); this.selectKey = selectKey; }
SqlgWhereStartStep(final Traversal.Admin traversal, final String selectKey) { super(traversal); this.selectKey = selectKey; }