@VisibleForTesting public static JoinBridgeManager<PartitionedLookupSourceFactory> lookupAllAtOnce(PartitionedLookupSourceFactory factory) { return new JoinBridgeManager<>( false, UNGROUPED_EXECUTION, UNGROUPED_EXECUTION, ignored -> factory, factory.getOutputTypes()); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); NestedLoopJoinBridge nestedLoopJoinBridge = joinBridgeManager.getJoinBridge(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, NestedLoopJoinOperator.class.getSimpleName()); joinBridgeManager.probeOperatorCreated(driverContext.getLifespan()); return new NestedLoopJoinOperator( operatorContext, nestedLoopJoinBridge, () -> joinBridgeManager.probeOperatorClosed(driverContext.getLifespan())); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "LookupOuterOperatorFactory is closed"); Lifespan lifespan = driverContext.getLifespan(); if (createdLifespans.contains(lifespan)) { throw new IllegalStateException("Only one outer operator can be created per Lifespan"); } createdLifespans.add(lifespan); ListenableFuture<OuterPositionIterator> outerPositionsFuture = joinBridgeManager.getOuterPositionsFuture(lifespan); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LookupOuterOperator.class.getSimpleName()); joinBridgeManager.outerOperatorCreated(lifespan); return new LookupOuterOperator(operatorContext, outerPositionsFuture, probeOutputTypes, buildOutputTypes, () -> joinBridgeManager.outerOperatorClosed(lifespan)); }
JoinBridgeManager<NestedLoopJoinBridge> nestedLoopJoinBridgeManager = new JoinBridgeManager<>( false, PipelineExecutionStrategy.UNGROUPED_EXECUTION, DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext(); NestedLoopBuildOperator nestedLoopBuildOperator = (NestedLoopBuildOperator) nestedLoopBuildOperatorFactory.createOperator(driverContext); NestedLoopJoinBridge nestedLoopJoinBridge = nestedLoopJoinBridgeManager.getJoinBridge(Lifespan.taskWide());
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, NestedLoopBuildOperator.class.getSimpleName()); return new NestedLoopBuildOperator(operatorContext, nestedLoopJoinBridgeManager.getJoinBridge(driverContext.getLifespan())); }
public void probeOperatorCreated(Lifespan lifespan) { initializeIfNecessary(); internalJoinBridgeDataManager.probeOperatorCreated(lifespan); }
joinBridgeManager.incrementProbeFactoryCount(); buildOutputTypes, lookupSourceFactoryManager), lookupSourceFactoryManager.getBuildExecutionStrategy()));
public NestedLoopJoinOperatorFactory(int operatorId, PlanNodeId planNodeId, JoinBridgeManager<NestedLoopJoinBridge> nestedLoopJoinBridgeManager) { this.operatorId = operatorId; this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); this.joinBridgeManager = nestedLoopJoinBridgeManager; this.joinBridgeManager.incrementProbeFactoryCount(); }
private void initializeIfNecessary() { if (!initialized.get()) { synchronized (this) { if (initialized.get()) { return; } int finalProbeFactoryCount = probeFactoryCount.get(); internalJoinBridgeDataManager = internalJoinBridgeDataManager(probeExecutionStrategy, buildExecutionStrategy, joinBridgeProvider, finalProbeFactoryCount, buildOuter ? 1 : 0); initialized.set(true); } } }
probeTypes, probeOutputChannelTypes, lookupSourceFactoryManager.getBuildOutputTypes(), joinType, new JoinProbeFactory(probeOutputChannels.stream().mapToInt(i -> i).toArray(), probeJoinChannel, probeHashChannel),
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory( buildPages.getTypes(), rangeList(buildPages.getTypes().size()).stream()
JoinBridgeManager<NestedLoopJoinBridge> nestedLoopJoinBridgeManager = new JoinBridgeManager<>( false, PipelineExecutionStrategy.UNGROUPED_EXECUTION, DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext(); NestedLoopBuildOperator nestedLoopBuildOperator = (NestedLoopBuildOperator) nestedLoopBuildOperatorFactory.createOperator(driverContext); NestedLoopJoinBridge nestedLoopJoinBridge = nestedLoopJoinBridgeManager.getJoinBridge(Lifespan.taskWide());
@Override public HashBuilderOperator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, HashBuilderOperator.class.getSimpleName()); PartitionedLookupSourceFactory lookupSourceFactory = this.lookupSourceFactoryManager.getJoinBridge(driverContext.getLifespan()); int partitionIndex = getAndIncrementPartitionIndex(driverContext.getLifespan()); verify(partitionIndex < lookupSourceFactory.partitions()); return new HashBuilderOperator( operatorContext, lookupSourceFactory, partitionIndex, outputChannels, hashChannels, preComputedHashChannel, filterFunctionFactory, sortChannel, searchFunctionFactories, expectedPositions, pagesIndexFactory, spillEnabled, singleStreamSpillerFactory); }
public void outerOperatorFactoryClosed(Lifespan lifespan) { initializeIfNecessary(); internalJoinBridgeDataManager.outerOperatorFactoryClosed(lifespan); }
private NestedLoopJoinOperatorFactory(NestedLoopJoinOperatorFactory other) { requireNonNull(other, "other is null"); this.operatorId = other.operatorId; this.planNodeId = other.planNodeId; this.joinBridgeManager = other.joinBridgeManager; // closed is intentionally not copied closed = false; joinBridgeManager.incrementProbeFactoryCount(); }
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); LookupSourceFactory lookupSourceFactory = joinBridgeManager.getJoinBridge(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LookupJoinOperator.class.getSimpleName()); lookupSourceFactory.setTaskContext(driverContext.getPipelineContext().getTaskContext()); joinBridgeManager.probeOperatorCreated(driverContext.getLifespan()); return new LookupJoinOperator( operatorContext, probeTypes, buildOutputTypes, joinType, lookupSourceFactory, joinProbeFactory, () -> joinBridgeManager.probeOperatorClosed(driverContext.getLifespan()), totalOperatorsCount, probeHashGenerator, partitioningSpillerFactory); }
checkArgument(node.getType() == INNER, "NestedLoopJoin is only used for inner join"); JoinBridgeManager<NestedLoopJoinBridge> nestedLoopJoinBridgeManager = new JoinBridgeManager<>( false, probeSource.getPipelineExecutionStrategy(),
private void buildLookupSource(BuildSideSetup buildSideSetup) { requireNonNull(buildSideSetup, "buildSideSetup is null"); LookupSourceFactory lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager().getJoinBridge(Lifespan.taskWide()); Future<LookupSourceProvider> lookupSourceProvider = lookupSourceFactory.createLookupSourceProvider(); List<Driver> buildDrivers = buildSideSetup.getBuildDrivers(); while (!lookupSourceProvider.isDone()) { for (Driver buildDriver : buildDrivers) { buildDriver.process(); } } getFutureValue(lookupSourceProvider).close(); for (Driver buildDriver : buildDrivers) { runDriverInThread(executor, buildDriver); } }
public void outerOperatorClosed(Lifespan lifespan) { initializeIfNecessary(); internalJoinBridgeDataManager.outerOperatorClosed(lifespan); }
private LookupJoinOperatorFactory(LookupJoinOperatorFactory other) { requireNonNull(other, "other is null"); checkArgument(!other.closed, "cannot duplicated closed OperatorFactory"); operatorId = other.operatorId; planNodeId = other.planNodeId; probeTypes = other.probeTypes; buildOutputTypes = other.buildOutputTypes; joinType = other.joinType; joinProbeFactory = other.joinProbeFactory; joinBridgeManager = other.joinBridgeManager; outerOperatorFactoryResult = other.outerOperatorFactoryResult; totalOperatorsCount = other.totalOperatorsCount; probeHashGenerator = other.probeHashGenerator; partitioningSpillerFactory = other.partitioningSpillerFactory; closed = false; joinBridgeManager.incrementProbeFactoryCount(); }