@Override public SourceOperator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, sourceId, MergeOperator.class.getSimpleName()); return new MergeOperator( operatorContext, sourceId, exchangeClientSupplier, serdeFactory.createPagesSerde(), orderingCompiler.compilePageWithPositionComparator(types, sortChannels, sortOrder), outputChannels, outputTypes); }
@VisibleForTesting public PagesIndexOrdering internalCompilePagesIndexOrdering(List<Type> sortTypes, List<Integer> sortChannels, List<SortOrder> sortOrders) { requireNonNull(sortChannels, "sortChannels is null"); requireNonNull(sortOrders, "sortOrders is null"); PagesIndexComparator comparator; try { Class<? extends PagesIndexComparator> pagesHashStrategyClass = compilePagesIndexComparator(sortTypes, sortChannels, sortOrders); comparator = pagesHashStrategyClass.getConstructor().newInstance(); } catch (Throwable e) { log.error(e, "Error compiling comparator for channels %s with order %s", sortChannels, sortOrders); comparator = new SimplePagesIndexComparator(sortTypes, sortChannels, sortOrders); } // we may want to load a separate PagesIndexOrdering for each comparator return new PagesIndexOrdering(comparator); }
private PagesIndexOrdering createPagesIndexComparator(List<Integer> sortChannels, List<SortOrder> sortOrders) { List<Type> sortTypes = sortChannels.stream() .map(types::get) .collect(toImmutableList()); return orderingCompiler.compilePagesIndexOrdering(sortTypes, sortChannels, sortOrders); }
@BeforeMethod public void setUp() { executor = newSingleThreadScheduledExecutor(daemonThreadsNamed("test-merge-operator-%s")); serdeFactory = new TestingPagesSerdeFactory(); taskBuffers = CacheBuilder.newBuilder().build(CacheLoader.from(TestingTaskBuffer::new)); httpClient = new TestingHttpClient(new TestingExchangeHttpClientHandler(taskBuffers), executor); exchangeClientFactory = new ExchangeClientFactory(new ExchangeClientConfig(), httpClient, executor); orderingCompiler = new OrderingCompiler(); }
private PageWithPositionComparator internalCompilePageWithPositionComparator(List<Type> types, List<Integer> sortChannels, List<SortOrder> sortOrders) { PageWithPositionComparator comparator; try { Class<? extends PageWithPositionComparator> pageWithPositionsComparatorClass = generatePageWithPositionComparatorClass(types, sortChannels, sortOrders); comparator = pageWithPositionsComparatorClass.getConstructor().newInstance(); } catch (Throwable t) { log.error(t, "Error compiling merge sort comparator for channels %s with order %s", sortChannels, sortChannels); comparator = new SimplePageWithPositionComparator(types, sortChannels, sortOrders); } return comparator; }
private Class<? extends PageWithPositionComparator> generatePageWithPositionComparatorClass(List<Type> sortTypes, List<Integer> sortChannels, List<SortOrder> sortOrders) { CallSiteBinder callSiteBinder = new CallSiteBinder(); ClassDefinition classDefinition = new ClassDefinition( a(PUBLIC, FINAL), makeClassName("PageWithPositionComparator"), type(Object.class), type(PageWithPositionComparator.class)); classDefinition.declareDefaultConstructor(a(PUBLIC)); generateMergeSortCompareTo(classDefinition, callSiteBinder, sortTypes, sortChannels, sortOrders); return defineClass(classDefinition, PageWithPositionComparator.class, callSiteBinder.getBindings(), getClass().getClassLoader()); }
private Class<? extends PagesIndexComparator> compilePagesIndexComparator( List<Type> sortTypes, List<Integer> sortChannels, List<SortOrder> sortOrders) { CallSiteBinder callSiteBinder = new CallSiteBinder(); ClassDefinition classDefinition = new ClassDefinition( a(PUBLIC, FINAL), makeClassName("PagesIndexComparator"), type(Object.class), type(PagesIndexComparator.class)); classDefinition.declareDefaultConstructor(a(PUBLIC)); generatePageIndexCompareTo(classDefinition, callSiteBinder, sortTypes, sortChannels, sortOrders); return defineClass(classDefinition, PagesIndexComparator.class, callSiteBinder.getBindings(), getClass().getClassLoader()); }
@Override public PagesIndexOrdering load(PagesIndexComparatorCacheKey key) throws Exception { return internalCompilePagesIndexOrdering(key.getSortTypes(), key.getSortChannels(), key.getSortOrders()); } });
private Class<? extends PagesIndexComparator> compilePagesIndexComparator( List<Type> sortTypes, List<Integer> sortChannels, List<SortOrder> sortOrders) { CallSiteBinder callSiteBinder = new CallSiteBinder(); ClassDefinition classDefinition = new ClassDefinition( a(PUBLIC, FINAL), makeClassName("PagesIndexComparator"), type(Object.class), type(PagesIndexComparator.class)); classDefinition.declareDefaultConstructor(a(PUBLIC)); generateCompareTo(classDefinition, callSiteBinder, sortTypes, sortChannels, sortOrders); return defineClass(classDefinition, PagesIndexComparator.class, callSiteBinder.getBindings(), getClass().getClassLoader()); }
joinCompiler, new LookupJoinOperators(), new OrderingCompiler());
new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()), new LookupJoinOperators(), new OrderingCompiler());
@Override public Operator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); LocalExchange localExchange = localExchangeFactory.getLocalExchange(driverContext.getLifespan()); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, LocalMergeSourceOperator.class.getSimpleName()); PageWithPositionComparator comparator = orderingCompiler.compilePageWithPositionComparator(types, sortChannels, orderings); List<LocalExchangeSource> sources = IntStream.range(0, localExchange.getBufferCount()) .boxed() .map(index -> localExchange.getNextSource()) .collect(toImmutableList()); return new LocalMergeSourceOperator(operatorContext, sources, types, comparator); }
private PagesIndexOrdering createPagesIndexComparator(List<Integer> sortChannels, List<SortOrder> sortOrders) { List<Type> sortTypes = sortChannels.stream() .map(types::get) .collect(toImmutableList()); return orderingCompiler.compilePagesIndexOrdering(sortTypes, sortChannels, sortOrders); }
@VisibleForTesting public PagesIndexOrdering internalCompilePagesIndexOrdering(List<Type> sortTypes, List<Integer> sortChannels, List<SortOrder> sortOrders) throws Exception { requireNonNull(sortChannels, "sortChannels is null"); requireNonNull(sortOrders, "sortOrders is null"); PagesIndexComparator comparator; try { Class<? extends PagesIndexComparator> pagesHashStrategyClass = compilePagesIndexComparator(sortTypes, sortChannels, sortOrders); comparator = pagesHashStrategyClass.newInstance(); } catch (Throwable e) { log.error(e, "Error compiling comparator for channels %s with order %s", sortChannels, sortChannels); comparator = new SimplePagesIndexComparator(sortTypes, sortChannels, sortOrders); } // we may want to load a separate PagesIndexOrdering for each comparator return new PagesIndexOrdering(comparator); }
@Benchmark public List<Page> runPagesMergeSortBenchmark(MergeSortedBenchmarkData data) { WorkProcessor<Page> sortedPagesWork = mergeSortedPages( data.getSplitPages().stream() .map(WorkProcessor::fromIterable) .collect(toImmutableList()), ORDERING_COMPILER.compilePageWithPositionComparator(data.getSortTypes(), data.getSortChannels(), data.getSortOrders()), data.getOutputChannels(), data.getTypes(), (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, newSimpleAggregatedMemoryContext(), new DriverYieldSignal()); ImmutableList.Builder<Page> sortedPages = ImmutableList.builder(); while (true) { sortedPagesWork.process(); if (sortedPagesWork.isFinished()) { return sortedPages.build(); } sortedPages.add(sortedPagesWork.getResult()); } }