@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split"); URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation(); exchangeClient.addLocation(location); return Optional::empty; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finished) { return Optional::empty; } this.split = (TestingSplit) split.getConnectorSplit(); blocked.set(null); return Optional::empty; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finished) { return Optional::empty; } this.split = (TestingSplit) split.getConnectorSplit(); blocked.set(null); return Optional::empty; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split"); URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation(); exchangeClient.addLocation(location); return Optional::empty; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finishing) { return Optional::empty; } this.split = split; Object splitInfo = split.getInfo(); if (splitInfo != null) { operatorContext.setInfoSupplier(() -> new SplitOperatorInfo(splitInfo)); } blocked.set(null); if (split.getConnectorSplit() instanceof EmptySplit) { pageSource = new EmptySplitPageSource(); } return () -> { if (pageSource instanceof UpdatablePageSource) { return Optional.of((UpdatablePageSource) pageSource); } return Optional.empty(); }; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finishing) { return Optional::empty; } this.split = split; Object splitInfo = split.getInfo(); if (splitInfo != null) { operatorContext.setInfoSupplier(() -> new SplitOperatorInfo(splitInfo)); } blocked.set(null); if (split.getConnectorSplit() instanceof EmptySplit) { pageSource = new EmptySplitPageSource(); } return () -> { if (pageSource instanceof UpdatablePageSource) { return Optional.of((UpdatablePageSource) pageSource); } return Optional.empty(); }; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finished) { return Optional::empty; } this.split = split; Object splitInfo = split.getInfo(); if (splitInfo != null) { operatorContext.setInfoSupplier(() -> new SplitOperatorInfo(splitInfo)); } blocked.set(null); if (split.getConnectorSplit() instanceof EmptySplit) { source = new EmptySplitPageSource(); } return () -> { if (source instanceof UpdatablePageSource) { return Optional.of((UpdatablePageSource) source); } return Optional.empty(); }; }
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { requireNonNull(split, "split is null"); requireNonNull(columns, "columns is null"); ConnectorSession connectorSession = session.toConnectorSession(split.getConnectorId()); return getPageSourceProvider(split).createPageSource(split.getTransactionHandle(), connectorSession, split.getConnectorSplit(), columns); }
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { requireNonNull(split, "split is null"); requireNonNull(columns, "columns is null"); ConnectorSession connectorSession = session.toConnectorSession(split.getConnectorId()); return getPageSourceProvider(split).createPageSource(split.getTransactionHandle(), connectorSession, split.getConnectorSplit(), columns); }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(this.split == null, "Table scan split already set"); if (finished) { return Optional::empty; } this.split = split; Object splitInfo = split.getInfo(); if (splitInfo != null) { operatorContext.setInfoSupplier(() -> new SplitOperatorInfo(splitInfo)); } blocked.set(null); if (split.getConnectorSplit() instanceof EmptySplit) { source = new EmptySplitPageSource(); } return () -> { if (source instanceof UpdatablePageSource) { return Optional.of((UpdatablePageSource) source); } return Optional.empty(); }; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(source == null, "Index source split already set"); IndexSplit indexSplit = (IndexSplit) split.getConnectorSplit(); // Normalize the incoming RecordSet to something that can be consumed by the index RecordSet normalizedRecordSet = probeKeyNormalizer.apply(indexSplit.getKeyRecordSet()); ConnectorPageSource result = index.lookup(normalizedRecordSet); source = new PageSourceOperator(result, operatorContext); Object splitInfo = split.getInfo(); if (splitInfo != null) { operatorContext.setInfoSupplier(() -> new SplitOperatorInfo(splitInfo)); } return Optional::empty; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkArgument(split.getConnectorSplit() instanceof RemoteSplit, "split is not a remote split"); checkState(!blockedOnSplits.isDone(), "noMoreSplits has been called already"); URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation(); ExchangeClient exchangeClient = closer.register(exchangeClientSupplier.get(operatorContext.localSystemMemoryContext())); exchangeClient.addLocation(location); exchangeClient.noMoreLocations(); pageProducers.add(exchangeClient.pages() .map(serializedPage -> { operatorContext.recordRawInput(serializedPage.getSizeInBytes()); return pagesSerde.deserialize(serializedPage); })); return Optional::empty; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkState(source == null, "Index source split already set"); IndexSplit indexSplit = (IndexSplit) split.getConnectorSplit(); // Normalize the incoming RecordSet to something that can be consumed by the index RecordSet normalizedRecordSet = probeKeyNormalizer.apply(indexSplit.getKeyRecordSet()); ConnectorPageSource result = index.lookup(normalizedRecordSet); source = new PageSourceOperator(result, operatorContext); Object splitInfo = split.getInfo(); if (splitInfo != null) { operatorContext.setInfoSupplier(() -> new SplitOperatorInfo(splitInfo)); } return Optional::empty; }
@Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkArgument(split.getConnectorSplit() instanceof RemoteSplit, "split is not a remote split"); checkState(!blockedOnSplits.isDone(), "noMoreSplits has been called already"); URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation(); ExchangeClient exchangeClient = closer.register(exchangeClientSupplier.get(operatorContext.localSystemMemoryContext())); exchangeClient.addLocation(location); exchangeClient.noMoreLocations(); pageProducers.add(exchangeClient.pages() .map(serializedPage -> { operatorContext.recordNetworkInput(serializedPage.getSizeInBytes()); return pagesSerde.deserialize(serializedPage); })); return Optional::empty; }
private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle()); checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle); return split -> { int bucket; if (split.getConnectorSplit() instanceof EmptySplit) { bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId(); } else { bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit()); } if (!split.getLifespan().isTaskWide()) { checkArgument(split.getLifespan().getId() == bucket); } return bucket; }; }
private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle()); checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle); return split -> { int bucket; if (split.getConnectorSplit() instanceof EmptySplit) { bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId(); } else { bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit()); } if (!split.getLifespan().isTaskWide()) { checkArgument(split.getLifespan().getId() == bucket); } return bucket; }; }
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { assertInstanceOf(split.getConnectorSplit(), FunctionAssertions.TestSplit.class); FunctionAssertions.TestSplit testSplit = (FunctionAssertions.TestSplit) split.getConnectorSplit(); if (testSplit.isRecordSet()) { RecordSet records = InMemoryRecordSet.builder(ImmutableList.of(BIGINT, VARCHAR, DOUBLE, BOOLEAN, BIGINT, VARCHAR, VARCHAR, TIMESTAMP_WITH_TIME_ZONE, VARBINARY, INTEGER)) .addRow( 1234L, "hello", 12.34, true, new DateTime(2001, 8, 22, 3, 4, 5, 321, DateTimeZone.UTC).getMillis(), "%el%", null, packDateTimeWithZone(new DateTime(1970, 1, 1, 0, 1, 0, 999, DateTimeZone.UTC).getMillis(), TimeZoneKey.getTimeZoneKey("Z")), Slices.wrappedBuffer((byte) 0xab), 1234) .build(); return new RecordPageSource(records); } else { return new FixedPageSource(ImmutableList.of(SOURCE_PAGE)); } } }
@Override public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) { assertInstanceOf(split.getConnectorSplit(), FunctionAssertions.TestSplit.class); FunctionAssertions.TestSplit testSplit = (FunctionAssertions.TestSplit) split.getConnectorSplit(); if (testSplit.isRecordSet()) { RecordSet records = InMemoryRecordSet.builder(ImmutableList.of(BIGINT, VARCHAR, DOUBLE, BOOLEAN, BIGINT, VARCHAR, VARCHAR, TIMESTAMP_WITH_TIME_ZONE, VARBINARY, INTEGER)) .addRow( 1234L, "hello", 12.34, true, new DateTime(2001, 8, 22, 3, 4, 5, 321, DateTimeZone.UTC).getMillis(), "%el%", null, packDateTimeWithZone(new DateTime(1970, 1, 1, 0, 1, 0, 999, DateTimeZone.UTC).getMillis(), TimeZoneKey.getTimeZoneKey("Z")), Slices.wrappedBuffer((byte) 0xab), 1234) .build(); return new RecordPageSource(records); } else { return new FixedPageSource(ImmutableList.of(SOURCE_PAGE)); } } }