private Split getLocalQuerySplit(Session session, TableLayoutHandle handle) { SplitSource splitSource = localQueryRunner.getSplitManager().getSplits(session, handle, UNGROUPED_SCHEDULING); List<Split> splits = new ArrayList<>(); while (!splitSource.isFinished()) { splits.addAll(getNextBatch(splitSource)); } checkArgument(splits.size() == 1, "Expected only one split for a local query, but got %s splits", splits.size()); return splits.get(0); }
private synchronized void addConnectorInternal(MaterializedConnector connector) { checkState(!stopped.get(), "ConnectorManager is stopped"); ConnectorId connectorId = connector.getConnectorId(); checkState(!connectors.containsKey(connectorId), "A connector %s already exists", connectorId); connectors.put(connectorId, connector); splitManager.addConnectorSplitManager(connectorId, connector.getSplitManager()); pageSourceManager.addConnectorPageSourceProvider(connectorId, connector.getPageSourceProvider()); connector.getPageSinkProvider() .ifPresent(pageSinkProvider -> pageSinkManager.addConnectorPageSinkProvider(connectorId, pageSinkProvider)); connector.getIndexProvider() .ifPresent(indexProvider -> indexManager.addIndexProvider(connectorId, indexProvider)); connector.getPartitioningProvider() .ifPresent(partitioningProvider -> nodePartitioningManager.addPartitioningProvider(connectorId, partitioningProvider)); metadataManager.getProcedureRegistry().addProcedures(connectorId, connector.getProcedures()); connector.getAccessControl() .ifPresent(accessControl -> accessControlManager.addCatalogAccessControl(connectorId, accessControl)); metadataManager.getTablePropertyManager().addProperties(connectorId, connector.getTableProperties()); metadataManager.getColumnPropertyManager().addProperties(connectorId, connector.getColumnProperties()); metadataManager.getSchemaPropertyManager().addProperties(connectorId, connector.getSchemaProperties()); metadataManager.getSessionPropertyManager().addConnectorSessionProperties(connectorId, connector.getSessionProperties()); }
public SplitSource getSplits(Session session, TableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) { ConnectorId connectorId = layout.getConnectorId(); ConnectorSplitManager splitManager = getConnectorSplitManager(connectorId); ConnectorSession connectorSession = session.toConnectorSession(connectorId); ConnectorSplitSource source = splitManager.getSplits( layout.getTransactionHandle(), connectorSession, layout.getConnectorHandle(), splitSchedulingStrategy); SplitSource splitSource = new ConnectorAwareSplitSource(connectorId, layout.getTransactionHandle(), source); if (minScheduleSplitBatchSize > 1) { splitSource = new BufferingSplitSource(splitSource, minScheduleSplitBatchSize); } return splitSource; }
this.nodePartitioningManager = new NodePartitioningManager(nodeScheduler); this.splitManager = new SplitManager(new QueryManagerConfig()); this.blockEncodingManager = new BlockEncodingManager(typeRegistry); this.metadata = new MetadataManager(
private synchronized void removeConnectorInternal(ConnectorId connectorId) { splitManager.removeConnectorSplitManager(connectorId); pageSourceManager.removeConnectorPageSourceProvider(connectorId); pageSinkManager.removeConnectorPageSinkProvider(connectorId); indexManager.removeIndexProvider(connectorId); nodePartitioningManager.removePartitioningProvider(connectorId); metadataManager.getProcedureRegistry().removeProcedures(connectorId); accessControlManager.removeCatalogAccessControl(connectorId); metadataManager.getTablePropertyManager().removeProperties(connectorId); metadataManager.getColumnPropertyManager().removeProperties(connectorId); metadataManager.getSchemaPropertyManager().removeProperties(connectorId); metadataManager.getSessionPropertyManager().removeConnectorSessionProperties(connectorId); MaterializedConnector materializedConnector = connectors.remove(connectorId); if (materializedConnector != null) { Connector connector = materializedConnector.getConnector(); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(connector.getClass().getClassLoader())) { connector.shutdown(); } catch (Throwable t) { log.error(t, "Error shutting down connector: %s", connectorId); } } }
this.nodePartitioningManager = new NodePartitioningManager(nodeScheduler); this.splitManager = new SplitManager(); this.blockEncodingSerde = new BlockEncodingManager(typeRegistry); this.metadata = new MetadataManager(
@Override public Map<PlanNodeId, SplitSource> visitTableScan(TableScanNode node, Void context) { // get dataSource for table SplitSource splitSource = splitManager.getSplits( session, node.getLayout().get(), stageExecutionDescriptor.isScanGroupedExecution(node.getId()) ? GROUPED_SCHEDULING : UNGROUPED_SCHEDULING); splitSources.add(splitSource); return ImmutableMap.of(node.getId(), splitSource); }
public SplitSource getSplits(Session session, TableLayoutHandle layout) { String connectorId = layout.getConnectorId(); ConnectorSplitManager splitManager = getConnectorSplitManager(connectorId); // assumes connectorId and catalog are the same ConnectorSession connectorSession = session.toConnectorSession(connectorId); ConnectorSplitSource source = splitManager.getSplits(layout.getTransactionHandle(), connectorSession, layout.getConnectorHandle()); return new ConnectorAwareSplitSource(connectorId, layout.getTransactionHandle(), source); }
splitManager.addConnectorSplitManager(connectorId, connectorSplitManager); pageSourceManager.addConnectorPageSourceProvider(connectorId, connectorPageSourceProvider);
try (SplitSource splitSource = splitManager.getSplits(session, layouts.get(0).getLayout().getHandle(), UNGROUPED_SCHEDULING)) { while (!Thread.currentThread().isInterrupted()) { SplitBatch splitBatch = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000));
TableLayoutHandle layout = tableScan.getLayout().get(); SplitSource splitSource = splitManager.getSplits( session, layout,
@Override public Optional<SplitSource> visitTableScan(TableScanNode node, Void context) { // get dataSource for table SplitSource splitSource = splitManager.getSplits(session, node.getLayout().get()); return Optional.of(splitSource); }
private Split getLocalQuerySplit(Session session, TableLayoutHandle handle) { SplitSource splitSource = localQueryRunner.getSplitManager().getSplits(session, handle, UNGROUPED_SCHEDULING); List<Split> splits = new ArrayList<>(); while (!splitSource.isFinished()) { splits.addAll(getNextBatch(splitSource)); } checkArgument(splits.size() == 1, "Expected only one split for a local query, but got %s splits", splits.size()); return splits.get(0); }
private Split getLocalQuerySplit(Session session, TableLayoutHandle handle) { SplitSource splitSource = splitManager.getSplits(session, handle); List<Split> splits = new ArrayList<>(); splits.addAll(getFutureValue(splitSource.getNextBatch(1000))); while (!splitSource.isFinished()) { splits.addAll(getFutureValue(splitSource.getNextBatch(1000))); } checkArgument(splits.size() == 1, "Expected only one split for a local query, but got %s splits", splits.size()); return splits.get(0); }
TableLayoutHandle layout = tableScan.getLayout().get(); SplitSource splitSource = splitManager.getSplits(session, layout);