private static void assertCreateConnector(String metastoreUri) { HiveConnectorFactory connectorFactory = new HiveConnectorFactory( "hive-test", HiveConnector.class.getClassLoader(), Optional.empty()); Map<String, String> config = ImmutableMap.<String, String>builder() .put("hive.metastore.uri", metastoreUri) .build(); Connector connector = connectorFactory.create("hive-test", config, new TestingConnectorContext()); ConnectorTransactionHandle transaction = connector.beginTransaction(READ_UNCOMMITTED, true); assertInstanceOf(connector.getMetadata(transaction), ClassLoaderSafeConnectorMetadata.class); assertInstanceOf(connector.getSplitManager(), ClassLoaderSafeConnectorSplitManager.class); assertInstanceOf(connector.getPageSourceProvider(), ConnectorPageSourceProvider.class); connector.commit(transaction); }
splitManager = connector.getSplitManager(); checkState(splitManager != null, "Connector %s does not have a split manager", connectorId); Set<SystemTable> systemTables = connector.getSystemTables(); requireNonNull(systemTables, "Connector %s returned a null system tables set"); this.systemTables = ImmutableSet.copyOf(systemTables); Set<Procedure> procedures = connector.getProcedures(); requireNonNull(procedures, "Connector %s returned a null procedures set"); this.procedures = ImmutableSet.copyOf(procedures); connectorPageSourceProvider = connector.getPageSourceProvider(); requireNonNull(connectorPageSourceProvider, format("Connector %s returned a null page source provider", connectorId)); ConnectorRecordSetProvider connectorRecordSetProvider = null; try { connectorRecordSetProvider = connector.getRecordSetProvider(); requireNonNull(connectorRecordSetProvider, format("Connector %s returned a null record set provider", connectorId)); connectorPageSinkProvider = connector.getPageSinkProvider(); requireNonNull(connectorPageSinkProvider, format("Connector %s returned a null page sink provider", connectorId)); indexProvider = connector.getIndexProvider(); requireNonNull(indexProvider, format("Connector %s returned a null index provider", connectorId)); partitioningProvider = connector.getNodePartitioningProvider(); requireNonNull(partitioningProvider, format("Connector %s returned a null partitioning provider", connectorId)); accessControl = connector.getAccessControl();
@PreDestroy public synchronized void stop() { if (stopped.getAndSet(true)) { return; } for (Map.Entry<ConnectorId, MaterializedConnector> entry : connectors.entrySet()) { Connector connector = entry.getValue().getConnector(); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(connector.getClass().getClassLoader())) { connector.shutdown(); } catch (Throwable t) { log.error(t, "Error shutting down connector: %s", entry.getKey()); } } }
ConnectorSplitManager connectorSplitManager = connector.getSplitManager(); checkState(connectorSplitManager != null, "Connector %s does not have a split manager", connectorId); Set<SystemTable> systemTables = connector.getSystemTables(); requireNonNull(systemTables, "Connector %s returned a null system tables set"); Set<Procedure> procedures = connector.getProcedures(); requireNonNull(procedures, "Connector %s returned a null procedures set"); connectorPageSourceProvider = connector.getPageSourceProvider(); requireNonNull(connectorPageSourceProvider, format("Connector %s returned a null page source provider", connectorId)); ConnectorRecordSetProvider connectorRecordSetProvider = null; try { connectorRecordSetProvider = connector.getRecordSetProvider(); requireNonNull(connectorRecordSetProvider, format("Connector %s returned a null record set provider", connectorId)); connectorPageSinkProvider = connector.getPageSinkProvider(); requireNonNull(connectorPageSinkProvider, format("Connector %s returned a null page sink provider", connectorId)); ConnectorRecordSinkProvider connectorRecordSinkProvider = null; try { connectorRecordSinkProvider = connector.getRecordSinkProvider(); requireNonNull(connectorRecordSinkProvider, format("Connector %s returned a null record sink provider", connectorId)); connectorPageSinkProvider = new RecordPageSinkProvider(connectorRecordSinkProvider); indexProvider = connector.getIndexProvider(); requireNonNull(indexProvider, format("Connector %s returned a null index provider", connectorId));
@BeforeClass public void setup() throws Exception { EmbeddedCassandra.start(); String keyspace = "test_connector"; createTestTables(EmbeddedCassandra.getSession(), keyspace, DATE); String connectorId = "cassandra-test"; CassandraConnectorFactory connectorFactory = new CassandraConnectorFactory(connectorId); Connector connector = connectorFactory.create(connectorId, ImmutableMap.of( "cassandra.contact-points", EmbeddedCassandra.getHost(), "cassandra.native-protocol-port", Integer.toString(EmbeddedCassandra.getPort())), new TestingConnectorContext()); metadata = connector.getMetadata(CassandraTransactionHandle.INSTANCE); assertInstanceOf(metadata, CassandraMetadata.class); splitManager = connector.getSplitManager(); assertInstanceOf(splitManager, CassandraSplitManager.class); recordSetProvider = connector.getRecordSetProvider(); assertInstanceOf(recordSetProvider, CassandraRecordSetProvider.class); database = keyspace; table = new SchemaTableName(database, TABLE_ALL_TYPES.toLowerCase(ENGLISH)); tableUnpartitioned = new SchemaTableName(database, "presto_test_unpartitioned"); invalidTable = new SchemaTableName(database, "totally_invalid_table_name"); }
public ConnectorTransactionMetadata(ConnectorId connectorId, Connector connector, ConnectorTransactionHandle transactionHandle) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.connector = requireNonNull(connector, "connector is null"); this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null"); this.connectorMetadata = connector.getMetadata(transactionHandle); }
private synchronized void addCatalogConnector(String catalogName, String connectorId, ConnectorFactory factory, Map<String, String> properties) { Connector connector = createConnector(connectorId, factory, properties); addConnectorInternal(ConnectorType.STANDARD, catalogName, connectorId, connector); String informationSchemaId = makeInformationSchemaConnectorId(connectorId); addConnectorInternal(ConnectorType.INFORMATION_SCHEMA, catalogName, informationSchemaId, new InformationSchemaConnector(catalogName, nodeManager, metadataManager)); String systemId = makeSystemTablesConnectorId(connectorId); addConnectorInternal(ConnectorType.SYSTEM, catalogName, systemId, new SystemConnector( systemId, nodeManager, connector.getSystemTables(), transactionId -> transactionManager.getConnectorTransaction(transactionId, connectorId))); // Register session and table properties once per catalog metadataManager.getSessionPropertyManager().addConnectorSessionProperties(catalogName, connector.getSessionProperties()); metadataManager.getTablePropertyManager().addTableProperties(catalogName, connector.getTableProperties()); }
public void commit() { if (finished.compareAndSet(false, true)) { connector.commit(transactionHandle); } }
private ConnectorTransactionHandle beginTransaction(Connector connector) { if (connector instanceof InternalConnector) { return ((InternalConnector) connector).beginTransaction(transactionId, isolationLevel, readOnly); } else { return connector.beginTransaction(isolationLevel, readOnly); } }
private Catalog createTestingCatalog(String catalogName, ConnectorId connectorId) { ConnectorId systemId = createSystemTablesConnectorId(connectorId); Connector connector = createTestingConnector(); InternalNodeManager nodeManager = new InMemoryNodeManager(); return new Catalog( catalogName, connectorId, connector, createInformationSchemaConnectorId(connectorId), new InformationSchemaConnector(catalogName, nodeManager, metadata, accessControl), systemId, new SystemConnector( systemId, nodeManager, connector.getSystemTables(), transactionId -> transactionManager.getConnectorTransaction(transactionId, connectorId))); }
@BeforeClass public void setup() throws Exception { EmbeddedCassandra.start(); String keyspace = "test_connector"; createTestTables(EmbeddedCassandra.getSession(), keyspace, DATE); String connectorId = "cassandra-test"; CassandraConnectorFactory connectorFactory = new CassandraConnectorFactory(connectorId); Connector connector = connectorFactory.create(connectorId, ImmutableMap.of( "cassandra.contact-points", EmbeddedCassandra.getHost(), "cassandra.native-protocol-port", Integer.toString(EmbeddedCassandra.getPort())), new TestingConnectorContext()); metadata = connector.getMetadata(CassandraTransactionHandle.INSTANCE); assertInstanceOf(metadata, CassandraMetadata.class); splitManager = connector.getSplitManager(); assertInstanceOf(splitManager, CassandraSplitManager.class); recordSetProvider = connector.getRecordSetProvider(); assertInstanceOf(recordSetProvider, CassandraRecordSetProvider.class); database = keyspace; table = new SchemaTableName(database, TABLE_ALL_TYPES.toLowerCase(ENGLISH)); tableUnpartitioned = new SchemaTableName(database, "presto_test_unpartitioned"); invalidTable = new SchemaTableName(database, "totally_invalid_table_name"); }
public ConnectorTransactionMetadata(Connector connector, ConnectorTransactionHandle transactionHandle) { this.connector = requireNonNull(connector, "connector is null"); this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null"); this.connectorMetadataSupplier = Suppliers.memoize(() -> connector.getMetadata(transactionHandle)); }
public void commit() { if (finished.compareAndSet(false, true)) { connector.commit(transactionHandle); } }
private ConnectorTransactionHandle beginTransaction(Connector connector) { if (connector instanceof InternalConnector) { return ((InternalConnector) connector).beginTransaction(transactionId, isolationLevel, readOnly); } else { return connector.beginTransaction(isolationLevel, readOnly); } }
private static ConnectorId registerBogusConnector(CatalogManager catalogManager, TransactionManager transactionManager, AccessControl accessControl, String catalogName) { ConnectorId connectorId = new ConnectorId(catalogName); Connector connector = new TpchConnectorFactory().create(catalogName, ImmutableMap.of(), new TestingConnectorContext()); InMemoryNodeManager nodeManager = new InMemoryNodeManager(); MetadataManager metadata = MetadataManager.createTestMetadataManager(catalogManager); ConnectorId systemId = createSystemTablesConnectorId(connectorId); catalogManager.registerCatalog(new Catalog( catalogName, connectorId, connector, createInformationSchemaConnectorId(connectorId), new InformationSchemaConnector(catalogName, nodeManager, metadata, accessControl), systemId, new SystemConnector( systemId, nodeManager, connector.getSystemTables(), transactionId -> transactionManager.getConnectorTransaction(transactionId, connectorId)))); return connectorId; }
private static void assertCreateConnector(String metastoreUri) { HiveConnectorFactory connectorFactory = new HiveConnectorFactory( "hive-test", HiveConnector.class.getClassLoader(), Optional.empty()); Map<String, String> config = ImmutableMap.<String, String>builder() .put("hive.metastore.uri", metastoreUri) .build(); Connector connector = connectorFactory.create("hive-test", config, new TestingConnectorContext()); ConnectorTransactionHandle transaction = connector.beginTransaction(READ_UNCOMMITTED, true); assertInstanceOf(connector.getMetadata(transaction), ClassLoaderSafeConnectorMetadata.class); assertInstanceOf(connector.getSplitManager(), ClassLoaderSafeConnectorSplitManager.class); assertInstanceOf(connector.getPageSourceProvider(), ConnectorPageSourceProvider.class); connector.commit(transaction); }
@Test public void testCreateConnector() { MongoPlugin plugin = new MongoPlugin(); ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); Connector connector = factory.create("test", ImmutableMap.of("mongodb.seeds", seed), new TestingConnectorContext()); Type type = getOnlyElement(plugin.getTypes()); assertEquals(type, OBJECT_ID); connector.shutdown(); }
private static void registerConnector( CatalogManager catalogManager, TransactionManager transactionManager, String catalogName, ConnectorId connectorId, Connector connector) { ConnectorId systemId = createSystemTablesConnectorId(connectorId); InternalNodeManager nodeManager = new InMemoryNodeManager(); MetadataManager metadata = MetadataManager.createTestMetadataManager(catalogManager); catalogManager.registerCatalog(new Catalog( catalogName, connectorId, connector, createInformationSchemaConnectorId(connectorId), new InformationSchemaConnector(catalogName, nodeManager, metadata, new AllowAllAccessControl()), systemId, new SystemConnector( systemId, nodeManager, connector.getSystemTables(), transactionId -> transactionManager.getConnectorTransaction(transactionId, connectorId)))); }
private static void assertCreateConnector(String metastoreUri) { HiveConnectorFactory connectorFactory = new HiveConnectorFactory( "hive-test", ImmutableMap.<String, String>builder() .put("node.environment", "test") .put("hive.metastore.uri", metastoreUri) .build(), HiveConnector.class.getClassLoader(), null, new TypeRegistry(), new GroupByHashPageIndexerFactory()); Connector connector = connectorFactory.create("hive-test", ImmutableMap.<String, String>of()); ConnectorTransactionHandle transaction = connector.beginTransaction(READ_UNCOMMITTED, true); assertInstanceOf(connector.getMetadata(transaction), ClassLoaderSafeConnectorMetadata.class); assertInstanceOf(connector.getSplitManager(), ClassLoaderSafeConnectorSplitManager.class); assertInstanceOf(connector.getPageSourceProvider(), ConnectorPageSourceProvider.class); connector.commit(transaction); }
private synchronized void removeConnectorInternal(ConnectorId connectorId) { splitManager.removeConnectorSplitManager(connectorId); pageSourceManager.removeConnectorPageSourceProvider(connectorId); pageSinkManager.removeConnectorPageSinkProvider(connectorId); indexManager.removeIndexProvider(connectorId); nodePartitioningManager.removePartitioningProvider(connectorId); metadataManager.getProcedureRegistry().removeProcedures(connectorId); accessControlManager.removeCatalogAccessControl(connectorId); metadataManager.getTablePropertyManager().removeProperties(connectorId); metadataManager.getColumnPropertyManager().removeProperties(connectorId); metadataManager.getSchemaPropertyManager().removeProperties(connectorId); metadataManager.getSessionPropertyManager().removeConnectorSessionProperties(connectorId); MaterializedConnector materializedConnector = connectors.remove(connectorId); if (materializedConnector != null) { Connector connector = materializedConnector.getConnector(); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(connector.getClass().getClassLoader())) { connector.shutdown(); } catch (Throwable t) { log.error(t, "Error shutting down connector: %s", connectorId); } } }