public static TableLayout fromConnectorLayout(ConnectorId connectorId, ConnectorTransactionHandle transactionHandle, ConnectorTableLayout layout) { return new TableLayout(new TableLayoutHandle(connectorId, transactionHandle, layout.getHandle()), layout); }
private List<HivePartition> getOrComputePartitions(HiveTableLayoutHandle layoutHandle, ConnectorSession session, ConnectorTableHandle tableHandle) { if (layoutHandle.getPartitions().isPresent()) { return layoutHandle.getPartitions().get(); } else { TupleDomain<ColumnHandle> promisedPredicate = layoutHandle.getPromisedPredicate(); Predicate<Map<ColumnHandle, NullableValue>> predicate = convertToPredicate(promisedPredicate); List<ConnectorTableLayoutResult> tableLayoutResults = getTableLayouts(session, tableHandle, new Constraint<>(promisedPredicate, predicate), Optional.empty()); return ((HiveTableLayoutHandle) Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).getPartitions().get(); } }
public static TableLayout fromConnectorLayout(ConnectorId connectorId, ConnectorTransactionHandle transactionHandle, ConnectorTableLayout layout) { return new TableLayout(new TableLayoutHandle(connectorId, transactionHandle, layout.getHandle()), layout); }
@Test public void testSanity() { List<ConnectorTableLayoutResult> layouts = metadata.getTableLayouts(SESSION, tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertEquals(layouts.size(), 1); ConnectorTableLayoutResult layout = getOnlyElement(layouts); assertInstanceOf(layout.getTableLayout().getHandle(), RaptorTableLayoutHandle.class); ConnectorSplitSource splitSource = getSplits(raptorSplitManager, layout); int splitCount = 0; while (!splitSource.isFinished()) { splitCount += getSplits(splitSource, 1000).size(); } assertEquals(splitCount, 4); }
@Override public Optional<Object> getInfo(Session session, TableLayoutHandle handle) { ConnectorId connectorId = handle.getConnectorId(); ConnectorMetadata metadata = getMetadata(session, connectorId); ConnectorTableLayout tableLayout = metadata.getTableLayout(session.toConnectorSession(connectorId), handle.getConnectorHandle()); return metadata.getInfo(tableLayout.getHandle()); }
@Override public Optional<Object> getInfo(Session session, TableLayoutHandle handle) { ConnectorId connectorId = handle.getConnectorId(); ConnectorMetadata metadata = getMetadata(session, connectorId); ConnectorTableLayout tableLayout = metadata.getTableLayout(session.toConnectorSession(connectorId), handle.getConnectorHandle()); return metadata.getInfo(tableLayout.getHandle()); }
private static ConnectorSplitSource getSplits(RaptorSplitManager splitManager, ConnectorTableLayoutResult layout) { ConnectorTransactionHandle transaction = new RaptorTransactionHandle(); return splitManager.getSplits(transaction, SESSION, layout.getTableLayout().getHandle(), UNGROUPED_SCHEDULING); }
@Test public void testGetPartitionNamesUnpartitioned() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertEquals(getAllPartitions(getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).size(), 1); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), unpartitionedTableLayout); } }
@Test public void testReadTableBeforeCreationCompleted() { assertNoTables(); SchemaTableName tableName = new SchemaTableName("default", "temp_table"); ConnectorOutputTableHandle table = metadata.beginCreateTable( SESSION, new ConnectorTableMetadata(tableName, ImmutableList.of(), ImmutableMap.of()), Optional.empty()); List<SchemaTableName> tableNames = metadata.listTables(SESSION, Optional.empty()); assertTrue(tableNames.size() == 1, "Expected exactly one table"); ConnectorTableHandle tableHandle = metadata.getTableHandle(SESSION, tableName); List<ConnectorTableLayoutResult> tableLayouts = metadata.getTableLayouts(SESSION, tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertTrue(tableLayouts.size() == 1, "Expected exactly one layout."); ConnectorTableLayout tableLayout = tableLayouts.get(0).getTableLayout(); ConnectorTableLayoutHandle tableLayoutHandle = tableLayout.getHandle(); assertTrue(tableLayoutHandle instanceof MemoryTableLayoutHandle); assertTrue(((MemoryTableLayoutHandle) tableLayoutHandle).getDataFragments().isEmpty(), "Data fragments should be empty"); metadata.finishCreateTable(SESSION, table, ImmutableList.of(), ImmutableList.of()); }
protected void assertExpectedTableLayout(ConnectorTableLayout actualTableLayout, ConnectorTableLayout expectedTableLayout) { assertExpectedTableLayoutHandle(actualTableLayout.getHandle(), expectedTableLayout.getHandle()); assertEquals(actualTableLayout.getPredicate(), expectedTableLayout.getPredicate()); assertEquals(actualTableLayout.getDiscretePredicates().isPresent(), expectedTableLayout.getDiscretePredicates().isPresent()); actualTableLayout.getDiscretePredicates().ifPresent(actual -> { DiscretePredicates expected = expectedTableLayout.getDiscretePredicates().get(); assertEquals(actual.getColumns(), expected.getColumns()); assertEqualsIgnoreOrder(actual.getPredicates(), expected.getPredicates()); }); assertEquals(actualTableLayout.getStreamPartitioningColumns(), expectedTableLayout.getStreamPartitioningColumns()); assertEquals(actualTableLayout.getLocalProperties(), expectedTableLayout.getLocalProperties()); }
@Test public void testGetPartitionNamesUnpartitioned() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, Constraint.alwaysTrue(), Optional.empty()); assertEquals(getAllPartitions(getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).size(), 1); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), unpartitionedTableLayout); } }
protected void assertExpectedTableLayout(ConnectorTableLayout actualTableLayout, ConnectorTableLayout expectedTableLayout) { assertExpectedTableLayoutHandle(actualTableLayout.getHandle(), expectedTableLayout.getHandle()); assertEquals(actualTableLayout.getPredicate(), expectedTableLayout.getPredicate()); assertEquals(actualTableLayout.getDiscretePredicates().isPresent(), expectedTableLayout.getDiscretePredicates().isPresent()); actualTableLayout.getDiscretePredicates().ifPresent(actual -> { DiscretePredicates expected = expectedTableLayout.getDiscretePredicates().get(); assertEquals(actual.getColumns(), expected.getColumns()); assertEqualsIgnoreOrder(actual.getPredicates(), expected.getPredicates()); }); assertEquals(actualTableLayout.getStreamPartitioningColumns(), expectedTableLayout.getStreamPartitioningColumns()); assertEquals(actualTableLayout.getLocalProperties(), expectedTableLayout.getLocalProperties()); }
@Test public void testGetPartitionSplitsTableNotReadablePartition() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableNotReadable); assertNotNull(tableHandle); ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); assertNotNull(dsColumn); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); try { getSplitCount(splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING)); fail("Expected HiveNotReadableException"); } catch (HiveNotReadableException e) { assertThat(e).hasMessageMatching("Table '.*\\.presto_test_not_readable' is not readable: reason for not readable"); assertEquals(e.getTableName(), tableNotReadable); assertEquals(e.getPartition(), Optional.empty()); } } }
@Test public void testGetPartitionSplitsBatch() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING); assertEquals(getSplitCount(splitSource), partitionCount); } }
@Test public void testGetPartitionSplitsBatchUnpartitioned() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING); assertEquals(getSplitCount(splitSource), 1); } }
@Test public void testGetPartitionSplitsBatch() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING); assertEquals(getSplitCount(splitSource), partitionCount); } }
@Test public void testGetPartitionSplitsBatchUnpartitioned() { try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableUnpartitioned); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle(), UNGROUPED_SCHEDULING); assertEquals(getSplitCount(splitSource), 1); } }
/** * Tests information schema predicate pushdown when both schema and table name are specified. */ @Test public void testInformationSchemaPredicatePushdown() { TransactionId transactionId = transactionManager.beginTransaction(false); ImmutableMap.Builder<ColumnHandle, Domain> domains = new ImmutableMap.Builder<>(); domains.put(new InformationSchemaColumnHandle("table_schema"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_schema"))); domains.put(new InformationSchemaColumnHandle("table_name"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_view"))); Constraint<ColumnHandle> constraint = new Constraint<>(TupleDomain.withColumnDomains(domains.build())); InformationSchemaMetadata informationSchemaMetadata = new InformationSchemaMetadata("test_catalog", metadata); List<ConnectorTableLayoutResult> layoutResults = informationSchemaMetadata.getTableLayouts( createNewSession(transactionId), new InformationSchemaTableHandle("test_catalog", "information_schema", "views"), constraint, Optional.empty()); assertEquals(layoutResults.size(), 1); ConnectorTableLayoutHandle handle = layoutResults.get(0).getTableLayout().getHandle(); assertTrue(handle instanceof InformationSchemaTableLayoutHandle); InformationSchemaTableLayoutHandle tableHandle = (InformationSchemaTableLayoutHandle) handle; assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view"))); }
private List<ConnectorSplit> getAllSplits(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING)); } }
private List<ConnectorSplit> getAllSplits(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain), Optional.empty()); ConnectorTableLayoutHandle layoutHandle = getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); return getAllSplits(splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle, UNGROUPED_SCHEDULING)); } }