private static Set<Table> tablesFromReadEntities(Set<ReadEntity> readEntities) { return readEntities.stream() .filter(entity -> entity.getType() == Entity.Type.TABLE) .map(entity -> entity.getTable()) .collect(Collectors.toSet()); }
/** * If the optimizer has determined that it only has to read some of the partitions of the * target table to satisfy the query, then we know that the write side of update/delete * (and update/delete parts of merge) * can only write (at most) that set of partitions (since we currently don't allow updating * partition (or bucket) columns). So we want to replace the table level * WriteEntity in the outputs with WriteEntity for each of these partitions * ToDo: see if this should be moved to SemanticAnalyzer itself since it applies to any * insert which does a select against the same table. Then SemanticAnalyzer would also * be able to not use DP for the Insert... * * Note that the Insert of Merge may be creating new partitions and writing to partitions * which were not read (WHEN NOT MATCHED...). WriteEntity for that should be created * in MoveTask (or some other task after the query is complete). */ private List<ReadEntity> getRestrictedPartitionSet(Table targetTable) { List<ReadEntity> partitionsRead = new ArrayList<>(); for (ReadEntity re : inputs) { if (re.isFromTopLevelQuery && re.getType() == Entity.Type.PARTITION && isTargetTable(re, targetTable)) { partitionsRead.add(re); } } return partitionsRead; }
public Stream<String> getTableNames() { return queryInfo.getInputs().stream() .filter(readEntity -> readEntity.getType() == Type.TABLE) .map(readEntity -> readEntity.getTable().getFullyQualifiedName()); } }
/** * If the optimizer has determined that it only has to read some of the partitions of the * target table to satisfy the query, then we know that the write side of update/delete * (and update/delete parts of merge) * can only write (at most) that set of partitions (since we currently don't allow updating * partition (or bucket) columns). So we want to replace the table level * WriteEntity in the outputs with WriteEntity for each of these partitions * ToDo: see if this should be moved to SemanticAnalyzer itself since it applies to any * insert which does a select against the same table. Then SemanticAnalyzer would also * be able to not use DP for the Insert... * * Note that the Insert of Merge may be creating new partitions and writing to partitions * which were not read (WHEN NOT MATCHED...). WriteEntity for that should be created * in MoveTask (or some other task after the query is complete) */ private List<ReadEntity> getRestrictedPartitionSet(Table targetTable) { List<ReadEntity> partitionsRead = new ArrayList<>(); for(ReadEntity re : inputs) { if(re.isFromTopLevelQuery && re.getType() == Entity.Type.PARTITION && isTargetTable(re, targetTable)) { partitionsRead.add(re); } } return partitionsRead; } /**
private void putAccessedColumnsToReadEntity(HashSet<ReadEntity> inputs, ColumnAccessInfo columnAccessInfo) { Map<String, List<String>> tableToColumnAccessMap = columnAccessInfo.getTableToColumnAccessMap(); if (tableToColumnAccessMap != null && !tableToColumnAccessMap.isEmpty()) { for(ReadEntity entity: inputs) { List<String> cols; switch (entity.getType()) { case TABLE: cols = tableToColumnAccessMap.get(entity.getTable().getCompleteName()); if (cols != null && !cols.isEmpty()) { entity.getAccessedColumns().addAll(cols); } break; case PARTITION: cols = tableToColumnAccessMap.get(entity.getPartition().getTable().getCompleteName()); if (cols != null && !cols.isEmpty()) { entity.getAccessedColumns().addAll(cols); } break; default: // no-op } } } }
JSONArray inputPartitionInfo = new JSONArray(); for (ReadEntity input: work.getInputs()) { switch (input.getType()) { case TABLE: Table table = input.getTable();
private void putAccessedColumnsToReadEntity(HashSet<ReadEntity> inputs, ColumnAccessInfo columnAccessInfo) { Map<String, List<String>> tableToColumnAccessMap = columnAccessInfo.getTableToColumnAccessMap(); if (tableToColumnAccessMap != null && !tableToColumnAccessMap.isEmpty()) { for(ReadEntity entity: inputs) { List<String> cols; switch (entity.getType()) { case TABLE: cols = tableToColumnAccessMap.get(entity.getTable().getCompleteName()); if (cols != null && !cols.isEmpty()) { entity.getAccessedColumns().addAll(cols); } break; case PARTITION: cols = tableToColumnAccessMap.get(entity.getPartition().getTable().getCompleteName()); if (cols != null && !cols.isEmpty()) { entity.getAccessedColumns().addAll(cols); } break; default: // no-op } } } }
JSONArray inputPartitionInfo = new JSONArray(); for (ReadEntity input: work.getInputs()) { switch (input.getType()) { case TABLE: Table table = input.getTable();
@Test public void testGetJSONDependenciesJsonShhouldMatch() throws Exception { ExplainWork work = mockExplainWork(); when(work.getDependency()).thenReturn(true); // Mock inputs HashSet<ReadEntity> inputs = new HashSet<>(); // One input table Table table = mock(Table.class); when(table.getCompleteName()).thenReturn("table-name-mock"); when(table.getTableType()).thenReturn(TableType.EXTERNAL_TABLE); ReadEntity input1 = mock(ReadEntity.class); when(input1.getType()).thenReturn(Entity.Type.TABLE); when(input1.getTable()).thenReturn(table); inputs.add(input1); // And one partition Partition partition = mock(Partition.class); when(partition.getCompleteName()).thenReturn("partition-name-mock"); ReadEntity input2 = mock(ReadEntity.class); when(input2.getType()).thenReturn(Entity.Type.PARTITION); when(input2.getPartition()).thenReturn(partition); inputs.add(input2); when(work.getInputs()).thenReturn(inputs); JsonNode result = objectMapper.readTree(ExplainTask.getJSONDependencies(work).toString()); JsonNode expected = objectMapper.readTree("{\"input_partitions\":[{\"partitionName\":" + "\"partition-name-mock\"}],\"input_tables\":[{\"tablename\":\"table-name-mock\"," + "\"tabletype\":\"EXTERNAL_TABLE\"}]}"); assertEquals(expected, result); }
for (ReadEntity readEntity : queryInfo.getInputs()) { if (readEntity.getType() == Type.TABLE) { Table tableUsed = readEntity.getTable(); Map<String, Table> tempTables =
private Map<String, List<String>> getColsFromReadEntity(HashSet<ReadEntity> inputs) { Map<String, List<String>> tableColsMap = new HashMap<String, List<String>>(); for(ReadEntity entity: inputs) { switch (entity.getType()) { case TABLE: if (entity.getAccessedColumns() != null && !entity.getAccessedColumns().isEmpty()) { tableColsMap.put(entity.getTable().getCompleteName(), entity.getAccessedColumns()); } break; case PARTITION: if (entity.getAccessedColumns() != null && !entity.getAccessedColumns().isEmpty()) { tableColsMap.put(entity.getPartition().getTable().getCompleteName(), entity.getAccessedColumns()); } break; default: // no-op } } return tableColsMap; }
ReadEntity.Type type = readEntity.getType();
ReadEntity.Type type = readEntity.getType();
ReadEntity.Type typ = re.getType(); switch(typ) {
ReadEntity.Type typ = re.getType(); switch(typ) {
switch (input.getType()) { case DATABASE: compBuilder.setDbName(input.getDatabase().getName());