@Override public BatchSchema getSchema() { return dataset.getSchema(); }
@JsonIgnore public BatchSchema getSchema() { return dataset.getSchema(); }
@Override public RelDataType getRowType(RelDataTypeFactory relDataTypeFactory) { return dataset.getSchema().toCalciteRecordType(relDataTypeFactory, SYSTEM_COLUMNS); }
public BatchSchema getBatchSchema(){ return getTableMetadata().getSchema(); }
@Override public BatchSchema getSchema(FunctionLookupContext context) { return tableMetadata.getSchema().maskAndReorder(getProjectedColumns()); }
@Override public TableScan projectInvisibleColumn(String name) { // Check if already included if (getProjectedColumns().stream().map(path -> path.getRootSegment().getPath()) .anyMatch(java.util.function.Predicate.isEqual(name))) { // already included. return this; } // Check if present in table schema try { if (getTableMetadata().getSchema().findField(name) == null) { // safe-guarding. Method is supposed to throw if field is not found return null; } } catch (IllegalArgumentException e) { // field not found return null; } List<SchemaPath> columns = new ArrayList<>(); columns.addAll(projectedColumns); columns.add(SchemaPath.getSimplePath(name)); return this.cloneWithProject(columns); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { List<DatasetSplit> splits = new ArrayList<>(work.size()); BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } boolean storageImpersonationEnabled = dataset.getStoragePluginId().getCapabilities().getCapability(SourceCapabilities.STORAGE_IMPERSONATION); String userName = storageImpersonationEnabled ? getUserName() : ImpersonationUtil.getProcessUserName(); final ReadDefinition readDefinition = dataset.getReadDefinition(); return new HiveSubScan(splits, userName, schema, dataset.getName().getPathComponents(), filter, dataset.getStoragePluginId(), columns, readDefinition.getPartitionColumnsList()); }
private Prel newHardScan(RelDataType rowType) { TableMetadata metadata = Mockito.mock(TableMetadata.class); when(metadata.getName()).thenReturn(new NamespaceKey(ImmutableList.of("sys", "memory"))); when(metadata.getSchema()).thenReturn(SystemTable.MEMORY.getSchema()); StoragePluginId pluginId = new StoragePluginId(new SourceConfig().setConfig(new SystemPluginConf().toBytesString()), new SystemPluginConf(), SourceCapabilities.NONE); when(metadata.getStoragePluginId()).thenReturn(pluginId); List<SchemaPath> columns = FluentIterable.from(SystemTable.MEMORY.getSchema()).transform(new Function<Field, SchemaPath>(){ @Override public SchemaPath apply(Field input) { return SchemaPath.getSimplePath(input.getName()); }}).toList(); return new SystemScanPrel(cluster, traits, Mockito.mock(RelOptTable.class), metadata, columns, 1.0d, rowType); }
private Prel newScan(RelDataType rowType, double rowCount, double splitRatio) throws Exception { TableMetadata metadata = Mockito.mock(TableMetadata.class); when(metadata.getName()).thenReturn(new NamespaceKey(ImmutableList.of("sys", "version"))); when(metadata.getSchema()).thenReturn(SystemTable.VERSION.getSchema()); when(metadata.getSplitRatio()).thenReturn(splitRatio); StoragePluginId pluginId = new StoragePluginId(new SourceConfig().setConfig(new SystemPluginConf().toBytesString()), new SystemPluginConf(), SourceCapabilities.NONE); when(metadata.getStoragePluginId()).thenReturn(pluginId); List<SchemaPath> columns = FluentIterable.from(SystemTable.VERSION.getSchema()).transform(input -> SchemaPath.getSimplePath(input.getName())).toList(); final RelOptTable relOptTable = Mockito.mock(RelOptTable.class); when(relOptTable.getRowCount()).thenReturn(rowCount); return new SystemScanPrel(cluster, traits, relOptTable, metadata, columns, 1.0d, rowType); }
private Prel newSoftScan(RelDataType rowType) { TableMetadata metadata = Mockito.mock(TableMetadata.class); when(metadata.getName()).thenReturn(new NamespaceKey(ImmutableList.of("sys", "version"))); when(metadata.getSchema()).thenReturn(SystemTable.VERSION.getSchema()); StoragePluginId pluginId = new StoragePluginId(new SourceConfig().setConfig(new SystemPluginConf().toBytesString()), new SystemPluginConf(), SourceCapabilities.NONE); when(metadata.getStoragePluginId()).thenReturn(pluginId); List<SchemaPath> columns = FluentIterable.from(SystemTable.VERSION.getSchema()).transform(new Function<Field, SchemaPath>(){ @Override public SchemaPath apply(Field input) { return SchemaPath.getSimplePath(input.getName()); }}).toList(); return new SystemScanPrel(cluster, traits, Mockito.mock(RelOptTable.class), metadata, columns, 1.0d, rowType); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final BatchSchema schema = cachedRelDataType == null ? getDataset().getSchema(): BatchSchema.fromCalciteRowType(cachedRelDataType); // Create an abridged version of the splits to save network bytes. List<DatasetSplit> splits = work.stream().map( workSplit -> ProtostuffUtil.copy(workSplit.getSplit()) .setExtendedProperty(convertToScanXAttr(workSplit.getSplit().getExtendedProperty())) ).collect(Collectors.toList()); return new ParquetSubScan(dataset.getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), filter == null ? null : filter.getConditions(), dataset.getStoragePluginId(), columns, dataset.getReadDefinition().getPartitionColumnsList(), globalDictionaryEncodedColumns, dataset.getReadDefinition().getExtendedProperty()); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final List<DatasetSplit> splits = new ArrayList<>(work.size()); final BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } return new EasySubScan( getDataset().getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), dataset.getStoragePluginId(), columns, getDataset().getReadDefinition().getPartitionColumnsList(), getDataset().getReadDefinition().getExtendedProperty()); }
final int fieldCount = getLeafColumnCount(tableMetadata.getSchema(), projectedColumns);