@Override public boolean matches(RelOptRuleCall call) { final ScanRelBase scan = call.rel(1); if (scan.getPluginId().getType().equals(pluginType)) { try { if(scan.getTableMetadata().getSplitRatio() == 1.0d){ final List<String> partitionColumns = scan.getTableMetadata().getReadDefinition().getPartitionColumnsList(); return partitionColumns != null && !partitionColumns.isEmpty(); } } catch (NamespaceException e) { logger.warn("Unable to calculate split.", e); return false; } } return false; }
@Override public boolean matches(RelOptRuleCall call) { final ScanRelBase scan = call.rel(2); if (scan.getPluginId().getType().equals(pluginType)) { try { if(scan.getTableMetadata().getSplitRatio() == 1.0d){ final List<String> partitionColumns = scan.getTableMetadata().getReadDefinition().getPartitionColumnsList(); return partitionColumns != null && !partitionColumns.isEmpty(); } } catch (NamespaceException e) { logger.warn("Unable to calculate split.", e); return false; } } return false; }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { List<DatasetSplit> splits = new ArrayList<>(work.size()); BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } boolean storageImpersonationEnabled = dataset.getStoragePluginId().getCapabilities().getCapability(SourceCapabilities.STORAGE_IMPERSONATION); String userName = storageImpersonationEnabled ? getUserName() : ImpersonationUtil.getProcessUserName(); final ReadDefinition readDefinition = dataset.getReadDefinition(); return new HiveSubScan(splits, userName, schema, dataset.getName().getPathComponents(), filter, dataset.getStoragePluginId(), columns, readDefinition.getPartitionColumnsList()); }
final Map<String, Integer> partitionColumnsToIdMap = Maps.newHashMap(); int index = 0; for (String column : scanRel.getTableMetadata().getReadDefinition().getPartitionColumnsList()) { partitionColumnsToIdMap.put(column, index++);
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final BatchSchema schema = cachedRelDataType == null ? getDataset().getSchema(): BatchSchema.fromCalciteRowType(cachedRelDataType); // Create an abridged version of the splits to save network bytes. List<DatasetSplit> splits = work.stream().map( workSplit -> ProtostuffUtil.copy(workSplit.getSplit()) .setExtendedProperty(convertToScanXAttr(workSplit.getSplit().getExtendedProperty())) ).collect(Collectors.toList()); return new ParquetSubScan(dataset.getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), filter == null ? null : filter.getConditions(), dataset.getStoragePluginId(), columns, dataset.getReadDefinition().getPartitionColumnsList(), globalDictionaryEncodedColumns, dataset.getReadDefinition().getExtendedProperty()); }
private List<String> getPartitionColumnsForDataSet(String folder) throws Exception { URL stream = (TestParquetPartitionColumns.class.getClassLoader() .getResource(folder)); File fileDir = new File(stream.getFile()); DatasetPath parquet = new DatasetPath(ImmutableList.of("dfs", fileDir.getAbsolutePath())); DatasetConfig config = addDataSet(parquet); DremioTable table = p(CatalogService.class).get().getCatalog(SchemaConfig.newBuilder (DEFAULT_USERNAME) .build()) .getTable(config.getId().getId()); return table.getDatasetConfig().getReadDefinition() .getPartitionColumnsList(); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final List<DatasetSplit> splits = new ArrayList<>(work.size()); final BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } return new EasySubScan( getDataset().getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), dataset.getStoragePluginId(), columns, getDataset().getReadDefinition().getPartitionColumnsList(), getDataset().getReadDefinition().getExtendedProperty()); }
final JobConf job = new JobConf(hiveConf); if (metadata.getPartitionColumnsList().isEmpty()) { Class<? extends InputFormat<?, ?>> inputFormat = getInputFormatClass(job, table, null); allowParquetNative = allowParquetNative(allowParquetNative, inputFormat);