public Configuration getFsConf() { return plugin.getFsConf(); }
@JsonIgnore public Configuration getFsConf() { return plugin.getFsConf(); }
@JsonIgnore public Configuration getFsConf() { return plugin.getFsConf(); }
@Override public RecordReader getRecordReader(OperatorContext context, FileSystemWrapper fs, FileStatus status) throws ExecutionSetupException { try { final ParquetMetadata footer = ParquetFileReader.readFooter(fsPlugin.getFsConf(), status, ParquetMetadataConverter.NO_FILTER); if (footer.getBlocks().size() == 0) { return null; } return new PreviewReader(context, fs, status, footer); } catch (IOException e) { throw new ExecutionSetupException(e); } }
public ParquetGroupScanUtils( String userName, FileSelection selection, FileSystemPlugin plugin, ParquetFormatPlugin formatPlugin, String selectionRoot, List<SchemaPath> columns, BatchSchema schema, Map<String, GlobalDictionaryFieldInfo> globalDictionaryColumns, List<ParquetFilterCondition> conditions, SystemOptionManager optionManager) throws IOException { this.schema = schema; this.formatPlugin = formatPlugin; this.conditions = conditions; this.columns = columns; this.fs = ImpersonationUtil.createFileSystem(userName, plugin.getFsConf()); this.plugin = plugin; this.selectionRoot = selectionRoot; this.entries = selection.getStatuses(); this.globalDictionaryColumns = (globalDictionaryColumns == null)? Collections.<String, GlobalDictionaryFieldInfo>emptyMap() : globalDictionaryColumns; this.optionManager = optionManager; init(); }
public VectorContainer loadDictionary(String fieldName) throws IOException, ExecutionSetupException { final StoragePluginId id = config.getDictionaryEncodedFields().get(fieldName).getStoragePluginId(); final StoragePlugin storagePlugin = config.getCatalogService().getSource(id); if (storagePlugin instanceof FileSystemPlugin) { final FileSystemPlugin fsPlugin = (FileSystemPlugin) storagePlugin; final FileSystem fs = FileSystemWrapper.get(fsPlugin.getFsConf()); return ParquetFormatPlugin.loadDictionary(fs, new Path(config.getDictionaryEncodedFields().get(fieldName).getDictionaryPath()), context.getAllocator()); } else { throw new ExecutionSetupException(format("Storage plugin %s is not a filesystem plugin", id.getName())); } }
@Override public RecordReader getRecordReader(OperatorContext context, FileSystemWrapper dfs, EasyDatasetSplitXAttr splitAttributes, List<SchemaPath> columns) throws ExecutionSetupException { return new AvroRecordReader(context, splitAttributes.getPath(), splitAttributes.getStart(), splitAttributes.getLength(), columns, getFsPlugin().getFsConf()); }
public FileSystemWrapper getFs(String userName, OperatorStats stats) { if (!config.isImpersonationEnabled()) { userName = ImpersonationUtil.getProcessUserName(); } return ImpersonationUtil.createFileSystem(ImpersonationUtil.createProxyUgi(userName), getFsConf(), stats, getConnectionUniqueProperties()); }
private void initFromSelection(FileSelection selection, EasyFormatPlugin<?> formatPlugin) throws IOException { final FileSystemWrapper dfs = ImpersonationUtil.createFileSystem(userName, plugin.getFsConf()); this.selection = selection; BlockMapBuilder b = new BlockMapBuilder(dfs, plugin.getContext().getExecutors()); this.chunks = b.generateFileWork(selection.getStatuses(), formatPlugin.isBlockSplittable()); }
ParquetMetadata footer = ParquetFileReader.readFooter(fsPlugin.getFsConf(), firstFile, ParquetMetadataConverter.NO_FILTER);