private void initFromSelection(FileSelection selection, EasyFormatPlugin<?> formatPlugin) throws IOException { final FileSystemWrapper dfs = ImpersonationUtil.createFileSystem(userName, plugin.getFsConf()); this.selection = selection; BlockMapBuilder b = new BlockMapBuilder(dfs, plugin.getContext().getExecutors()); this.chunks = b.generateFileWork(selection.getStatuses(), formatPlugin.isBlockSplittable()); }
private List<DatasetSplit> getSplits(DatasetConfig datasetConfig, EasyGroupScanUtils easyGroupScanUtils) throws IOException { final List<DatasetSplit> splits = Lists.newArrayList(); final ImplicitFilesystemColumnFinder finder = new ImplicitFilesystemColumnFinder(getFsPlugin().getContext().getOptionManager(), fs, GroupScan.ALL_COLUMNS); final List<CompleteFileWork> work = easyGroupScanUtils.getChunks(); final List<List<NameValuePair<?>>> pairs = finder.getImplicitFields(easyGroupScanUtils.getSelectionRoot(), work);
ListMultimap<String, NodeEndpoint> hostEndpointMap = FluentIterable.from(plugin.getContext().getExecutors()) .index(new Function<NodeEndpoint, String>() { @Override
private List<DatasetSplit> getSplits(ParquetGroupScanUtils parquetGroupScanUtils, ReadDefinition readDefinition) throws IOException { final List<DatasetSplit> splits = Lists.newArrayList(); final ImplicitFilesystemColumnFinder finder = new ImplicitFilesystemColumnFinder(getFsPlugin().getContext().getOptionManager(), fs, GroupScan.ALL_COLUMNS); List<RowGroupInfo> rowGroups = parquetGroupScanUtils.getRowGroupInfos(); if (fsPlugin.getContext().getOptionManager().getOption(ExecConstants.PARQUET_CACHED_ENTITY_SET_FILE_SIZE)) { length = rowGroupInfo.getStatus().getLen();