@JsonProperty("formatConfig") public FormatPluginConfig getFormatConfig() { return formatPlugin.getConfig(); }
@Override public Writer getWriter(PhysicalOperator child) throws IOException { return formatPlugin.getWriter(child, userName, location, plugin, options); }
FormatPlugin formatPlugin = (FormatPlugin) c.newInstance(e.getKey(), context, e.getValue(), fsPlugin); pluginsByName.put(e.getKey(), formatPlugin); pluginsByConfig.put(formatPlugin.getConfig(), formatPlugin); formatMatchers.add(formatPlugin.getMatcher()); } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e1) { logger.warn("Failure initializing storage config named '{}' of type '{}'.", e.getKey(), e.getValue().getClass().getName(), e1); if (pluginsByName.containsKey(plugin.getName())) { continue; pluginsByName.put(plugin.getName(), plugin); pluginsByConfig.put(plugin.getConfig(), plugin); formatMatchers.add(plugin.getMatcher()); } catch (Exception e) { logger.warn(String.format("Failure while trying instantiate FormatPlugin %s.", pluginClass.getName()), e);
private SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPlugin formatPlugin, FileSystemWrapper fs, FileConfig fileConfig) throws IOException { final List<FileSystemCachedEntity> cachedEntities = Lists.newArrayList(); final FileStatus rootStatus = fs.getFileStatus(new Path(fileConfig.getLocation())); final Path combined = new Path("/", PathUtils.removeLeadingSlash(fileConfig.getLocation())); final FileSelection fileSelection = FileSelection.create(fs, combined); if (fileSelection == null) { return null; } // first entity is always a root if (rootStatus.isDirectory()) { cachedEntities.add(fromFileStatus(rootStatus)); } for (FileStatus dirStatus: fileSelection.getAllDirectories()) { cachedEntities.add(fromFileStatus(dirStatus)); } if(cachedEntities.isEmpty()){ // this is a single file. cachedEntities.add(fromFileStatus(rootStatus)); } final FileUpdateKey updateKey = new FileUpdateKey().setCachedEntitiesList(cachedEntities); final boolean hasDirectories = fileSelection.containsDirectories(); // Expand selection by copying it first used to check extensions of files in directory. final FileSelection fileSelectionWithoutDir = hasDirectories? fileSelection.minusDirectories(): fileSelection; if(fileSelectionWithoutDir == null){ // no files in the found directory, not a table. return null; } return formatPlugin.getDatasetAccessor(oldConfig, fs, fileSelectionWithoutDir, this, datasetPath, datasetPath.getName(), updateKey); }
@Override public BatchSchema getBatchSchema(final FileSelection selection, final FileSystemWrapper dfs) { final SabotContext context = formatPlugin.getContext(); try ( BufferAllocator sampleAllocator = context.getAllocator().newChildAllocator("sample-alloc", 0, Long.MAX_VALUE);
RecordReader reader = formatPlugin.getRecordReader(opCtxt, filesystem, status)) {
formatPlugin = formatCreator.newFormatPlugin(formatPluginConfig); datasetAccessor = formatPlugin.getDatasetAccessor(oldConfig, fs, fileSelectionWithoutDir, this, datasetPath, tableName, updateKey); try { if (matcher.matches(fs, fileSelection, codecFactory)) { datasetAccessor = matcher.getFormatPlugin().getDatasetAccessor(oldConfig, fs, fileSelectionWithoutDir, this, datasetPath, tableName, updateKey); if (datasetAccessor != null) { break;
final TextFileConfig textFileConfig = new TextFileConfig(); TextParsingSettings settings = new TextParsingSettings(); settings.set((TextFormatConfig) formatPlugin.getConfig()); textFileConfig.setComment(new Character((char) settings.getComment()).toString()); textFileConfig.setEscape(new Character((char) settings.getQuoteEscape()).toString()); final ExcelFormatPluginConfig excelFormatPluginConfig = (ExcelFormatPluginConfig)formatPlugin.getConfig(); if (excelFormatPluginConfig.xls) { final XlsFileConfig xlsFileConfig = new XlsFileConfig();