public HadoopResource(final Resource resource, final Configuration configuration, final String clusterReferenceName) { super(resource.getQualifiedPath()); _configuration = configuration; _clusterReferenceName = clusterReferenceName; }
@Override public String createPath(final Resource resource) { final String prefix = getScheme() + "://"; String path = resource.getQualifiedPath(); if (path.startsWith(prefix)) { path = path.substring(prefix.length()); } return path; }
final String qualifiedPath = resource.getQualifiedPath();
final String qualifiedPath = resource.getQualifiedPath();
/** * Creates a filename string to externalize, based on a given {@link Resource}. * * @param resource * @return * @throws UnsupportedOperationException */ protected String toFilename(final Resource resource) throws UnsupportedOperationException { if (resource instanceof FileResource) { return ((FileResource) resource).getFile().getPath(); } if (resource instanceof HadoopResource) { return ((HadoopResource) resource).getTemplatedPath(); } if (resource instanceof HdfsResource) { return resource.getQualifiedPath(); } throw new UnsupportedOperationException("Unsupported resource type: " + resource); }
public String getFilename() { if (_datastore instanceof FileDatastore) { if (_datastore instanceof CsvDatastore || _datastore instanceof JsonDatastore) { final Resource resource = ((ResourceDatastore) _datastore).getResource(); if (resource instanceof HadoopResource) { return resource.getQualifiedPath(); } } final String filename = ((FileDatastore) _datastore).getFilename(); return filename; } else { return null; } }
public Resource getResourceToUse(final Resource resource) { if (resource == null) { return null; } if (_hadoopConfiguration == null || resource instanceof HadoopResource) { return resource; } if (resource instanceof HdfsResource) { // wrap the resource with our known configuration return new HadoopResource(resource, _hadoopConfiguration, HadoopResource.DEFAULT_CLUSTERREFERENCE); } if (resource instanceof FileResource) { // this may very well be a path that was mis-interpreted as a local // file because no scheme was defined if (resource.getQualifiedPath().startsWith("/")) { return new HadoopResource(resource, _hadoopConfiguration, HadoopResource.DEFAULT_CLUSTERREFERENCE); } } return resource; }
/** * Creates a {@link Resource} replacement to use for configured properties. * * @param resource * @param partitionNumber * @return a replacement resource, or null if it shouldn't be replaced */ private Resource createReplacementResource(final Resource resource, final int partitionNumber) { final String formattedPartitionNumber = String.format("%05d", partitionNumber); if (resource instanceof HdfsResource || resource instanceof HadoopResource) { final String path = resource.getQualifiedPath() + "/part-" + formattedPartitionNumber; final URI uri = URI.create(path); return HdfsHelper.createHelper().getResourceToUse(uri); } if (resource instanceof FileResource) { final File file = ((FileResource) resource).getFile(); if (file.exists() && file.isFile()) { // a file already exists - we cannot just create a directory // then return resource; } if (!file.exists()) { file.mkdirs(); } return new FileResource(resource.getQualifiedPath() + "/part-" + formattedPartitionNumber); } return null; }
public static OutputWriter getWriter(final Resource resource, final String[] headers, final String encoding, final char separatorChar, final char quoteChar, final char escapeChar, final boolean includeHeader, final InputColumn<?>... columns) { final CsvConfiguration csvConfiguration = getConfiguration(encoding, separatorChar, quoteChar, escapeChar, includeHeader); CsvOutputWriter outputWriter; final String qualifiedPath = resource.getQualifiedPath(); synchronized (outputWritersPerPath) { outputWriter = outputWritersPerPath.get(qualifiedPath); if (outputWriter == null) { if (resource instanceof FileResource) { final File file = ((FileResource) resource).getFile(); final File parentFile = file.getParentFile(); if (parentFile != null && !parentFile.exists()) { parentFile.mkdirs(); } } outputWritersPerPath.put(qualifiedPath, outputWriter); counters.put(qualifiedPath, new AtomicInteger(1)); outputWriter = new CsvOutputWriter(resource, csvConfiguration, headers, columns); // write the headers } else { outputWriter = new CsvOutputWriter(resource, csvConfiguration, headers, columns); counters.get(qualifiedPath).incrementAndGet(); } } return outputWriter; }
public static Datastore createDatastoreFromEnum(final FileDatastoreEnum fileDatastore, final Resource resource, final String datastoreName) { if (fileDatastore == null) { throw new IllegalArgumentException("Unrecognized file type for: " + resource.getQualifiedPath()); return new CsvDatastore(datastoreName, resource, csvConfiguration); case EXCEL: return new ExcelDatastore(datastoreName, resource, resource.getQualifiedPath()); case ACCESS: return new AccessDatastore(datastoreName, resource.getQualifiedPath()); case SAS: final FileResource fileResource = (FileResource) resource; return new SasDatastore(datastoreName, fileResource.getFile()); case DBASE: return new DbaseDatastore(datastoreName, resource.getQualifiedPath()); case JSON: return new JsonDatastore(datastoreName, resource); case OPENOFFICE: return new OdbDatastore(datastoreName, resource.getQualifiedPath()); case XML: return new XmlDatastore(datastoreName, resource.getQualifiedPath()); default: throw new IllegalArgumentException("No such datastore type");
private static void saveResult(final AnalysisResultFuture result, final Resource resultResource) { final AnalysisResultSaveHandler analysisResultSaveHandler = new AnalysisResultSaveHandler(result, resultResource); try { analysisResultSaveHandler.saveOrThrow(); } catch (final SerializationException e) { // attempt to save what we can - and then rethrow final AnalysisResult safeAnalysisResult = analysisResultSaveHandler.createSafeAnalysisResult(); if (safeAnalysisResult == null) { logger.error("Serialization of result failed without any safe result elements to persist"); } else { final Map<ComponentJob, AnalyzerResult> unsafeResultElements = analysisResultSaveHandler.getUnsafeResultElements(); logger.error("Serialization of result failed with the following unsafe elements: {}", unsafeResultElements); logger.warn("Partial AnalysisResult will be persisted to filename '{}'", resultResource.getQualifiedPath()); analysisResultSaveHandler.saveWithoutUnsafeResultElements(); } // rethrow the exception regardless throw e; } }
logger.info("The resource path is " + resource.getQualifiedPath()); return nextPageController(resource);
private FixedWidthDatastore createDatastore(final String name, final Resource resource, final boolean failOnInconsistencies, final boolean skipEbcdicHeader, final boolean eolPresent) { final int[] valueWidths = getValueWidths(failOnInconsistencies); return new FixedWidthDatastore(name, resource, resource.getQualifiedPath(), _encodingComboBox.getSelectedItem(), valueWidths, failOnInconsistencies, skipEbcdicHeader, eolPresent, getHeaderLine(), getColumnNames()); }
private void getResultFileFromCluster(TenantContext tenantContext, ExecutionLogger executionLogger, String hadoopResultFileName, String jobName) { HdfsResource resultsResource = null; try { resultsResource = new HdfsResource(HadoopUtils.getFileSystem().getUri().resolve(hadoopResultFileName) .toString()); if (resultsResource != null && resultsResource.isExists()) { final RepositoryFolder repositoryResultFolder = tenantContext.getResultFolder(); final String fileName = HadoopJobExecutionUtils.getUrlReadyJobName(jobName) + FileFilters.ANALYSIS_RESULT_SER.getExtension(); final Resource resourceFile = repositoryResultFolder.createFile(fileName, null).toResource(); logger.info("Writing the result to" + resourceFile.getQualifiedPath()); FileHelper.copy(resultsResource, resourceFile); } else { final String message = "An error has occured while running the job. The result was not persisted on Hadoop. Please check Hadoop and/or DataCleaner logs"; final Exception error = new Exception(message); executionLogger.setStatusFailed(null, null, error); } } catch (Exception e) { executionLogger.setStatusFailed(null, null, e); } } }
/** * Writes the {@link Workbook} to a {@link Resource}. The {@link Workbook} will be closed as a result of this * operation! * * @param dataContext * @param wb */ public static void writeAndCloseWorkbook(ExcelDataContext dataContext, final Workbook wb) { // first write to a temp file to avoid that workbook source is the same // as the target (will cause read+write cyclic overflow) final Resource realResource = dataContext.getResource(); final Resource tempResource = new InMemoryResource(realResource.getQualifiedPath()); tempResource.write(out -> wb.write(out)); FileHelper.safeClose(wb); FileHelper.copy(tempResource, realResource); }
/** * Writes the {@link Workbook} to a {@link Resource}. The {@link Workbook} will be closed as a result of this * operation! * * @param dataContext * @param wb */ public static void writeAndCloseWorkbook(ExcelDataContext dataContext, final Workbook wb) { // first write to a temp file to avoid that workbook source is the same // as the target (will cause read+write cyclic overflow) final Resource realResource = dataContext.getResource(); final Resource tempResource = new InMemoryResource(realResource.getQualifiedPath()); tempResource.write(out -> wb.write(out)); FileHelper.safeClose(wb); FileHelper.copy(tempResource, realResource); }
private CsvDatastore createDatastore(final String name, final Resource resource, final boolean failOnInconsistentRecords) { return new CsvDatastore(name, resource, resource.getQualifiedPath(), getQuoteChar(), getSeparatorChar(), getEscapeChar(), getEncoding(), failOnInconsistentRecords, isMultilineValues(), getHeaderLine(), _columnNames); }
return ((ResourceDatastore) datastore).getResource().getQualifiedPath(); } else if (datastore instanceof FileDatastore) { return ((FileDatastore) datastore).getFilename();
final Resource resource = csvDatastore.getResource(); assert resource != null; final String datastorePath = resource.getQualifiedPath(); } else if (datastore instanceof JsonDatastore) { final JsonDatastore jsonDatastore = (JsonDatastore) datastore; final String datastorePath = jsonDatastore.getResource().getQualifiedPath(); final JavaRDD<String> rawInput; if (_minPartitions != null) { final String datastorePath = resource.getQualifiedPath(); final FixedWidthConfiguration fixedWidthConfiguration = fixedWidthDatastore.getConfiguration(); final JavaRDD<String> rawInput;