/** * Creates a {@link Datastore} replacement to use for configured properties * * @param cb * @param datastore * @param replacementResource * @return a replacement datastore, or null if it shouldn't be replaced */ private ResourceDatastore createReplacementDatastore(final ComponentBuilder cb, final ResourceDatastore datastore, final Resource replacementResource) { final String name = datastore.getName(); if (datastore instanceof CsvDatastore) { final CsvConfiguration csvConfiguration = ((CsvDatastore) datastore).getCsvConfiguration(); return new CsvDatastore(name, replacementResource, csvConfiguration); } if (datastore instanceof JsonDatastore) { return new JsonDatastore(name, replacementResource, ((JsonDatastore) datastore).getSchemaBuilder()); } logger.warn("Could not replace datastore '{}' because it is of an unsupported type: ", name, datastore.getClass().getSimpleName()); return datastore; }
@Override protected UsageAwareDatastoreConnection<UpdateableDataContext> createDatastoreConnection() { final UpdateableDataContext dataContext; final Resource resource = getResource(); if (resource == null) { logger.warn("Resource was not available, a local file reference will be created with path: {}", _filename); dataContext = new CsvDataContext(new File(_filename), getCsvConfiguration()); } else { dataContext = new CsvDataContext(resource, getCsvConfiguration()); } return new UpdateableDatastoreConnectionImpl<>(dataContext, this); }
datastoreElement.setAttribute("name", datastore.getName()); final String description = datastore.getDescription(); if (!Strings.isNullOrEmpty(description)) { datastoreElement.setAttribute("description", description); appendElement(datastoreElement, "quote-char", datastore.getQuoteChar()); appendElement(datastoreElement, "separator-char", datastore.getSeparatorChar()); appendElement(datastoreElement, "escape-char", datastore.getEscapeChar()); appendElement(datastoreElement, "encoding", datastore.getEncoding()); appendElement(datastoreElement, "fail-on-inconsistencies", datastore.isFailOnInconsistencies()); appendElement(datastoreElement, "multiline-values", datastore.isMultilineValues()); appendElement(datastoreElement, "header-line-number", datastore.getHeaderLineNumber()); if (datastore.getCustomColumnNames() != null && datastore.getCustomColumnNames().size() > 0) { final Element customColumnNamesElement = getDocument().createElement("custom-column-names"); datastoreElement.appendChild(customColumnNamesElement); datastore.getCustomColumnNames() .forEach(columnName -> appendElement(customColumnNamesElement, "column-name", columnName));
@Override public Element createDatastoreElement(DocumentBuilder documentBuilder) { final CsvDatastore datastore = new CsvDatastore(_name, _resource, _configuration); datastore.setDescription(_description); final DomConfigurationWriter writer = new DomConfigurationWriter(); final Element element = writer.externalize(datastore); return element; } }
@Override public WriteDataResult getResult() { _writeBuffer.flushBuffer(); final int updatedRowCount = _updatedRowCount.get(); final FileDatastore errorDatastore; if (_errorDataContext != null) { final Resource resource = _errorDataContext.getResource(); errorDatastore = new CsvDatastore(resource.getName(), resource); } else { errorDatastore = null; } return new WriteDataResultImpl(0, updatedRowCount, datastore, schemaName, tableName, _errorRowCount.get(), errorDatastore); }
public static boolean isHdfsResourcedDatastore(final Datastore datastore) { if (datastore instanceof CsvDatastore) { final CsvDatastore csvDatastore = (CsvDatastore) datastore; final Resource resource = csvDatastore.getResource(); if (!isHdfsResource(resource)) { return false; } } else if (datastore instanceof JsonDatastore) { final JsonDatastore jsonDatastore = (JsonDatastore) datastore; final Resource resource = jsonDatastore.getResource(); if (!isHdfsResource(resource)) { return false; } } else if (datastore instanceof FixedWidthDatastore) { final FixedWidthDatastore fixedWidthDatastore = (FixedWidthDatastore) datastore; final Resource resource = fixedWidthDatastore.getResource(); if (!isHdfsResource(resource)) { return false; } } else { // other type of datastore return false; } return true; }
public static boolean isValidMultilines(final CsvDatastore datastore) { final CsvConfiguration csvConfiguration = datastore.getCsvConfiguration(); if (csvConfiguration.isMultilineValues()) { return false; } return true; }
@Override public String toString() { return "CsvDatastore[name=" + getName() + ", filename=" + _filename + ", quoteChar='" + _quoteChar + "', separatorChar='" + _separatorChar + "', encoding=" + _encoding + ", headerLineNumber=" + _headerLineNumber + "]"; } }
@Override public WriteDataResult getResult() { _writeBuffer.flushBuffer(); final int writtenRowCount = _writtenRowCount.get(); final FileDatastore errorDatastore; if (_errorDataContext != null) { final Resource resource = _errorDataContext.getResource(); errorDatastore = new CsvDatastore(resource.getName(), resource); } else { errorDatastore = null; } return new WriteDataResultImpl(writtenRowCount, datastore, schemaName, tableName, _errorRowCount.get(), errorDatastore); }
final Resource resource = ((CsvDatastore) datastore).getResource(); if (resource instanceof FileResource) { return true;
public static boolean isValidEnconding(final CsvDatastore datastore) { final CsvConfiguration csvConfiguration = datastore.getCsvConfiguration(); final String encoding = csvConfiguration.getEncoding(); if (!encoding.equals(FileHelper.UTF_8_ENCODING)) { return false; } return true; }
_failOnInconsistenciesCheckBox.setSelected(originalDatastore.isFailOnInconsistencies()); _multilineValuesCheckBox.setSelected(originalDatastore.isMultilineValues()); _encodingComboBox.setSelectedItem(originalDatastore.getEncoding()); _headerLineComboBox.setSelectedItem(originalDatastore.getHeaderLineNumber()); final Character separatorChar = originalDatastore.getSeparatorChar(); String separator = null; if (separatorChar != null) { final Character quoteChar = originalDatastore.getQuoteChar(); final String quote; if (quoteChar == null) { final Character escapeChar = originalDatastore.getEscapeChar(); final String escape; if (escapeChar == null) { _columnNames = originalDatastore.getCustomColumnNames(); _addColumnNamesButton.setEnabled(true); } else {
@Override public WriteDataResult getResult() { _writeBuffer.flushBuffer(); final int updatedRowCount = _updatedRowCount.get(); final FileDatastore errorDatastore; if (_errorDataContext != null) { final Resource resource = _errorDataContext.getResource(); errorDatastore = new CsvDatastore(resource.getName(), resource); } else { errorDatastore = null; } return new WriteDataResultImpl(0, updatedRowCount, datastore, schemaName, tableName, _errorRowCount.get(), errorDatastore); }
private JavaRDD<InputRow> openSourceDatastore(final Datastore datastore) { if (datastore instanceof CsvDatastore) { final CsvDatastore csvDatastore = (CsvDatastore) datastore; final Resource resource = csvDatastore.getResource(); assert resource != null; final String datastorePath = resource.getQualifiedPath(); final CsvConfiguration csvConfiguration = csvDatastore.getCsvConfiguration();
final Resource resource = ((CsvDatastore) datastore).getResource(); final String filename = toFilename(resource); elem = toElement((CsvDatastore) datastore, filename);
private CsvDatastore createDatastore(final String name, final Resource resource, final boolean failOnInconsistentRecords) { return new CsvDatastore(name, resource, resource.getQualifiedPath(), getQuoteChar(), getSeparatorChar(), getEscapeChar(), getEncoding(), failOnInconsistentRecords, isMultilineValues(), getHeaderLine(), _columnNames); }
final CsvConfigurationDetection detection = new CsvConfigurationDetection(resource); final CsvConfiguration csvConfiguration = detection.suggestCsvConfiguration(); return new CsvDatastore(datastoreName, resource, csvConfiguration); case EXCEL: return new ExcelDatastore(datastoreName, resource, resource.getQualifiedPath());
final Datastore datastore = new CsvDatastore(file.getName(), file, csvConfiguration); return new WriteDataResultImpl(rowCount, datastore, null, null);
return new CsvDatastore(name, resource, filename, quoteChar, separatorChar, escapeChar, encoding, failOnInconsistencies, multilineValues, headerLineNumber, customColumnNames);