private String getDatastoreName() { if (_job != null) { final Datastore datastore = _job.getDatastore(); if (datastore != null) { final String datastoreName = datastore.getName(); if (!StringUtils.isNullOrEmpty(datastoreName)) { return datastoreName; } } } return null; }
/** * Validates a job before executing it. * * @param context * @param job * @param analysisJob * * @throws NoSuchDatastoreException */ private void preExecuteJob(TenantContext context, DataCleanerJobContext job, AnalysisJob analysisJob) throws NoSuchDatastoreException { final Datastore datastore = analysisJob.getDatastore(); if (datastore instanceof PlaceholderDatastore) { // the job was materialized using a placeholder datastore - ie. // the real datastore was not found! final String sourceDatastoreName = job.getSourceDatastoreName(); logger.warn( "Raising a NoSuchDatastoreException since a PlaceholderDatastore was found at execution time: {}", sourceDatastoreName); throw new NoSuchDatastoreException(sourceDatastoreName); } }
@Override protected RowProcessingQueryOptimizer fetch() { final Datastore datastore = getAnalysisJob().getDatastore(); try (DatastoreConnection con = datastore.openConnection()) { final DataContext dataContext = con.getDataContext(); final Column[] columnArray = _physicalColumns.toArray(new Column[_physicalColumns.size()]); final Query baseQuery = dataContext.query().from(getTable()).select(columnArray).toQuery(); logger.debug("Base query for row processing: {}", baseQuery); // try to optimize return new RowProcessingQueryOptimizerImpl(datastore, getConsumersSorted(), baseQuery); } catch (final RuntimeException e) { logger.error("Failed to build query optimizer! {}", e.getMessage(), e); throw e; } } };
@Override public void rowProcessingSuccess(final AnalysisJob job, final RowProcessingMetrics metrics) { logger.info("rowProcessingSuccess: {}", job.getDatastore().getName()); _progressInformationPanel.updateProgressFinished(metrics.getTable()); _progressInformationPanel.addUserLog( "Processing of " + metrics.getTable().getName() + " is successful. Generating results..."); }
private static void addJobConfigurations(final AnalysisJob job, final Set<Datastore> datastores, final Set<Dictionary> dictionaries, final Set<StringPattern> stringPatterns, final Set<SynonymCatalog> synonymCatalogs) { datastores.add(job.getDatastore()); Iterators.concat(job.getAnalyzerJobs().iterator(), job.getFilterJobs().iterator(), job.getTransformerJobs().iterator()).forEachRemaining(component -> { component.getDescriptor().getConfiguredProperties().forEach(descriptor -> { final Class<?> type = descriptor.getBaseType(); if (type == Datastore.class) { datastores.addAll(getProperties(component, descriptor)); } else if (type == Dictionary.class) { dictionaries.addAll(getProperties(component, descriptor)); } else if (type == StringPattern.class) { stringPatterns.addAll(getProperties(component, descriptor)); } else if (type == SynonymCatalog.class) { synonymCatalogs.addAll(getProperties(component, descriptor)); } }); for (final OutputDataStreamJob outputDataStreamJob : component.getOutputDataStreamJobs()) { addJobConfigurations(outputDataStreamJob.getJob(), datastores, dictionaries, stringPatterns, synonymCatalogs); } }); }
@Override public AnalysisJob getAnalysisJob(Map<String, String> variableOverrides, Map<String, String> overrideProperties) { if ((variableOverrides == null || variableOverrides.isEmpty()) && overrideProperties == null) { // cached job definition may be used, if not outdated final long configurationLastModified = _tenantContext.getConfigurationFile().getLastModified(); long lastModified = Math.max(_file.getLastModified(), configurationLastModified); if (_job == null || lastModified != _lastModifiedCache) { synchronized (this) { lastModified = Math.max(_file.getLastModified(), configurationLastModified); if (_job == null || lastModified != _lastModifiedCache) { _lastModifiedCache = lastModified; final DataCleanerConfiguration configuration = _tenantContext.getConfiguration(); final MonitorJobReader reader = new MonitorJobReader(configuration, _file); _job = reader.readJob(); _sourceDatastoreName = _job.getDatastore().getName(); } } } return _job; } final DataCleanerConfiguration configuration = _tenantContext.getConfiguration(overrideProperties); final MonitorJobReader reader = new MonitorJobReader(configuration, _file); final AnalysisJob job = reader.readJob(variableOverrides); _sourceDatastoreName = job.getDatastore().getName(); return job; }
final Datastore datastore = analysisJob.getDatastore(); if (!HadoopJobExecutionUtils.isValidSourceDatastore(datastore)) { final Exception exception = new Exception(
@Override public void rowProcessingBegin(final AnalysisJob job, final RowProcessingMetrics metrics) { logger.info("rowProcessingBegin: {}", job.getDatastore().getName()); final int expectedRows = metrics.getExpectedRows(); final Table table = metrics.getTable();
final Datastore datastore = analysisJob.getDatastore(); if (!(datastore instanceof OutputDataStreamDatastore)) { final DataContextType dataContextType = new DataContextType();
public void importJob(final AnalysisJob job) { _builder.setDatastore(job.getDatastore());
public AnalysisResultFuture run() { _sparkJobContext.triggerOnJobStart(); final AnalysisJob analysisJob = _sparkJobContext.getAnalysisJob(); final Datastore datastore = analysisJob.getDatastore();
countQuery.getSelectClause().getItem(0).setFunctionApproximationAllowed(true); final Datastore datastore = _publisher.getStream().getAnalysisJob().getDatastore(); try (DatastoreConnection connection = datastore.openConnection()) { try (DataSet countDataSet = connection.getDataContext().executeQuery(countQuery)) {
return new ComponentContextImpl(_job); } else if (baseType == Datastore.class && _job != null) { return _job.getDatastore(); } else if (baseType == DatastoreConnection.class && _job != null) { throw new UnsupportedOperationException("DatastoreConnections cannot be injected as of AnalyzerBeans 0.16. "
new RowConsumerTaskListener(getAnalysisJob(), analysisListener, getTaskRunner()); final Datastore datastore = getAnalysisJob().getDatastore();