@Override public List<InputColumn<?>> getSourceColumns() { return _delegateJob.getSourceColumns(); }
@Override protected Map<InputColumn<?>, InputColumn<?>> getInputColumnConversion(final AnalysisJob wrappedAnalysisJob) { final Collection<InputColumn<?>> sourceColumns = wrappedAnalysisJob.getSourceColumns(); if (input.length < sourceColumns.size()) { throw new IllegalStateException( "Wrapped job defines " + sourceColumns.size() + " columns, but transformer input only defines " + input.length); } final Map<InputColumn<?>, InputColumn<?>> result = new LinkedHashMap<>(); int i = 0; final Iterator<InputColumn<?>> it = sourceColumns.iterator(); while (it.hasNext()) { final InputColumn<?> parentColumn = input[i]; final InputColumn<?> childColumn = it.next(); result.put(parentColumn, childColumn); i++; } return result; }
public Table[] getTables(final SourceColumnFinder sourceColumnFinder, final ComponentJob componentJob, Column[] physicalColumns) { if (physicalColumns == null) { physicalColumns = getPhysicalColumns(sourceColumnFinder, componentJob); } final Table[] tables; if (physicalColumns.length == 0) { // if not dependent on any specific tables, make component available // for all tables final Set<Table> allTables = new HashSet<>(); final Collection<InputColumn<?>> allSourceColumns = _analysisJob.getSourceColumns(); for (final InputColumn<?> inputColumn : allSourceColumns) { allTables.add(inputColumn.getPhysicalColumn().getTable()); } tables = allTables.toArray(new Table[allTables.size()]); } else { tables = MetaModelHelper.getTables(physicalColumns); } if (tables.length > 1) { if (!componentJob.getDescriptor().isMultiStreamComponent()) { throw new IllegalStateException("Component has input columns from multiple tables: " + componentJob); } } if (tables.length == 0) { throw new IllegalStateException("Component has no dependent tables: " + componentJob); } return tables; }
/** * Inspects the row processed tables primary keys. If all primary keys are * in the source columns of the AnalysisJob, they will be added to the * physically queried columns. * * Adding the primary keys to the query is a trade-off: It helps a lot in * making eg. annotated rows referenceable to the source table, but it may * also potentially make the job heavier to execute since a lot of (unique) * values will be retrieved. */ public void addPrimaryKeysIfSourced() { final List<Column> primaryKeyColumns = getTable().getPrimaryKeys(); if (primaryKeyColumns == null || primaryKeyColumns.isEmpty()) { logger.info("No primary keys defined for table {}, not pre-selecting primary keys", getTable().getName()); return; } final Collection<InputColumn<?>> sourceInputColumns = getAnalysisJob().getSourceColumns(); final List<Column> sourceColumns = CollectionUtils.map(sourceInputColumns, InputColumn::getPhysicalColumn); for (final Column primaryKeyColumn : primaryKeyColumns) { if (!sourceColumns.contains(primaryKeyColumn)) { logger.info("Primary key column {} not added to source columns, not pre-selecting primary keys"); return; } } addPhysicalColumns(primaryKeyColumns); }
@Override public InputRow call(final Tuple2<Object[], Long> tuple) throws Exception { final Object[] values = tuple._1; final Long rowNumber = tuple._2; final MockInputRow inputRow = new MockInputRow(rowNumber.intValue()); final List<InputColumn<?>> sourceColumns = _sparkJobContext.getAnalysisJob().getSourceColumns(); for (final InputColumn<?> sourceColumn : sourceColumns) { assert sourceColumn.isPhysicalColumn(); final int columnIndex = sourceColumn.getPhysicalColumn().getColumnNumber(); final Object value = values[columnIndex]; inputRow.put(sourceColumn, value); } return inputRow; }
/** * Constructor to use for creating a * {@link SourceTableRowProcessingPublisher} which feeds data from a source * datastore. * * @param publishers * @param stream */ public SourceTableRowProcessingPublisher(final RowProcessingPublishers publishers, final RowProcessingStream stream) { super(publishers, stream); _queryOptimizerRef = createQueryOptimizerRef(); final boolean aggressiveOptimizeSelectClause = SystemProperties.getBoolean(SystemProperties.QUERY_SELECTCLAUSE_OPTIMIZE, false); if (!aggressiveOptimizeSelectClause) { final Collection<InputColumn<?>> sourceColumns = stream.getAnalysisJob().getSourceColumns(); final List<Column> columns = new ArrayList<>(); for (final InputColumn<?> sourceColumn : sourceColumns) { final Column column = sourceColumn.getPhysicalColumn(); if (column != null && getTable().equals(column.getTable())) { columns.add(column); } } addPhysicalColumns(columns); } }
public void addSources(final AnalysisJob job) { addSources(new SourceColumns(job.getSourceColumns())); addSources(job.getFilterJobs()); addSources(job.getTransformerJobs()); addSources(job.getAnalyzerJobs()); }
final Collection<InputColumn<?>> sourceColumns = job.getSourceColumns(); for (final InputColumn<?> inputColumn : sourceColumns) { _builder.addSourceColumn((MetaModelInputColumn) inputColumn);
final Collection<InputColumn<?>> sourceColumns = analysisJob.getSourceColumns(); final String columnPathQualification = getColumnPathQualification(datastore, sourceColumns); for (final InputColumn<?> inputColumn : sourceColumns) {