/** * Creates a configuration builder that can be used to set the input format's parameters to the config in a fluent * fashion. * * @return A config builder for setting parameters. */ public static ConfigBuilder configureFileFormat(GenericDataSourceBase<?, ?> target) { return new ConfigBuilder(target.getParameters()); }
protected GenericDataSourceBase<OUT, ?> translateToDataFlow() { String name = this.name != null ? this.name : this.inputFormat.toString(); if (name.length() > 100) { name = name.substring(0, 100); } @SuppressWarnings({ "unchecked", "rawtypes" }) GenericDataSourceBase<OUT, ?> source = new GenericDataSourceBase(this.inputFormat, new OperatorInformation<OUT>(getType()), name); source.setDegreeOfParallelism(dop); return source; } }
public void setSourceStatistics(GenericDataSourceBase<?, ?> source, FileBaseStatistics stats) { final String key = CACHE_KEY + this.statCounter++; this.dataStats.cacheBaseStatistics(stats, key); source.setStatisticsKey(key); }
format = getPactContract().getFormatWrapper().getUserCodeObject(); Configuration config = getPactContract().getParameters(); config.setClassLoader(format.getClass().getClassLoader()); format.configure(config); final String statisticsKey = getPactContract().getStatisticsKey(); final BaseStatistics cachedStatistics = statistics.getBaseStatistics(statisticsKey);
@Override public List<PlanNode> getAlternativePlans(CostEstimator estimator) { if (this.cachedPlans != null) { return this.cachedPlans; } SourcePlanNode candidate = new SourcePlanNode(this, "DataSource("+this.getPactContract().getName()+")"); candidate.updatePropertiesWithUniqueSets(getUniqueFields()); final Costs costs = new Costs(); if (FileInputFormat.class.isAssignableFrom(getPactContract().getFormatWrapper().getUserCodeClass()) && this.estimatedOutputSize >= 0) { estimator.addFileInputCost(this.estimatedOutputSize, costs); } candidate.setCosts(costs); // since there is only a single plan for the data-source, return a list with that element only List<PlanNode> plans = new ArrayList<PlanNode>(1); plans.add(candidate); this.cachedPlans = plans; return plans; }
@SuppressWarnings("unchecked") private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) { Operator<?> op = node.getOptimizerNode().getPactContract(); if (op instanceof GenericDataSourceBase) { return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType(); } else { throw new RuntimeException("Wrong operator type found in post pass."); } }
/** * Creates a new DataSourceNode for the given contract. * * @param pactContract * The data source contract object. */ public DataSourceNode(GenericDataSourceBase<?, ?> pactContract) { super(pactContract); if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) { throw new IllegalArgumentException("Input format has not been set."); } if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) { setDegreeOfParallelism(1); setSubtasksPerInstance(1); this.sequentialInput = true; } else { this.sequentialInput = false; } }
format = getPactContract().getFormatWrapper().getUserCodeObject(); Configuration config = getPactContract().getParameters(); config.setClassLoader(format.getClass().getClassLoader()); format.configure(config); final String statisticsKey = getPactContract().getStatisticsKey(); final BaseStatistics cachedStatistics = statistics.getBaseStatistics(statisticsKey);
@Override public List<PlanNode> getAlternativePlans(CostEstimator estimator) { if (this.cachedPlans != null) { return this.cachedPlans; } SourcePlanNode candidate = new SourcePlanNode(this, "DataSource("+this.getPactContract().getName()+")"); candidate.updatePropertiesWithUniqueSets(getUniqueFields()); final Costs costs = new Costs(); if (FileInputFormat.class.isAssignableFrom(getPactContract().getFormatWrapper().getUserCodeClass()) && this.estimatedOutputSize >= 0) { estimator.addFileInputCost(this.estimatedOutputSize, costs); } candidate.setCosts(costs); // since there is only a single plan for the data-source, return a list with that element only List<PlanNode> plans = new ArrayList<PlanNode>(1); plans.add(candidate); this.cachedPlans = plans; return plans; }
@SuppressWarnings("unchecked") private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) { Operator<?> op = node.getOptimizerNode().getPactContract(); if (op instanceof GenericDataSourceBase) { return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType(); } else { throw new RuntimeException("Wrong operator type found in post pass."); } }
/** * Creates a new DataSourceNode for the given contract. * * @param pactContract * The data source contract object. */ public DataSourceNode(GenericDataSourceBase<?, ?> pactContract) { super(pactContract); if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) { throw new IllegalArgumentException("Input format has not been set."); } if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) { setDegreeOfParallelism(1); setSubtasksPerInstance(1); this.sequentialInput = true; } else { this.sequentialInput = false; } }
/** * Creates a configuration builder that can be used to set the input format's parameters to the config in a fluent * fashion. * * @return A config builder for setting parameters. */ public static ConfigBuilder configureFileFormat(GenericDataSourceBase<?, ?> target) { return new ConfigBuilder(target.getParameters()); }