List<Map<String, String>> configs = getConfigs( job ); JobConf[] jobConfs = HadoopUtil.getJobConfs( job, configs ); InputFormat[] inputFormats = getInputFormats( jobConfs ); return collapse( getSplits( inputFormats, jobConfs, new int[]{numSplits} ), configs ); return collapse( getSplits( inputFormats, jobConfs, indexedSplits ), configs ); long[] inputSplitSizes = getInputSplitSizes( inputFormats, jobConfs, numSplits ); long totalSplitSize = sum( inputSplitSizes ); return collapse( getSplits( inputFormats, jobConfs, indexedSplits ), configs ); return collapse( getSplits( inputFormats, jobConfs, indexedSplits ), configs );
private DataSet<Tuple> translateSource(FlowProcess flowProcess, ExecutionEnvironment env, FlowNode node, int dop) { Tap tap = this.getSingle(node.getSourceTaps()); JobConf tapConfig = new JobConf(this.getNodeConfig(node)); tap.sourceConfInit(flowProcess, tapConfig); tapConfig.set( "cascading.step.source", Tap.id( tap ) ); Fields outFields = tap.getSourceFields(); registerKryoTypes(outFields); JobConf sourceConfig = new JobConf(this.getNodeConfig(node)); MultiInputFormat.addInputFormat(sourceConfig, tapConfig); DataSet<Tuple> src = env .createInput(new TapInputFormat(node), new TupleTypeInfo(outFields)) .name(tap.getIdentifier()) .setParallelism(dop) .withParameters(FlinkConfigConverter.toFlinkConfig(new Configuration(sourceConfig))); return src; }
MultiInputFormat.addInputFormat( conf, streamedJobs ); //must come last
List<Map<String, String>> configs = getConfigs( job ); JobConf[] jobConfs = HadoopUtil.getJobConfs( job, configs ); InputFormat[] inputFormats = getInputFormats( jobConfs ); return collapse( getSplits( inputFormats, jobConfs, new int[]{numSplits} ), configs ); return collapse( getSplits( inputFormats, jobConfs, indexedSplits ), configs ); long[] inputSplitSizes = getInputSplitSizes( inputFormats, jobConfs, numSplits ); long totalSplitSize = sum( inputSplitSizes ); return collapse( getSplits( inputFormats, jobConfs, indexedSplits ), configs ); return collapse( getSplits( inputFormats, jobConfs, indexedSplits ), configs );
MultiInputFormat.addInputFormat( conf, streamedJobs ); //must come last