@Override public void configureSource(Job job, int inputId) throws IOException { source.configureSource(job, inputId); }
@Override public PType<S> getPType() { return source.getType(); }
@Override public Converter<?, ?, ?, ?> getConverter() { return source.getConverter(); }
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) { try { Job job = new Job(runtime.getConfiguration()); FileInputFormat.addInputPaths(job, "/tmp"); //placeholder source.configureSource(job, 0); Converter converter = source.getConverter(); JavaPairRDD<?, ?> input = runtime.getSparkContext().newAPIHadoopRDD( job.getConfiguration(), CrunchInputFormat.class, converter.getKeyClass(), converter.getValueClass()); input.rdd().setName(getName()); MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance(); return input .map(new InputConverterFunction(source.getConverter())) .map(new MapFunction(mapFn, runtime.getRuntimeContext())); } catch (IOException e) { throw new RuntimeException(e); } } }
public static <S> DoNode createInputNode(Source<S> source) { Converter srcConverter = source.getConverter(); PType<?> ptype = source.getType(); DoFn<?, ?> fn = srcConverter.applyPTypeTransforms() ? ptype.getInputMapFn() : IdentityFn.getInstance(); return new DoNode(fn, source.toString(), ptype, allowsChildren(), null, source, null); }
@Override public long getSize(Configuration configuration) { return source.getSize(configuration); }
@Override public Source<T> inputConf(String key, String value) { source.inputConf(key, value); return this; }
@Override public long getLastModifiedAt(Configuration configuration) { return source.getLastModifiedAt(configuration); }
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) { try { Job job = new Job(runtime.getConfiguration()); FileInputFormat.addInputPaths(job, "/tmp"); //placeholder source.configureSource(job, 0); Converter converter = source.getConverter(); JavaPairRDD<?, ?> input = runtime.getSparkContext().newAPIHadoopRDD( job.getConfiguration(), CrunchInputFormat.class, converter.getKeyClass(), converter.getValueClass()); input.rdd().setName(getName()); MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance(); return input .map(new InputConverterFunction(source.getConverter())) .map(new MapFunction(mapFn, runtime.getRuntimeContext())); } catch (IOException e) { throw new RuntimeException(e); } } }
@Override public long getSize(Configuration configuration) { return source.getSize(configuration); }
@Override public SourceTarget<T> conf(String key, String value) { source.inputConf(key, value); target.outputConf(key, value); return this; } }
@Override public long getLastModifiedAt() { return source.getLastModifiedAt(pipeline.getConfiguration()); }
@Override public PType<T> getType() { return source.getType(); }
@Override public void configureSource(Job job, int inputId) throws IOException { source.configureSource(job, inputId); }
@Override public long getSize(Configuration configuration) { return source.getSize(configuration); }
public RTNode toRTNode(boolean inputNode, Configuration conf, NodeContext nodeContext) { List<RTNode> childRTNodes = Lists.newArrayList(); if (options != null) { options.configure(conf); } fn.configure(conf); for (DoNode child : children) { childRTNodes.add(child.toRTNode(false, conf, nodeContext)); } Converter inputConverter = null; if (inputNode) { if (nodeContext == NodeContext.MAP) { inputConverter = source.getConverter(); } else { inputConverter = ((PGroupedTableType<?, ?>) ptype).getGroupingConverter(); } } return new RTNode(fn, (PType<Object>) getPType(), name, childRTNodes, inputConverter, outputConverter, outputName); }
@Override public PType<S> getPType() { return source.getType(); }
@Override public void configureSource(Job job, int inputId) throws IOException { source.configureSource(job, inputId); }
@Override protected long getSizeInternal() { long sz = source.getSize(pipeline.getConfiguration()); if (sz < 0) { throw new IllegalStateException("Input source " + source + " does not exist!"); } return sz; }
@Override public PType<T> getType() { return source.getType(); }