@Override public Converter<?, ?, ?, ?> getConverter() { return source.getConverter(); }
public static <S> DoNode createInputNode(Source<S> source) { Converter srcConverter = source.getConverter(); PType<?> ptype = source.getType(); DoFn<?, ?> fn = srcConverter.applyPTypeTransforms() ? ptype.getInputMapFn() : IdentityFn.getInstance(); return new DoNode(fn, source.toString(), ptype, allowsChildren(), null, source, null); }
public RTNode toRTNode(boolean inputNode, Configuration conf, NodeContext nodeContext) { List<RTNode> childRTNodes = Lists.newArrayList(); if (options != null) { options.configure(conf); } fn.configure(conf); for (DoNode child : children) { childRTNodes.add(child.toRTNode(false, conf, nodeContext)); } Converter inputConverter = null; if (inputNode) { if (nodeContext == NodeContext.MAP) { inputConverter = source.getConverter(); } else { inputConverter = ((PGroupedTableType<?, ?>) ptype).getGroupingConverter(); } } return new RTNode(fn, (PType<Object>) getPType(), name, childRTNodes, inputConverter, outputConverter, outputName); }
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) { try { Job job = new Job(runtime.getConfiguration()); FileInputFormat.addInputPaths(job, "/tmp"); //placeholder source.configureSource(job, 0); Converter converter = source.getConverter(); JavaPairRDD<?, ?> input = runtime.getSparkContext().newAPIHadoopRDD( job.getConfiguration(), CrunchInputFormat.class, converter.getKeyClass(), converter.getValueClass()); input.rdd().setName(getName()); MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance(); return input .map(new InputConverterFunction(source.getConverter())) .map(new MapFunction(mapFn, runtime.getRuntimeContext())); } catch (IOException e) { throw new RuntimeException(e); } } }
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) { try { Job job = new Job(runtime.getConfiguration()); FileInputFormat.addInputPaths(job, "/tmp"); //placeholder source.configureSource(job, 0); Converter converter = source.getConverter(); JavaPairRDD<?, ?> input = runtime.getSparkContext().newAPIHadoopRDD( job.getConfiguration(), CrunchInputFormat.class, converter.getKeyClass(), converter.getValueClass()); input.rdd().setName(getName()); MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance(); return input .map(new InputConverterFunction(source.getConverter())) .map(new MapFunction(mapFn, runtime.getRuntimeContext())); } catch (IOException e) { throw new RuntimeException(e); } } }