@Override public PType<S> getPType() { return source.getType(); }
@Override public PType<T> getType() { return source.getType(); }
@Override public PType<S> getPType() { return source.getType(); }
@Override public PType<S> getPType() { return source.getType(); }
@Override public PType<T> getType() { return source.getType(); }
@Override public PType<T> getType() { return source.getType(); }
@Override public <S> SourceTarget<S> asSourceTarget(PType<S> ptype) { if (ptype != null && ptype.equals(source.getType())) { return (SourceTarget<S>) this; } return target.asSourceTarget(ptype); }
@Override public <T> PCollection<T> read(Source<T> source) { if (source instanceof ReadableSource) { try { Iterable<T> iterable = ((ReadableSource<T>) source).read(conf); return new MemCollection<T>(iterable, source.getType(), source.toString()); } catch (IOException e) { LOG.error("Exception reading source: " + source.toString(), e); throw new IllegalStateException(e); } } LOG.error("Source " + source + " is not readable"); throw new IllegalStateException("Source " + source + " is not readable"); }
@Override public <T> PCollection<T> read(Source<T> source) { if (source instanceof ReadableSource) { try { Iterable<T> iterable = ((ReadableSource<T>) source).read(conf); return new MemCollection<T>(iterable, source.getType(), source.toString()); } catch (IOException e) { LOG.error("Exception reading source: " + source.toString(), e); throw new IllegalStateException(e); } } LOG.error("Source " + source + " is not readable"); throw new IllegalStateException("Source " + source + " is not readable"); }
@Override public <T> PCollection<T> read(Source<T> source, String named) { String name = named == null ? source.toString() : named; if (source instanceof ReadableSource) { try { Iterable<T> iterable = ((ReadableSource<T>) source).read(conf); return new MemCollection<T>(iterable, source.getType(), name); } catch (IOException e) { LOG.error("Exception reading source: " + name, e); throw new IllegalStateException(e); } } LOG.error("Source {} is not readable", name); throw new IllegalStateException("Source " + name + " is not readable"); }
public static <S> DoNode createInputNode(Source<S> source) { PType<?> ptype = source.getType(); DoFn<?, ?> fn = ptype.getInputMapFn(); return new DoNode(fn, source.toString(), ptype, allowsChildren(), null, source); }
public static <S> DoNode createInputNode(Source<S> source) { PType<?> ptype = source.getType(); DoFn<?,?> fn = ptype.getInputMapFn(); return new DoNode(fn, source.toString(), ptype, allowsChildren(), null, source); }
public static <S> DoNode createInputNode(Source<S> source) { Converter srcConverter = source.getConverter(); PType<?> ptype = source.getType(); DoFn<?, ?> fn = srcConverter.applyPTypeTransforms() ? ptype.getInputMapFn() : IdentityFn.getInstance(); return new DoNode(fn, source.toString(), ptype, allowsChildren(), null, source, null); }
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) { try { Job job = new Job(runtime.getConfiguration()); FileInputFormat.addInputPaths(job, "/tmp"); //placeholder source.configureSource(job, 0); Converter converter = source.getConverter(); JavaPairRDD<?, ?> input = runtime.getSparkContext().newAPIHadoopRDD( job.getConfiguration(), CrunchInputFormat.class, converter.getKeyClass(), converter.getValueClass()); input.rdd().setName(getName()); MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance(); return input .map(new InputConverterFunction(source.getConverter())) .map(new MapFunction(mapFn, runtime.getRuntimeContext())); } catch (IOException e) { throw new RuntimeException(e); } } }
public JavaRDDLike<?, ?> getJavaRDDLike(SparkRuntime runtime) { try { Job job = new Job(runtime.getConfiguration()); FileInputFormat.addInputPaths(job, "/tmp"); //placeholder source.configureSource(job, 0); Converter converter = source.getConverter(); JavaPairRDD<?, ?> input = runtime.getSparkContext().newAPIHadoopRDD( job.getConfiguration(), CrunchInputFormat.class, converter.getKeyClass(), converter.getValueClass()); input.rdd().setName(getName()); MapFn mapFn = converter.applyPTypeTransforms() ? source.getType().getInputMapFn() : IdentityFn.getInstance(); return input .map(new InputConverterFunction(source.getConverter())) .map(new MapFunction(mapFn, runtime.getRuntimeContext())); } catch (IOException e) { throw new RuntimeException(e); } } }