@Override public void cleanup(Emitter<T> emitter) { intern.cleanup(emitter); } }
@Override public void cleanup(Emitter<T> emitter) { intern.cleanup(emitter); } }
public void cleanup() { fn.cleanup(emitter); emitter.flush(); for (RTNode child : children) { child.cleanup(); } }
public void cleanup() { fn.cleanup(emitter); emitter.flush(); for (RTNode child : children) { child.cleanup(); } }
public void cleanup() { fn.cleanup(emitter); emitter.flush(); for (RTNode child : children) { child.cleanup(); } }
@Override public boolean hasNext() { while (cache.isEmpty() && iter.hasNext()) { fn.process(iter.next(), cache); } if (cache.isEmpty() && !cleanup) { fn.cleanup(cache); cleanup = true; } return !cache.isEmpty(); }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
@Override public <T> PCollection<T> parallelDo(String name, DoFn<S, T> doFn, PType<T> type) { InMemoryEmitter<T> emitter = new InMemoryEmitter<T>(); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemCollection<T>(emitter.getOutput(), type, name); }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type, ParallelDoOptions options) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); doFn.setContext(getInMemoryContext(getPipeline().getConfiguration())); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
@Override public <T> PCollection<T> parallelDo(String name, DoFn<S, T> doFn, PType<T> type, ParallelDoOptions options) { InMemoryEmitter<T> emitter = new InMemoryEmitter<T>(); doFn.setContext(getInMemoryContext(getPipeline().getConfiguration())); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemCollection<T>(emitter.getOutput(), type, name); }
@Override public <K, V> PTable<K, V> parallelDo(String name, DoFn<S, Pair<K, V>> doFn, PTableType<K, V> type, ParallelDoOptions options) { InMemoryEmitter<Pair<K, V>> emitter = new InMemoryEmitter<Pair<K, V>>(); Configuration conf = getPipeline().getConfiguration(); doFn.configure(conf); doFn.setContext(getInMemoryContext(conf)); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemTable<K, V>(emitter.getOutput(), type, name); }
@Override public <T> PCollection<T> parallelDo(String name, DoFn<S, T> doFn, PType<T> type, ParallelDoOptions options) { doFn = verifySerializable(name, doFn); InMemoryEmitter<T> emitter = new InMemoryEmitter<T>(); Configuration conf = getPipeline().getConfiguration(); doFn.configure(conf); doFn.setContext(getInMemoryContext(conf)); doFn.initialize(); for (S s : collect) { doFn.process(s, emitter); } doFn.cleanup(emitter); return new MemCollection<T>(emitter.getOutput(), type, name); }