@Override public Iterable<T> read(Configuration conf) throws IOException { return ((ReadableSource<T>) source).read(conf); }
@Override public Iterable<T> read(Configuration conf) throws IOException { return ((ReadableSource<T>) source).read(conf); }
@Override public Iterable<T> read(Configuration conf) throws IOException { return ((ReadableSource<T>) source).read(conf); } }
@Override public Iterable<T> read(Configuration conf) throws IOException { return ((ReadableSource<T>) source).read(conf); }
@Override public Iterable<T> read(Configuration conf) throws IOException { return ((ReadableSource<T>) source).read(conf); } }
@Override public Iterable<T> read(Configuration conf) throws IOException { return ((ReadableSource<T>) source).read(conf); }
@Override public <T> PCollection<T> read(Source<T> source) { if (source instanceof ReadableSource) { try { Iterable<T> iterable = ((ReadableSource<T>) source).read(conf); return new MemCollection<T>(iterable, source.getType(), source.toString()); } catch (IOException e) { LOG.error("Exception reading source: " + source.toString(), e); throw new IllegalStateException(e); } } LOG.error("Source " + source + " is not readable"); throw new IllegalStateException("Source " + source + " is not readable"); }
@Override public <K, V> PTable<K, V> read(TableSource<K, V> source) { if (source instanceof ReadableSource) { try { Iterable<Pair<K, V>> iterable = ((ReadableSource<Pair<K, V>>) source).read(conf); return new MemTable<K, V>(iterable, source.getTableType(), source.toString()); } catch (IOException e) { LOG.error("Exception reading source: " + source.toString(), e); throw new IllegalStateException(e); } } LOG.error("Source " + source + " is not readable"); throw new IllegalStateException("Source " + source + " is not readable"); }
@Override public <T> PCollection<T> read(Source<T> source) { if (source instanceof ReadableSource) { try { Iterable<T> iterable = ((ReadableSource<T>) source).read(conf); return new MemCollection<T>(iterable, source.getType(), source.toString()); } catch (IOException e) { LOG.error("Exception reading source: " + source.toString(), e); throw new IllegalStateException(e); } } LOG.error("Source " + source + " is not readable"); throw new IllegalStateException("Source " + source + " is not readable"); }
@Override public <K, V> PTable<K, V> read(TableSource<K, V> source) { if (source instanceof ReadableSource) { try { Iterable<Pair<K, V>> iterable = ((ReadableSource<Pair<K, V>>) source).read(conf); return new MemTable<K, V>(iterable, source.getTableType(), source.toString()); } catch (IOException e) { LOG.error("Exception reading source: " + source.toString(), e); throw new IllegalStateException(e); } } LOG.error("Source " + source + " is not readable"); throw new IllegalStateException("Source " + source + " is not readable"); }
public void materialize() { try { materialized = source.read(pipeline.getConfiguration()); } catch (IOException e) { LOG.error("Could not materialize: {}", source, e); throw new CrunchRuntimeException(e); } } }
@Override public <T> PCollection<T> read(Source<T> source, String named) { String name = named == null ? source.toString() : named; if (source instanceof ReadableSource) { try { Iterable<T> iterable = ((ReadableSource<T>) source).read(conf); return new MemCollection<T>(iterable, source.getType(), name); } catch (IOException e) { LOG.error("Exception reading source: " + name, e); throw new IllegalStateException(e); } } LOG.error("Source {} is not readable", name); throw new IllegalStateException("Source " + name + " is not readable"); }
@Override public <K, V> PTable<K, V> read(TableSource<K, V> source, String named) { String name = named == null ? source.toString() : named; if (source instanceof ReadableSource) { try { Iterable<Pair<K, V>> iterable = ((ReadableSource<Pair<K, V>>) source).read(conf); return new MemTable<K, V>(iterable, source.getTableType(), name); } catch (IOException e) { LOG.error("Exception reading source: " + name, e); throw new IllegalStateException(e); } } LOG.error("Source {} is not readable", name); throw new IllegalStateException("Source " + name + " is not readable"); }
public void materialize() { try { materialized = source.read(pipeline.getConfiguration()); } catch (IOException e) { LOG.error("Could not materialize: " + source, e); throw new CrunchRuntimeException(e); } } }
@Override public Iterable<S> materialize() { if (!waitingOnTargets() && getSize() == 0) { System.err.println("Materializing an empty PCollection: " + this.getName()); return Collections.emptyList(); } if (materializedAt != null && (materializedAt instanceof ReadableSource)) { try { return ((ReadableSource<S>) materializedAt).read(getPipeline().getConfiguration()); } catch (IOException e) { throw new CrunchRuntimeException("Error reading materialized data", e); } } materialized = true; return pipeline.materialize(this); }