@Override public void setConfiguration(Configuration conf) { keys.setConfiguration(conf); values.setConfiguration(conf); }
@Override public void setConfiguration(Configuration conf) { this.first.setConfiguration(conf); this.second.setConfiguration(conf); }
@Override public void setConfiguration(Configuration conf) { mapFn.setConfiguration(conf); }
@Override public void initialize(Configuration conf) { this.inputFn.setConfiguration(conf); this.outputFn.setConfiguration(conf); this.inputFn.initialize(); this.outputFn.initialize(); for (PType subType : subTypes) { subType.initialize(conf); } this.initialized = true; }
@Override public void initialize(Configuration conf) { derivedToAvroFn.setConfiguration(conf); derivedToAvroFn.initialize(); avroToDerivedFn.setConfiguration(conf); avroToDerivedFn.initialize(); avroBaseType.initialize(conf); }
@Override public void initialize(Configuration conf) { baseInputMapFn.setConfiguration(conf); baseInputMapFn.initialize(); baseOutputMapFn.setConfiguration(conf); baseOutputMapFn.initialize(); deepCopier.initialize(conf); for (PType ptype : subTypes) { ptype.initialize(conf); } initialized = true; }
throws IOException { FileSystem fs = FileSystem.get(conf); outputFn.setConfiguration(conf); outputFn.initialize(); if (Text.class.equals(writableClass) && parallelism > 1) {
throws IOException { FileSystem fs = FileSystem.get(conf); baseOutputMapFn.setConfiguration(conf); baseOutputMapFn.initialize(); fs.mkdirs(path);
@Override public ReadableSource<Pair<K, V>> createSourceTarget( Configuration conf, Path path, Iterable<Pair<K, V>> contents, int parallelism) throws IOException { FileSystem fs = FileSystem.get(conf); outputFn.setConfiguration(conf); outputFn.initialize(); fs.mkdirs(path); List<SequenceFile.Writer> writers = Lists.newArrayListWithExpectedSize(parallelism); for (int i = 0; i < parallelism; i++) { Path out = new Path(path, "out" + i); writers.add(SequenceFile.createWriter(fs, conf, out, keyType.getSerializationClass(), valueType.getSerializationClass())); } int target = 0; for (Pair<K, V> value : contents) { Pair writablePair = (Pair) outputFn.map(value); writers.get(target).append(writablePair.first(), writablePair.second()); target = (target + 1) % parallelism; } for (SequenceFile.Writer writer : writers) { writer.close(); } ReadableSource<Pair<K, V>> ret = new SeqFileTableSource<K, V>(path, this); ret.inputConf(RuntimeParameters.DISABLE_COMBINE_FILE, "true"); return ret; }