/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress, Progressable progress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress), progressable(progress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress, CompressionCodec codec, Progressable progress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress, codec), progressable(progress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress, Progressable progress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress), progressable(progress)); }
private MapFile.Writer createWriter(String fileName, Class<? extends WritableComparable<?>> keyClass, Class<? extends Writable> valueClass) throws IOException { Path dirName = new Path(TEST_DIR, fileName); MapFile.Writer.setIndexInterval(conf, 4); return new MapFile.Writer(conf, dirName, MapFile.Writer.keyClass(keyClass), MapFile.Writer.valueClass(valueClass)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress, Progressable progress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress), progressable(progress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress, CompressionCodec codec, Progressable progress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress, codec), progressable(progress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass)); }
private MapFile.Writer createWriter(String fileName, Class<? extends WritableComparable<?>> keyClass, Class<? extends Writable> valueClass) throws IOException { Path dirName = new Path(TEST_DIR, fileName); MapFile.Writer.setIndexInterval(conf, 4); return new MapFile.Writer(conf, dirName, MapFile.Writer.keyClass(keyClass), MapFile.Writer.valueClass(valueClass)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress, Progressable progress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress), progressable(progress)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress)); }
private void writeBuffer(String destPath, List<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException { Path path = new Path(destPath, String.format("part-r-%05d", part)); FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); try (MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns.makeQualified(path), MapFile.Writer.keyClass(LogFileKey.class), MapFile.Writer.valueClass(LogFileValue.class))) { Collections.sort(buffer, Comparator.comparing(Pair::getFirst)); for (Pair<LogFileKey,LogFileValue> entry : buffer) { output.append(entry.getFirst(), entry.getSecond()); } } }
@SuppressWarnings("unchecked") private void open(Path[] inMapFiles, Path outMapFile) throws IOException { inReaders = new Reader[inMapFiles.length]; for (int i = 0; i < inMapFiles.length; i++) { Reader reader = new Reader(inMapFiles[i], conf); if (keyClass == null || valueClass == null) { keyClass = (Class<WritableComparable>) reader.getKeyClass(); valueClass = (Class<Writable>) reader.getValueClass(); } else if (keyClass != reader.getKeyClass() || valueClass != reader.getValueClass()) { throw new HadoopIllegalArgumentException( "Input files cannot be merged as they" + " have different Key and Value classes"); } inReaders[i] = reader; } if (comparator == null) { Class<? extends WritableComparable> cls; cls = keyClass.asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else if (comparator.getKeyClass() != keyClass) { throw new HadoopIllegalArgumentException( "Input files cannot be merged as they" + " have different Key class compared to" + " specified comparator"); } outWriter = new MapFile.Writer(conf, outMapFile, MapFile.Writer.keyClass(keyClass), MapFile.Writer.valueClass(valueClass)); }
/** Create the named map for keys of the named class. * @deprecated Use Writer(Configuration, Path, Option...) instead. */ @Deprecated public Writer(Configuration conf, FileSystem fs, String dirName, Class<? extends WritableComparable> keyClass, Class valClass, CompressionType compress, CompressionCodec codec, Progressable progress) throws IOException { this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass), compression(compress, codec), progressable(progress)); }