.asSubclass(Writable.class), conf); try (MapFile.Writer writer = new MapFile.Writer(conf, fs, out, reader.getKeyClass().asSubclass(WritableComparable.class), reader.getValueClass())) {
@SuppressWarnings("unchecked") private void open(Path[] inMapFiles, Path outMapFile) throws IOException { inReaders = new Reader[inMapFiles.length]; for (int i = 0; i < inMapFiles.length; i++) { Reader reader = new Reader(inMapFiles[i], conf); if (keyClass == null || valueClass == null) { keyClass = (Class<WritableComparable>) reader.getKeyClass(); valueClass = (Class<Writable>) reader.getValueClass(); } else if (keyClass != reader.getKeyClass() || valueClass != reader.getValueClass()) { throw new HadoopIllegalArgumentException( "Input files cannot be merged as they" + " have different Key and Value classes"); } inReaders[i] = reader; } if (comparator == null) { Class<? extends WritableComparable> cls; cls = keyClass.asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else if (comparator.getKeyClass() != keyClass) { throw new HadoopIllegalArgumentException( "Input files cannot be merged as they" + " have different Key class compared to" + " specified comparator"); } outWriter = new MapFile.Writer(conf, outMapFile, MapFile.Writer.keyClass(keyClass), MapFile.Writer.valueClass(valueClass)); }
private void writeBuffer(String destPath, List<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException { Path path = new Path(destPath, String.format("part-r-%05d", part)); FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); try (MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns.makeQualified(path), MapFile.Writer.keyClass(LogFileKey.class), MapFile.Writer.valueClass(LogFileValue.class))) { Collections.sort(buffer, Comparator.comparing(Pair::getFirst)); for (Pair<LogFileKey,LogFileValue> entry : buffer) { output.append(entry.getFirst(), entry.getSecond()); } } }
public static void main(String[] args) throws Exception { String usage = "Usage: MapFile inFile outFile"; if (args.length != 2) { System.err.println(usage); System.exit(-1); } String in = args[0]; String out = args[1]; Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); MapFile.Reader reader = new MapFile.Reader(fs, in, conf); MapFile.Writer writer = new MapFile.Writer(conf, fs, out, reader.getKeyClass().asSubclass(WritableComparable.class), reader.getValueClass()); WritableComparable key = ReflectionUtils.newInstance(reader.getKeyClass().asSubclass(WritableComparable.class), conf); Writable value = ReflectionUtils.newInstance(reader.getValueClass().asSubclass(Writable.class), conf); while (reader.next(key, value)) // copy all entries writer.append(key, value); writer.close(); }
@Test @SuppressWarnings("deprecation") public void testMidKey() throws Exception { // Write a mapfile of simple data: keys are Path dirName = new Path(TEST_DIR, "testMidKey.mapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); MapFile.Writer writer = null; MapFile.Reader reader = null; try { writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, IntWritable.class); writer.append(new IntWritable(1), new IntWritable(1)); writer.close(); // Now do getClosest on created mapfile. reader = new MapFile.Reader(qualifiedDirName, conf); assertEquals(new IntWritable(1), reader.midKey()); } finally { IOUtils.cleanup(null, writer, reader); } }
private void writeBuffer(String destPath, List<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException { Path path = new Path(destPath, String.format("part-r-%05d", part)); FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns.makeQualified(path), MapFile.Writer.keyClass(LogFileKey.class), MapFile.Writer.valueClass(LogFileValue.class)); try { Collections.sort(buffer, new Comparator<Pair<LogFileKey,LogFileValue>>() { @Override public int compare(Pair<LogFileKey,LogFileValue> o1, Pair<LogFileKey,LogFileValue> o2) { return o1.getFirst().compareTo(o2.getFirst()); } }); for (Pair<LogFileKey,LogFileValue> entry : buffer) { output.append(entry.getFirst(), entry.getSecond()); } } finally { output.close(); } }
@Override @SuppressWarnings("rawtypes") public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) { try { Closeable rout; HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath); Class<? extends WritableComparable> keyWritableClass = configuration.getKeyType().getWritableClass(); Class<? extends WritableComparable> valueWritableClass = configuration.getValueType().getWritableClass(); rout = new MapFile.Writer(hdfsInfo.getConf(), new Path(hdfsPath), MapFile.Writer.keyClass(keyWritableClass), MapFile.Writer.valueClass(valueWritableClass), MapFile.Writer.compression(configuration.getCompressionType(), configuration.getCompressionCodec().getCodec()), MapFile.Writer.progressable(new Progressable() { @Override public void progress() { } })); return rout; } catch (IOException ex) { throw new RuntimeCamelException(ex); } }
/** * test {@code MapFile.Writer} constructor * with IllegalArgumentException * */ @Test public void testKeyLessWriterCreation() { MapFile.Writer writer = null; try { writer = new MapFile.Writer(conf, TEST_DIR); fail("fail in testKeyLessWriterCreation !!!"); } catch (IllegalArgumentException ex) { } catch (Exception e) { fail("fail in testKeyLessWriterCreation. Other ex !!!"); } finally { IOUtils.cleanup(null, writer); } } /**
@Test @SuppressWarnings("deprecation") public void testMidKeyEmpty() throws Exception { // Write a mapfile of simple data: keys are Path dirName = new Path(TEST_DIR, "testMidKeyEmpty.mapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); MapFile.Writer writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, IntWritable.class); writer.close(); // Now do getClosest on created mapfile. MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf); try { assertEquals(null, reader.midKey()); } finally { reader.close(); } }
/** * test {@code MapFile.Writer} constructor * with IllegalArgumentException * */ @Test public void testKeyLessWriterCreation() { MapFile.Writer writer = null; try { writer = new MapFile.Writer(conf, TEST_DIR); fail("fail in testKeyLessWriterCreation !!!"); } catch (IllegalArgumentException ex) { } catch (Exception e) { fail("fail in testKeyLessWriterCreation. Other ex !!!"); } finally { IOUtils.cleanup(null, writer); } } /**
@Test @SuppressWarnings("deprecation") public void testMidKeyEmpty() throws Exception { // Write a mapfile of simple data: keys are Path dirName = new Path(TEST_DIR, "testMidKeyEmpty.mapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); MapFile.Writer writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, IntWritable.class); writer.close(); // Now do getClosest on created mapfile. MapFile.Reader reader = new MapFile.Reader(qualifiedDirName, conf); try { assertEquals(null, reader.midKey()); } finally { reader.close(); } }
private void writeBuffer(String destPath, ArrayList<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException { String path = destPath + String.format("/part-r-%05d", part++); MapFile.Writer output = new MapFile.Writer(fs.getConf(), fs, path, LogFileKey.class, LogFileValue.class); try { Collections.sort(buffer, new Comparator<Pair<LogFileKey,LogFileValue>>() { @Override public int compare(Pair<LogFileKey,LogFileValue> o1, Pair<LogFileKey,LogFileValue> o2) { return o1.getFirst().compareTo(o2.getFirst()); } }); for (Pair<LogFileKey,LogFileValue> entry : buffer) { output.append(entry.getFirst(), entry.getSecond()); } } finally { output.close(); } }
public void testMidKeyEmpty() throws Exception { // Write a mapfile of simple data: keys are Path dirName = new Path(System.getProperty("test.build.data",".") + getName() + ".mapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); MapFile.Writer writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, IntWritable.class); writer.close(); // Now do getClosest on created mapfile. MapFile.Reader reader = new MapFile.Reader(fs, qualifiedDirName.toString(), conf); assertEquals(null, reader.midKey()); } }
public void testMidKey() throws Exception { // Write a mapfile of simple data: keys are Path dirName = new Path(System.getProperty("test.build.data",".") + getName() + ".mapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); MapFile.Writer writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, IntWritable.class); writer.append(new IntWritable(1), new IntWritable(1)); writer.close(); // Now do getClosest on created mapfile. MapFile.Reader reader = new MapFile.Reader(fs, qualifiedDirName.toString(), conf); assertEquals(new IntWritable(1), reader.midKey()); }
private static void createMapFile(Configuration conf, FileSystem fs, Path path, CompressionCodec codec, CompressionType type, int records) throws IOException { MapFile.Writer writer = new MapFile.Writer(conf, path, MapFile.Writer.keyClass(Text.class), MapFile.Writer.valueClass(Text.class), MapFile.Writer.compression(type, codec)); Text key = new Text(); for (int j = 0; j < records; j++) { key.set(String.format("%03d", j)); writer.append(key, key); } writer.close(); }
private static void createMapFile(Configuration conf, FileSystem fs, Path path, CompressionCodec codec, CompressionType type, int records) throws IOException { MapFile.Writer writer = new MapFile.Writer(conf, path, MapFile.Writer.keyClass(Text.class), MapFile.Writer.valueClass(Text.class), MapFile.Writer.compression(type, codec)); Text key = new Text(); for (int j = 0; j < records; j++) { key.set(String.format("%03d", j)); writer.append(key, key); } writer.close(); }
private MapFile.Writer createWriter(String fileName, Class<? extends WritableComparable<?>> keyClass, Class<? extends Writable> valueClass) throws IOException { Path dirName = new Path(TEST_DIR, fileName); MapFile.Writer.setIndexInterval(conf, 4); return new MapFile.Writer(conf, dirName, MapFile.Writer.keyClass(keyClass), MapFile.Writer.valueClass(valueClass)); }
private MapFile.Writer createWriter(String fileName, Class<? extends WritableComparable<?>> keyClass, Class<? extends Writable> valueClass) throws IOException { Path dirName = new Path(TEST_DIR, fileName); MapFile.Writer.setIndexInterval(conf, 4); return new MapFile.Writer(conf, dirName, MapFile.Writer.keyClass(keyClass), MapFile.Writer.valueClass(valueClass)); }
private void prepareWriter() throws IOException { if (reader != null) { reader.close(); reader = null; } if (writer == null) { writer = new MapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, Text.class); } }
new MapFile.Writer(job, fs, text.toString(), Text.class, ParseText.class, CompressionType.RECORD, progress); new MapFile.Writer(job, fs, data.toString(), Text.class, ParseData.class, compType, progress); new MapFile.Writer(job, fs, linkdb.toString(), Text.class, Inlinks.class, compType, progress);