reader.getValueClass())) { while (reader.next(key, value)) { // copy all entries writer.append(key, value);
/** Append a value to the file. */ public synchronized void append(Writable value) throws IOException { super.append(count, value); // add to map count.set(count.get()+1); // increment count } }
outWriter.append(currentKey, currentValue);
public void write(WritableComparable<?> key, Writable value) throws IOException { out.append(key, value); }
/** Append a value to the file. */ public synchronized void append(Writable value) throws IOException { super.append(count, value); // add to map count.set(count.get()+1); // increment count } }
public void write(WritableComparable<?> key, Writable value) throws IOException { out.append(key, value); }
/** Append a value to the file. */ public synchronized void append(Writable value) throws IOException { super.append(count, value); // add to map count.set(count.get()+1); // increment count } }
@Override public synchronized void append(WritableComparable key, Writable val) throws IOException { super.append(key, val); buf.reset(); key.write(buf); bloomKey.set(buf.getData(), 1.0); bloomFilter.add(bloomKey); }
public V put(K key, V value) { try { prepareWriter(); Text k = new Text(SerializationUtil.serialize((Serializable) key)); Text v = new Text(SerializationUtil.serialize((Serializable) value)); writer.append(k, v); return null; } catch (Exception e) { throw new RuntimeException("could not store value: " + key + ", " + value, e); } }
@Override public synchronized void append(WritableComparable key, Writable val) throws IOException { super.append(key, val); buf.reset(); key.write(buf); bloomKey.set(byteArrayForBloomKey(buf), 1.0); bloomFilter.add(bloomKey); }
/** Append a value to the file. */ public synchronized void append(Writable value) throws IOException { super.append(count, value); // add to map count.set(count.get()+1); // increment count } }
/** Append a value to the file. */ public synchronized void append(Writable value) throws IOException { super.append(count, value); // add to map count.set(count.get()+1); // increment count } }
/** Append a value to the file. */ public synchronized void append(Writable value) throws IOException { super.append(count, value); // add to map count.set(count.get()+1); // increment count } }
@Override public synchronized void append(WritableComparable key, Writable val) throws IOException { super.append(key, val); buf.reset(); key.write(buf); bloomKey.set(byteArrayForBloomKey(buf), 1.0); bloomFilter.add(bloomKey); }
/** Append a value to the file. */ public synchronized void append(Writable value) throws IOException { super.append(count, value); // add to map count.set(count.get()+1); // increment count } }
@Override public synchronized void append(WritableComparable key, Writable val) throws IOException { super.append(key, val); buf.reset(); key.write(buf); bloomKey.set(byteArrayForBloomKey(buf), 1.0); bloomFilter.add(bloomKey); }
@Override public synchronized void append(WritableComparable key, Writable val) throws IOException { super.append(key, val); buf.reset(); key.write(buf); bloomKey.set(byteArrayForBloomKey(buf), 1.0); bloomFilter.add(bloomKey); }
private void writeBuffer(String destPath, List<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException { Path path = new Path(destPath, String.format("part-r-%05d", part)); FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); try (MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns.makeQualified(path), MapFile.Writer.keyClass(LogFileKey.class), MapFile.Writer.valueClass(LogFileValue.class))) { Collections.sort(buffer, Comparator.comparing(Pair::getFirst)); for (Pair<LogFileKey,LogFileValue> entry : buffer) { output.append(entry.getFirst(), entry.getSecond()); } } }
@Override public synchronized void append(WritableComparable key, Writable val) throws IOException { super.append(key, val); buf.reset(); key.write(buf); bloomKey.set(byteArrayForBloomKey(buf), 1.0); bloomFilter.add(bloomKey); }