@Override public void add(Key key) { if (key == null) { throw new NullPointerException("Key can not be null"); } BloomFilter bf = getActiveStandardBF(); if (bf == null) { addRow(); bf = matrix[matrix.length - 1]; currentNbRecord = 0; } bf.add(key); currentNbRecord++; }
public void add(String key) { if (key == null) { throw new NullPointerException("Key cannot by null"); } filter.add(new Key(key.getBytes(StandardCharsets.UTF_8))); }
@Override public void process(S input, Emitter<Pair<String, BloomFilter>> emitter) { Collection<Key> keys = generateKeys(input); if (CollectionUtils.isNotEmpty(keys)) { bloomFilter.add(keys); } }
@Override public void map(Text key, Text value, OutputCollector<NullWritable, BloomFilter> output, Reporter reporter) throws IOException { System.out.println("K[" + key + "]"); int age = Integer.valueOf(value.toString()); if (age > 30) { filter.add(new Key(key.toString().getBytes())); } collector = output; }
@Override protected void reduce(Text key, Iterable<Writable> values, Context context) throws IOException, InterruptedException { f.add(toKey(key)); }
@Override public void add(Key key) { if (key == null) { throw new NullPointerException("Key can not be null"); } BloomFilter bf = getActiveStandardBF(); if (bf == null) { addRow(); bf = matrix[matrix.length - 1]; currentNbRecord = 0; } bf.add(key); currentNbRecord++; }
@Override public void add(Key key) { if (key == null) { throw new NullPointerException("Key can not be null"); } BloomFilter bf = getActiveStandardBF(); if (bf == null) { addRow(); bf = matrix[matrix.length - 1]; currentNbRecord = 0; } bf.add(key); currentNbRecord++; }
@Override public void add(Key key) { if (key == null) { throw new NullPointerException("Key can not be null"); } BloomFilter bf = getActiveStandardBF(); if (bf == null) { addRow(); bf = matrix[matrix.length - 1]; currentNbRecord = 0; } bf.add(key); currentNbRecord++; }
@Override public void add(Key key) { if (key == null) { throw new NullPointerException("Key can not be null"); } BloomFilter bf = getActiveStandardBF(); if (bf == null) { addRow(); bf = matrix[matrix.length - 1]; currentNbRecord = 0; } bf.add(key); currentNbRecord++; }
@Override public void add(Key key) { if (key == null) { throw new NullPointerException("Key can not be null"); } BloomFilter bf = getActiveStandardBF(); if (bf == null) { addRow(); bf = matrix[matrix.length - 1]; currentNbRecord = 0; } bf.add(key); currentNbRecord++; }
@Override public void process(K input, Emitter<BloomFilter> emitter) { bloomFilter.add(new Key(keyToBytesFn.map(input))); }
private void addKeyToBloomFilter(Object key) throws ExecException { Key k = new Key(DataType.toBytes(key, keyType)); if (bloomFilters.length == 1) { if (bloomFilters[0] == null) { bloomFilters[0] = new BloomFilter(vectorSizeBytes * 8, numHash, hashType); } bloomFilters[0].add(k); } else { int partition = (key.hashCode() & Integer.MAX_VALUE) % numBloomFilters; BloomFilter filter = bloomFilters[partition]; if (filter == null) { filter = new BloomFilter(vectorSizeBytes * 8, numHash, hashType); bloomFilters[partition] = filter; } filter.add(k); } }
System.out.println("Reading " + status.getPath()); while ((line = rdr.readLine()) != null) { filter.add(new Key(line.getBytes())); ++numElements;
private void addToBloomFilter(final Object vertex, final BloomFilter filter) throws RetrieverException { try { filter.add(new org.apache.hadoop.util.bloom.Key(elementConverter.serialiseVertex(vertex))); } catch (final AccumuloElementConversionException e) { throw new RetrieverException("Failed to add identifier to the bloom key", e); } }
private Result createBloomFilter() throws IOException { // We get a bag of keys. Create a bloom filter from them // First do distinct of the keys. Not using DistinctBag as memory should not be a problem. HashSet<Object> bloomKeys = new HashSet<>(); Iterator<Tuple> iter = bags[0].iterator(); while (iter.hasNext()) { bloomKeys.add(iter.next().get(0)); } Object partition = key; detachInput(); // Free up the key and bags reference BloomFilter bloomFilter = new BloomFilter(vectorSizeBytes * 8, numHash, hashType); for (Object bloomKey: bloomKeys) { Key k = new Key(DataType.toBytes(bloomKey, bloomKeyType)); bloomFilter.add(k); } bloomKeys = null; return getSerializedBloomFilter(partition, bloomFilter, vectorSizeBytes + 64); }
public DataItem instantiateKaijuItem(byte[] value, Collection<String> allKeys, long timestamp) { BloomFilter bloomFilter = new BloomFilter(Config.getConfig().bloom_filter_num_entries, Config.getConfig().bloom_filter_hf); for(String key : allKeys) { bloomFilter.add(key); } return new DataItem(timestamp, value, bloomFilter); } }
@Override public void compute( Vertex<LongWritable, DoubleWritable, DoubleWritable> vertex, Iterable<LongIdAndValueBloomFilter> messages) throws IOException { BloomFilter filter = new BloomFilter(numBits, numFunctions, hashType); for (Edge<LongWritable, DoubleWritable> e : vertex.getEdges()) { filter.add(new Key(Longs.toByteArray(e.getTargetVertexId().get()))); } sendMessageToAllEdges(vertex, new LongIdAndValueBloomFilter(vertex.getValue(), filter)); } }
@Override public void compute( Vertex<LongWritable, NullWritable, DoubleWritable> vertex, Iterable<LongIdBloomFilter> messages) throws IOException { BloomFilter filter = new BloomFilter(numBits, numFunctions, hashType); for (Edge<LongWritable, DoubleWritable> e : vertex.getEdges()) { filter.add(new Key(Longs.toByteArray(e.getTargetVertexId().get()))); } sendMessageToAllEdges(vertex, new LongIdBloomFilter(vertex.getId(), filter, vertex.getNumEdges())); } }
alreadyIndexedTerms.add(alreadySeen); counters.increment(ContentIndexCounters.BLOOM_FILTER_ADDED, reporter);
@Override public Tuple exec(Tuple input) throws IOException { if (input == null || input.size() == 0) return null; // Strip off the initial level of bag DataBag values = (DataBag)input.get(0); Iterator<Tuple> it = values.iterator(); Tuple t = it.next(); // If the input tuple has only one field, then we'll extract // that field and serialize it into a key. If it has multiple // fields, we'll serialize the whole tuple. byte[] b; if (t.size() == 1) b = DataType.toBytes(t.get(0)); else b = DataType.toBytes(t, DataType.TUPLE); Key k = new Key(b); filter = new BloomFilter(vSize, numHash, hType); filter.add(k); return TupleFactory.getInstance().newTuple(bloomOut()); } }