@Override public void or(Filter filter) { if (filter == null || !(filter instanceof DynamicBloomFilter) || filter.vectorSize != this.vectorSize || filter.nbHash != this.nbHash) { throw new IllegalArgumentException("filters cannot be or-ed"); } DynamicBloomFilter dbf = (DynamicBloomFilter)filter; if (dbf.matrix.length != this.matrix.length || dbf.nr != this.nr) { throw new IllegalArgumentException("filters cannot be or-ed"); } for (int i = 0; i < matrix.length; i++) { matrix[i].or(dbf.matrix[i]); } }
@Override public void update(BloomFilter value) { bloomFilter.or(value); }
@Override public void reduce(NullWritable key, Iterator<BloomFilter> values, OutputCollector<AvroWrapper<GenericRecord>, NullWritable> output, Reporter reporter) throws IOException { while (values.hasNext()) { BloomFilter bf = values.next(); filter.or(bf); System.out.println(filter); } collector = output; }
@Override public void or(Filter filter) { if (filter == null || !(filter instanceof DynamicBloomFilter) || filter.vectorSize != this.vectorSize || filter.nbHash != this.nbHash) { throw new IllegalArgumentException("filters cannot be or-ed"); } DynamicBloomFilter dbf = (DynamicBloomFilter)filter; if (dbf.matrix.length != this.matrix.length || dbf.nr != this.nr) { throw new IllegalArgumentException("filters cannot be or-ed"); } for (int i = 0; i < matrix.length; i++) { matrix[i].or(dbf.matrix[i]); } }
@Override public void or(Filter filter) { if (filter == null || !(filter instanceof DynamicBloomFilter) || filter.vectorSize != this.vectorSize || filter.nbHash != this.nbHash) { throw new IllegalArgumentException("filters cannot be or-ed"); } DynamicBloomFilter dbf = (DynamicBloomFilter)filter; if (dbf.matrix.length != this.matrix.length || dbf.nr != this.nr) { throw new IllegalArgumentException("filters cannot be or-ed"); } for (int i = 0; i < matrix.length; i++) { matrix[i].or(dbf.matrix[i]); } }
@Override public void or(Filter filter) { if (filter == null || !(filter instanceof DynamicBloomFilter) || filter.vectorSize != this.vectorSize || filter.nbHash != this.nbHash) { throw new IllegalArgumentException("filters cannot be or-ed"); } DynamicBloomFilter dbf = (DynamicBloomFilter)filter; if (dbf.matrix.length != this.matrix.length || dbf.nr != this.nr) { throw new IllegalArgumentException("filters cannot be or-ed"); } for (int i = 0; i < matrix.length; i++) { matrix[i].or(dbf.matrix[i]); } }
@Override public void or(Filter filter) { if (filter == null || !(filter instanceof DynamicBloomFilter) || filter.vectorSize != this.vectorSize || filter.nbHash != this.nbHash) { throw new IllegalArgumentException("filters cannot be or-ed"); } DynamicBloomFilter dbf = (DynamicBloomFilter)filter; if (dbf.matrix.length != this.matrix.length || dbf.nr != this.nr) { throw new IllegalArgumentException("filters cannot be or-ed"); } for (int i = 0; i < matrix.length; i++) { matrix[i].or(dbf.matrix[i]); } }
@Override public void or(Filter filter) { if (filter == null || !(filter instanceof DynamicBloomFilter) || filter.vectorSize != this.vectorSize || filter.nbHash != this.nbHash) { throw new IllegalArgumentException("filters cannot be or-ed"); } DynamicBloomFilter dbf = (DynamicBloomFilter)filter; if (dbf.matrix.length != this.matrix.length || dbf.nr != this.nr) { throw new IllegalArgumentException("filters cannot be or-ed"); } for (int i = 0; i < matrix.length; i++) { matrix[i].or(dbf.matrix[i]); } }
protected DataByteArray bloomOr(Tuple input) throws IOException { filter = new BloomFilter(vSize, numHash, hType); try { DataBag values = (DataBag)input.get(0); for (Iterator<Tuple> it = values.iterator(); it.hasNext();) { Tuple t = it.next(); filter.or(bloomIn((DataByteArray)t.get(0))); } } catch (ExecException ee) { throw new IOException(ee); } return bloomOut(); }
private Result combineBloomFilters() throws IOException { // We get a bag of bloom filters. combine them into one Iterator<Tuple> iter = bags[0].iterator(); Tuple tup = iter.next(); DataByteArray bloomBytes = (DataByteArray) tup.get(0); BloomFilter bloomFilter = BuildBloomBase.bloomIn(bloomBytes); while (iter.hasNext()) { tup = iter.next(); bloomFilter.or(BuildBloomBase.bloomIn((DataByteArray) tup.get(0))); } Object partition = key; detachInput(); // Free up the key and bags reference return getSerializedBloomFilter(partition, bloomFilter, bloomBytes.get().length); }
@Override public void initialize() { super.initialize(); keyType.initialize(getConfiguration()); keyToBytesFn = getKeyToBytesMapFn(keyType, getConfiguration()); Iterable<BloomFilter> iterable; try { iterable = bloomData.read(getContext()); } catch (IOException e) { throw new CrunchRuntimeException("Error reading right-side of map side join: ", e); } bloomFilter = new BloomFilter(vectorSize, nbHash, Hash.MURMUR_HASH); for (BloomFilter subFilter : iterable) { bloomFilter.or(subFilter); } }