/** * Adds a collection of keys to <i>this</i> filter. * @param keys The collection of keys. */ public void add(Collection<Key> keys){ if(keys == null) { throw new IllegalArgumentException("Collection<Key> may not be null"); } for(Key key: keys) { add(key); } }//end add()
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); int sizeInWords = buckets2words(vectorSize); buckets = new long[sizeInWords]; for(int i = 0; i < sizeInWords; i++) { buckets[i] = in.readLong(); } } }
@Override public void write(DataOutput out) throws IOException { super.write(out); int sizeInWords = buckets2words(vectorSize); for(int i = 0; i < sizeInWords; i++) { out.writeLong(buckets[i]); } }
@Override public void assertWhat(Filter filter, int numInsertions, int hashId, ImmutableSet<Integer> falsePositives) { final Random rnd = new Random(); final DataOutputBuffer out = new DataOutputBuffer(); final DataInputBuffer in = new DataInputBuffer(); try { Filter tempFilter = getSymmetricFilter(filter.getClass(), numInsertions, hashId); ImmutableList.Builder<Integer> blist = ImmutableList.builder(); for (int i = 0; i < slotSize; i++) { blist.add(rnd.nextInt(numInsertions * 2)); } ImmutableList<Integer> list = blist.build(); // mark bits for later check for (Integer slot : list) { filter.add(new Key(String.valueOf(slot).getBytes())); } filter.write(out); in.reset(out.getData(), out.getLength()); tempFilter.readFields(in); for (Integer slot : list) { assertTrue("read/write mask check filter error on " + slot, filter.membershipTest(new Key(String.valueOf(slot).getBytes()))); } } catch (IOException ex) { Assert.fail("error ex !!!" + ex); } } }),
filter.add(keys); } catch (IllegalArgumentException ex) { filter.add(keys); } catch (IllegalArgumentException ex) { filter.add(keys); } catch (IllegalArgumentException ex) { filter.and(tfilter); } catch (IllegalArgumentException ex) { filter.or(tfilter); } catch (IllegalArgumentException ex) { filter.xor(tfilter); } catch (IllegalArgumentException ex) {
@Override public void assertWhat(Filter filter, int numInsertions, int hashId, ImmutableSet<Integer> falsePositives) { // add all even keys for (int i = 0; i < numInsertions; i += 2) { filter.add(new Key(Integer.toString(i).getBytes())); } // check on present even key for (int i = 0; i < numInsertions; i += 2) { Assert.assertTrue(" filter might contains " + i, filter.membershipTest(new Key(Integer.toString(i).getBytes()))); } // check on absent odd in event for (int i = 1; i < numInsertions; i += 2) { if (!falsePositives.contains(i)) { assertFalse(" filter should not contain " + i, filter.membershipTest(new Key(Integer.toString(i).getBytes()))); } } } }),
@Override public void assertWhat(Filter filter, int numInsertions, int hashId, ImmutableSet<Integer> falsePositives) { int startIntersection = numInsertions - (numInsertions - 100); int endIntersection = numInsertions - 100; Filter partialFilter = getSymmetricFilter(filter.getClass(), numInsertions, hashId); for (int i = 0; i < numInsertions; i++) { String digit = Integer.toString(i); filter.add(new Key(digit.getBytes())); if (i >= startIntersection && i <= endIntersection) { partialFilter.add(new Key(digit.getBytes())); } } // do logic AND filter.and(partialFilter); for (int i = 0; i < numInsertions; i++) { if (i >= startIntersection && i <= endIntersection) { Assert.assertTrue(" filter might contains " + i, filter.membershipTest(new Key(Integer.toString(i).getBytes()))); } } } }),
@Override public void assertWhat(Filter filter, int numInsertions, int hashId, ImmutableSet<Integer> falsePositives) { Filter evenFilter = getSymmetricFilter(filter.getClass(), numInsertions, hashId); // add all even for (int i = 0; i < numInsertions; i += 2) { evenFilter.add(new Key(Integer.toString(i).getBytes())); } // add all odd for (int i = 1; i < numInsertions; i += 2) { filter.add(new Key(Integer.toString(i).getBytes())); } // union odd with even filter.or(evenFilter); // check on present all key for (int i = 0; i < numInsertions; i++) { Assert.assertTrue(" filter might contains " + i, filter.membershipTest(new Key(Integer.toString(i).getBytes()))); } } });
try { filter.xor(symmetricFilter); filter.membershipTest(new Key(Integer.toString(i).getBytes()))); filter.add(new Key(Integer.toString(i).getBytes())); symmetricFilter.add(new Key(Integer.toString(i).getBytes())); filter.xor(symmetricFilter); filter.membershipTest(new Key(Integer.toString(i).getBytes())));
public Boolean evaluate(String key, String bloomFilter) throws HiveException { Filter bloom = BloomFactory.GetBloomFilter(bloomFilter); if (bloom != null) { return bloom.membershipTest(new Key(key.getBytes())); } else { throw new HiveException("Unable to find bloom " + bloomFilter); } }
public boolean merge(String partial) { try { if (bloomFilter == null) { bloomFilter = BloomFactory.ReadBloomFromString(partial); ///LOG.info(" read bloom from partial " + BloomFactory.WriteBloomToString(bloomFilter)); return true; } else { ///LOG.info(" ORng with merged before " + BloomFactory.WriteBloomToString(bloomFilter) ); Filter other = BloomFactory.ReadBloomFromString(partial); ///LOG.info("ORng with merged other " + BloomFactory.WriteBloomToString(other) ); bloomFilter.or(other); ///LOG.info(" ORing with merged after " + BloomFactory.WriteBloomToString(bloomFilter) ); return true; } } catch (IOException e) { LOG.error(" Error while evaluating Bloom ", e); return false; } }
public String evaluate(String bloom1Str, String bloom2Str) throws IOException { Filter bloom1 = BloomFactory.GetBloomFilter(bloom1Str); Filter bloom2 = BloomFactory.GetBloomFilter(bloom2Str); bloom1.and(bloom2); return BloomFactory.WriteBloomToString(bloom1); } }
@Override public void assertWhat(Filter filter, int numInsertions, int hashId, ImmutableSet<Integer> falsePositives) { final Random rnd = new Random(); final DataOutputBuffer out = new DataOutputBuffer(); final DataInputBuffer in = new DataInputBuffer(); try { Filter tempFilter = getSymmetricFilter(filter.getClass(), numInsertions, hashId); ImmutableList.Builder<Integer> blist = ImmutableList.builder(); for (int i = 0; i < slotSize; i++) { blist.add(rnd.nextInt(numInsertions * 2)); } ImmutableList<Integer> list = blist.build(); // mark bits for later check for (Integer slot : list) { filter.add(new Key(String.valueOf(slot).getBytes())); } filter.write(out); in.reset(out.getData(), out.getLength()); tempFilter.readFields(in); for (Integer slot : list) { assertTrue("read/write mask check filter error on " + slot, filter.membershipTest(new Key(String.valueOf(slot).getBytes()))); } } catch (IOException ex) { Assert.fail("error ex !!!" + ex); } } }),
filter.add(keys); } catch (IllegalArgumentException ex) { filter.add(keys); } catch (IllegalArgumentException ex) { filter.add(keys); } catch (IllegalArgumentException ex) { filter.and(tfilter); } catch (IllegalArgumentException ex) { filter.or(tfilter); } catch (IllegalArgumentException ex) { filter.xor(tfilter); } catch (IllegalArgumentException ex) {
@Override public void assertWhat(Filter filter, int numInsertions, int hashId, ImmutableSet<Integer> falsePositives) { // add all even keys for (int i = 0; i < numInsertions; i += 2) { filter.add(new Key(Integer.toString(i).getBytes())); } // check on present even key for (int i = 0; i < numInsertions; i += 2) { Assert.assertTrue(" filter might contains " + i, filter.membershipTest(new Key(Integer.toString(i).getBytes()))); } // check on absent odd in event for (int i = 1; i < numInsertions; i += 2) { if (!falsePositives.contains(i)) { assertFalse(" filter should not contain " + i, filter.membershipTest(new Key(Integer.toString(i).getBytes()))); } } } }),
@Override public void assertWhat(Filter filter, int numInsertions, int hashId, ImmutableSet<Integer> falsePositives) { int startIntersection = numInsertions - (numInsertions - 100); int endIntersection = numInsertions - 100; Filter partialFilter = getSymmetricFilter(filter.getClass(), numInsertions, hashId); for (int i = 0; i < numInsertions; i++) { String digit = Integer.toString(i); filter.add(new Key(digit.getBytes())); if (i >= startIntersection && i <= endIntersection) { partialFilter.add(new Key(digit.getBytes())); } } // do logic AND filter.and(partialFilter); for (int i = 0; i < numInsertions; i++) { if (i >= startIntersection && i <= endIntersection) { Assert.assertTrue(" filter might contains " + i, filter.membershipTest(new Key(Integer.toString(i).getBytes()))); } } } }),
@Override public void assertWhat(Filter filter, int numInsertions, int hashId, ImmutableSet<Integer> falsePositives) { Filter evenFilter = getSymmetricFilter(filter.getClass(), numInsertions, hashId); // add all even for (int i = 0; i < numInsertions; i += 2) { evenFilter.add(new Key(Integer.toString(i).getBytes())); } // add all odd for (int i = 1; i < numInsertions; i += 2) { filter.add(new Key(Integer.toString(i).getBytes())); } // union odd with even filter.or(evenFilter); // check on present all key for (int i = 0; i < numInsertions; i++) { Assert.assertTrue(" filter might contains " + i, filter.membershipTest(new Key(Integer.toString(i).getBytes()))); } } });
try { filter.xor(symmetricFilter); filter.membershipTest(new Key(Integer.toString(i).getBytes()))); filter.add(new Key(Integer.toString(i).getBytes())); symmetricFilter.add(new Key(Integer.toString(i).getBytes())); filter.xor(symmetricFilter); filter.membershipTest(new Key(Integer.toString(i).getBytes())));
@Nullable public Boolean evaluate(@Nullable Text bloomStr, @Nullable Text keyStr) throws HiveException { if (bloomStr == null) { return null; } if (keyStr == null) { return Boolean.FALSE; } Filter bloom = getFilter(bloomStr); key.set(keyStr.copyBytes(), 1.0d); return Boolean.valueOf(bloom.membershipTest(key)); }
public String evaluate(String bloom1Str, String bloom2Str) throws IOException { Filter bloom1 = BloomFactory.GetBloomFilter(bloom1Str); Filter bloom2 = BloomFactory.GetBloomFilter(bloom2Str); bloom1.or(bloom2); return BloomFactory.WriteBloomToString(bloom1); } }