public BloomFilterBuf(long expectedEntries, long maxEntries) { if (expectedEntries > maxEntries) { bloomFilter = new BloomFilter(1); } else { bloomFilter = new BloomFilter(expectedEntries); } }
public Aggregation(long expectedEntries) { bf = new BloomFilter(expectedEntries); }
private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, double fpp, List<String> partNames) { BloomFilter bloomFilter = new BloomFilter(maxPartsPerCacheNode, fpp); for (String partName : partNames) { bloomFilter.add(partName.getBytes()); } return bloomFilter; }
public Aggregation(long expectedEntries) { try { BloomFilter bf = new BloomFilter(expectedEntries); ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); BloomFilter.serialize(bytesOut, bf); bfBytes = bytesOut.toByteArray(); } catch (Exception err) { throw new IllegalArgumentException("Error creating aggregation buffer", err); } }
/** * Deserialize a bloom filter * Read a byte stream, which was written by {@linkplain #serialize(OutputStream, BloomFilter)} * into a {@code BloomFilter} * @param in input bytestream * @return deserialized BloomFilter */ public static BloomFilter deserialize(InputStream in) throws IOException { if (in == null) { throw new IOException("Input stream is null"); } try { DataInputStream dataInputStream = new DataInputStream(in); int numHashFunc = dataInputStream.readByte(); int numLongs = dataInputStream.readInt(); long[] data = new long[numLongs]; for (int i = 0; i < numLongs; i++) { data[i] = dataInputStream.readLong(); } return new BloomFilter(data, numHashFunc); } catch (RuntimeException e) { IOException io = new IOException( "Unable to deserialize BloomFilter"); io.initCause(e); throw io; } }
@Test public void testBloomFilterPredicateValuesNonExisting() { BloomFilter bloomFilter = new BloomFilter(TEST_VALUES.size() * 10, 0.01); for (Map.Entry<Object, Type> testValue : TEST_VALUES.entrySet()) { boolean matched = checkInBloomFilter(bloomFilter, testValue.getKey(), testValue.getValue()); assertFalse(matched, "type " + testValue.getKey().getClass()); } // test unsupported type: can be supported by ORC but is not implemented yet assertTrue(checkInBloomFilter(bloomFilter, new Date(), DATE), "unsupported type DATE should always return true"); }
/** * Prepares a bloom filter from the list of partition names * @param partNames * @return */ private BloomFilter prepareBloomFilter(List <String> partNames) { BloomFilter bloomFilter = new BloomFilter(MAX_PARTITIONS_PER_CACHE_NODE, FALSE_POSITIVE_PROBABILITY); for (String partName: partNames) { bloomFilter.add(partName.getBytes()); } return bloomFilter; }
@Test public void testBloomFilterPredicateValuesExisting() BloomFilter bloomFilter = new BloomFilter(TEST_VALUES.size() * 10, 0.01);
this.bloom1 = new BloomFilter(newKeyCount); } else { this.bloom1 = new BloomFilter(newKeyCount, fpp);
this.bloom1 = new BloomFilter(newKeyCount); } else { this.bloom1 = new BloomFilter(newKeyCount, fpp);
throws Exception BloomFilter bloomFilterWrite = new BloomFilter(1000L, 0.05);
HiveBloomFilter hiveBloomFilter = new HiveBloomFilter(new BloomFilter(1000, 0.01)); OrcProto.BloomFilter emptyOrcBloomFilter = toOrcBloomFilter(hiveBloomFilter); hiveBloomFilter.addLong(1234);
private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, float fpp, List<String> partNames) { BloomFilter bloomFilter = new BloomFilter(maxPartsPerCacheNode, fpp); for (String partName : partNames) { bloomFilter.add(partName.getBytes()); } return bloomFilter; }
private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, float fpp, List<String> partNames) { BloomFilter bloomFilter = new BloomFilter(maxPartsPerCacheNode, fpp); for (String partName : partNames) { bloomFilter.add(partName.getBytes()); } return bloomFilter; }
/** * Deserialize a bloom filter * Read a byte stream, which was written by {@linkplain #serialize(OutputStream, BloomFilter)} * into a {@code BloomFilter} * @param in input bytestream * @return deserialized BloomFilter */ public static BloomFilter deserialize(InputStream in) throws IOException { if (in == null) { throw new IOException("Input stream is null"); } try { DataInputStream dataInputStream = new DataInputStream(in); int numHashFunc = dataInputStream.readByte(); int numLongs = dataInputStream.readInt(); long[] data = new long[numLongs]; for (int i = 0; i < numLongs; i++) { data[i] = dataInputStream.readLong(); } return new BloomFilter(data, numHashFunc); } catch (RuntimeException e) { IOException io = new IOException( "Unable to deserialize BloomFilter"); io.initCause(e); throw io; } }
@Test public void testBloomFilterPredicateValuesNonExisting() { BloomFilter bloomFilter = new BloomFilter(TEST_VALUES.size() * 10, 0.01); for (Map.Entry<Object, Type> testValue : TEST_VALUES.entrySet()) { boolean matched = checkInBloomFilter(bloomFilter, testValue.getKey(), testValue.getValue()); assertFalse(matched, "type " + testValue.getKey().getClass()); } // test unsupported type: can be supported by ORC but is not implemented yet assertTrue(checkInBloomFilter(bloomFilter, new Date(), DATE), "unsupported type DATE should always return true"); }
@Test public void testBloomFilterPredicateValuesNonExisting() { BloomFilter bloomFilter = new BloomFilter(TEST_VALUES.size() * 10, 0.01); for (Map.Entry<Object, Type> testValue : TEST_VALUES.entrySet()) { boolean matched = checkInBloomFilter(bloomFilter, testValue.getKey(), testValue.getValue()); assertFalse(matched, "type " + testValue.getKey().getClass()); } // test unsupported type: can be supported by ORC but is not implemented yet assertTrue(checkInBloomFilter(bloomFilter, new Date(), DATE), "unsupported type DATE should always return true"); }
@Test public void testBloomFilterPredicateValuesNonExisting() { BloomFilter bloomFilter = new BloomFilter(TEST_VALUES.size() * 10, 0.01); for (Map.Entry<Object, Type> testValue : TEST_VALUES.entrySet()) { boolean matched = checkInBloomFilter(bloomFilter, testValue.getKey(), testValue.getValue()); assertFalse(matched, "type " + testValue.getKey().getClass()); } // test unsupported type: can be supported by ORC but is not implemented yet assertTrue(checkInBloomFilter(bloomFilter, new Date(), DATE), "unsupported type DATE should always return true"); }
throws Exception BloomFilter bloomFilterWrite = new BloomFilter(1000L, 0.05);
throws Exception BloomFilter bloomFilterWrite = new BloomFilter(1000L, 0.05);