Best code snippets using org.apache.hadoop.util.bloom.DynamicBloomFilter(Showing top 15 results out of 315)

@Override
public synchronized void append(WritableComparable key, Writable val)
  throws IOException {
 super.append(key, val);
 buf.reset();
 key.write(buf);
 bloomKey.set(byteArrayForBloomKey(buf), 1.0);
 bloomFilter.add(bloomKey);
}

@Override
public synchronized void close() throws IOException {
 super.close();
 DataOutputStream out = fs.create(new Path(dir, BLOOM_FILE_NAME), true);
 bloomFilter.write(out);
 out.flush();
 out.close();
}

private void initBloomFilter(FileSystem fs, String dirName,
  Configuration conf) {
 try {
  DataInputStream in = fs.open(new Path(dirName, BLOOM_FILE_NAME));
  bloomFilter = new DynamicBloomFilter();
  bloomFilter.readFields(in);
  in.close();
 } catch (IOException ioe) {
  LOG.warn("Can't open BloomFilter: " + ioe + " - fallback to MapFile.");
  bloomFilter = null;
 }
}

private synchronized void initBloomFilter(Configuration conf) {
 numKeys = conf.getInt("io.mapfile.bloom.size", 1024 * 1024);
 // vector size should be <code>-kn / (ln(1 - c^(1/k)))</code> bits for
 // single key, where <code> is the number of hash functions,
 // <code>n</code> is the number of keys and <code>c</code> is the desired
 // max. error rate.
 // Our desired error rate is by default 0.005, i.e. 0.5%
 float errorRate = conf.getFloat("io.mapfile.bloom.error.rate", 0.005f);
 vectorSize = (int)Math.ceil((double)(-HASH_COUNT * numKeys) /
   Math.log(1.0 - Math.pow(errorRate, 1.0/HASH_COUNT)));
 bloomFilter = new DynamicBloomFilter(vectorSize, HASH_COUNT,
   Hash.getHashType(conf), numKeys);
}

@Override
public void add(Key key) {
 if (key == null) {
  throw new NullPointerException("Key can not be null");
 }
 BloomFilter bf = getActiveStandardBF();
 if (bf == null) {
  addRow();
  bf = matrix[matrix.length - 1];
  currentNbRecord = 0;
 }
 bf.add(key);
 currentNbRecord++;
}

private void initBloomFilter(Path dirName, 
               Configuration conf) {
 
 DataInputStream in = null;
 try {
  FileSystem fs = dirName.getFileSystem(conf);
  in = fs.open(new Path(dirName, BLOOM_FILE_NAME));
  bloomFilter = new DynamicBloomFilter();
  bloomFilter.readFields(in);
  in.close();
  in = null;
 } catch (IOException ioe) {
  LOG.warn("Can't open BloomFilter: " + ioe + " - fallback to MapFile.");
  bloomFilter = null;
 } finally {
  IOUtils.closeStream(in);
 }
}

@Test
public void testDynamicBloomFilter() {
 int hashId = Hash.JENKINS_HASH;    
 Filter filter = new DynamicBloomFilter(bitSize, hashFunctionNumber,
   Hash.JENKINS_HASH, 3);    
 BloomFilterCommonTester.of(hashId, numInsertions)
   .withFilterInstance(filter)
   .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
       BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
       BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
       BloomFilterTestStrategy.WRITE_READ_STRATEGY,
       BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY))
       .test();
 
 assertNotNull("testDynamicBloomFilter error ", filter.toString());
}

@Override
public synchronized void append(WritableComparable key, Writable val)
  throws IOException {
 super.append(key, val);
 buf.reset();
 key.write(buf);
 bloomKey.set(buf.getData(), 1.0);
 bloomFilter.add(bloomKey);
}

@Override
public synchronized void close() throws IOException {
 super.close();
 DataOutputStream out = fs.create(new Path(dir, BLOOM_FILE_NAME), true);
 try {
  bloomFilter.write(out);
  out.flush();
  out.close();
  out = null;
 } finally {
  IOUtils.closeStream(out);
 }
}

@Override
public synchronized void append(WritableComparable key, Writable val)
  throws IOException {
 super.append(key, val);
 buf.reset();
 key.write(buf);
 bloomKey.set(byteArrayForBloomKey(buf), 1.0);
 bloomFilter.add(bloomKey);
}

private synchronized void initBloomFilter(Configuration conf) {
 numKeys = conf.getInt("io.mapfile.bloom.size", 1024 * 1024);
 // vector size should be <code>-kn / (ln(1 - c^(1/k)))</code> bits for
 // single key, where <code> is the number of hash functions,
 // <code>n</code> is the number of keys and <code>c</code> is the desired
 // max. error rate.
 // Our desired error rate is by default 0.005, i.e. 0.5%
 float errorRate = conf.getFloat("io.mapfile.bloom.error.rate", 0.005f);
 vectorSize = (int)Math.ceil((double)(-HASH_COUNT * numKeys) /
   Math.log(1.0 - Math.pow(errorRate, 1.0/HASH_COUNT)));
 bloomFilter = new DynamicBloomFilter(vectorSize, HASH_COUNT,
   Hash.getHashType(conf), numKeys);
}

@Override
public synchronized void close() throws IOException {
 super.close();
 DataOutputStream out = fs.create(new Path(dir, BLOOM_FILE_NAME), true);
 try {
  bloomFilter.write(out);
  out.flush();
  out.close();
  out = null;
 } finally {
  IOUtils.closeStream(out);
 }
}

@Override
public void add(Key key) {
 if (key == null) {
  throw new NullPointerException("Key can not be null");
 }
 BloomFilter bf = getActiveStandardBF();
 if (bf == null) {
  addRow();
  bf = matrix[matrix.length - 1];
  currentNbRecord = 0;
 }
 bf.add(key);
 currentNbRecord++;
}

@Override
public void add(Key key) {
 if (key == null) {
  throw new NullPointerException("Key can not be null");
 }
 BloomFilter bf = getActiveStandardBF();
 if (bf == null) {
  addRow();
  bf = matrix[matrix.length - 1];
  currentNbRecord = 0;
 }
 bf.add(key);
 currentNbRecord++;
}

private synchronized void initBloomFilter(Configuration conf) {
 numKeys = conf.getInt(
   IO_MAPFILE_BLOOM_SIZE_KEY, IO_MAPFILE_BLOOM_SIZE_DEFAULT);
 // vector size should be <code>-kn / (ln(1 - c^(1/k)))</code> bits for
 // single key, where <code> is the number of hash functions,
 // <code>n</code> is the number of keys and <code>c</code> is the desired
 // max. error rate.
 // Our desired error rate is by default 0.005, i.e. 0.5%
 float errorRate = conf.getFloat(
   IO_MAPFILE_BLOOM_ERROR_RATE_KEY, IO_MAPFILE_BLOOM_ERROR_RATE_DEFAULT);
 vectorSize = (int)Math.ceil((double)(-HASH_COUNT * numKeys) /
   Math.log(1.0 - Math.pow(errorRate, 1.0/HASH_COUNT)));
 bloomFilter = new DynamicBloomFilter(vectorSize, HASH_COUNT,
   Hash.getHashType(conf), numKeys);
}