private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { Path dirName = new Path(System.getProperty("test.build.data",".") + name + ".bloommapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in the opposite order of how keys were inserted BloomMapFile.Reader reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); }
key.set(i); value.set("00" + i); writer.append(key, value);
@SuppressWarnings("deprecation") private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); BloomMapFile.Writer writer = null; BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in opposite order of how keys were inserted reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); } finally { IOUtils.cleanup(null, writer, reader); } }
key.set(i); value.set("00" + i); writer.append(key, value);
key.set(i); value.set("00" + i); writer.append(key, value);
/** * test {@link BloomMapFile.Reader.get()} method */ public void testGetBloomMapFile() { int SIZE = 10; BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); for (int i = 0; i < SIZE; i++) { writer.append(new IntWritable(i), new Text()); } writer.close(); reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { assertNotNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(i), new Text())); } assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader.get()} method */ public void testGetBloomMapFile() { int SIZE = 10; BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); for (int i = 0; i < SIZE; i++) { writer.append(new IntWritable(i), new Text()); } writer.close(); reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { assertNotNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(i), new Text())); } assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader} constructor with * IOException */ public void testIOExceptionInWriterConstructor() { Path dirNameSpy = spy(TEST_FILE); BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
@SuppressWarnings("deprecation") private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); BloomMapFile.Writer writer = null; BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in opposite order of how keys were inserted reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader} constructor with * IOException */ public void testIOExceptionInWriterConstructor() { Path dirNameSpy = spy(TEST_FILE); BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
@Override public long append(HdfsOutputStream hdfsostr, Object key, Object value, TypeConverter typeConverter) { try { Holder<Integer> keySize = new Holder<>(); Writable keyWritable = getWritable(key, typeConverter, keySize); Holder<Integer> valueSize = new Holder<>(); Writable valueWritable = getWritable(value, typeConverter, valueSize); ((BloomMapFile.Writer) hdfsostr.getOut()).append((WritableComparable<?>) keyWritable, valueWritable); return keySize.value + valueSize.value; } catch (Exception ex) { throw new RuntimeCamelException(ex); } }