private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { Path dirName = new Path(System.getProperty("test.build.data",".") + name + ".bloommapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in the opposite order of how keys were inserted BloomMapFile.Reader reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); }
writer.append(key, value); writer.close();
@SuppressWarnings("deprecation") private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); BloomMapFile.Writer writer = null; BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in opposite order of how keys were inserted reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); } finally { IOUtils.cleanup(null, writer, reader); } }
defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, WritableComparator.get(Text.class), Text.class); assertNotNull("testBloomMapFileConstructors error !!!", writer);
defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, WritableComparator.get(Text.class), Text.class); assertNotNull("testBloomMapFileConstructors error !!!", writer);
writer.append(key, value); writer.close();
writer.append(key, value); writer.close();
/** * test {@link BloomMapFile.Reader.get()} method */ public void testGetBloomMapFile() { int SIZE = 10; BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); for (int i = 0; i < SIZE; i++) { writer.append(new IntWritable(i), new Text()); } writer.close(); reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { assertNotNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(i), new Text())); } assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
@SuppressWarnings("deprecation") private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); BloomMapFile.Writer writer = null; BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in opposite order of how keys were inserted reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader.get()} method */ public void testGetBloomMapFile() { int SIZE = 10; BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); for (int i = 0; i < SIZE; i++) { writer.append(new IntWritable(i), new Text()); } writer.close(); reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { assertNotNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(i), new Text())); } assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader} constructor with * IOException */ public void testIOExceptionInWriterConstructor() { Path dirNameSpy = spy(TEST_FILE); BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader} constructor with * IOException */ public void testIOExceptionInWriterConstructor() { Path dirNameSpy = spy(TEST_FILE); BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@code BloomMapFile.delete()} method */ public void testDeleteFile() { BloomMapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); assertNotNull("testDeleteFile error !!!", writer); writer.close(); BloomMapFile.delete(fs, TEST_FILE.toString()); } catch (Exception ex) { fail("unexpect ex in testDeleteFile !!!"); } finally { IOUtils.cleanup(null, writer); } }
/** * test {@code BloomMapFile.delete()} method */ public void testDeleteFile() { BloomMapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); assertNotNull("testDeleteFile error !!!", writer); writer.close(); BloomMapFile.delete(fs, TEST_FILE.toString()); } catch (Exception ex) { fail("unexpect ex in testDeleteFile !!!"); } finally { IOUtils.cleanup(null, writer); } }