private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { Path dirName = new Path(System.getProperty("test.build.data",".") + name + ".bloommapfile"); FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(dirName); BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in the opposite order of how keys were inserted BloomMapFile.Reader reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); }
Path qualifiedDirName = fs.makeQualified(dirName); conf.setInt("io.mapfile.bloom.size", 2048); BloomMapFile.Writer writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, Text.class); IntWritable key = new IntWritable();
@SuppressWarnings("deprecation") private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); BloomMapFile.Writer writer = null; BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in opposite order of how keys were inserted reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); } finally { IOUtils.cleanup(null, writer, reader); } }
FileSystem ts = FileSystem.get(conf); String testFileName = TEST_FILE.toString(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD); assertNotNull("testBloomMapFileConstructors error !!!", writer);
FileSystem ts = FileSystem.get(conf); String testFileName = TEST_FILE.toString(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.BLOCK); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultCodec, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD, defaultProgress); assertNotNull("testBloomMapFileConstructors error !!!", writer); writer.close(); writer = new BloomMapFile.Writer(conf, ts, testFileName, IntWritable.class, Text.class, CompressionType.RECORD); assertNotNull("testBloomMapFileConstructors error !!!", writer);
BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, Text.class); IntWritable key = new IntWritable();
BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), IntWritable.class, Text.class); IntWritable key = new IntWritable();
/** * test {@link BloomMapFile.Reader.get()} method */ public void testGetBloomMapFile() { int SIZE = 10; BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); for (int i = 0; i < SIZE; i++) { writer.append(new IntWritable(i), new Text()); } writer.close(); reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { assertNotNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(i), new Text())); } assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
@SuppressWarnings("deprecation") private void checkMembershipVaryingSizedKeys(String name, List<Text> keys) throws Exception { FileSystem fs = FileSystem.getLocal(conf); Path qualifiedDirName = fs.makeQualified(TEST_DIR); BloomMapFile.Writer writer = null; BloomMapFile.Reader reader = null; try { writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.toString(), Text.class, NullWritable.class); for (Text key : keys) { writer.append(key, NullWritable.get()); } writer.close(); // will check for membership in opposite order of how keys were inserted reader = new BloomMapFile.Reader(fs, qualifiedDirName.toString(), conf); Collections.reverse(keys); for (Text key : keys) { assertTrue("False negative for existing key " + key, reader.probablyHasKey(key)); } reader.close(); fs.delete(qualifiedDirName, true); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader} constructor with * IOException */ public void testIOExceptionInWriterConstructor() { Path dirNameSpy = spy(TEST_FILE); BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader.get()} method */ public void testGetBloomMapFile() { int SIZE = 10; BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); for (int i = 0; i < SIZE; i++) { writer.append(new IntWritable(i), new Text()); } writer.close(); reader = new BloomMapFile.Reader(TEST_FILE, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); for (int i = 0; i < SIZE; i++) { assertNotNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(i), new Text())); } assertNull("testGetBloomMapFile error !!!", reader.get(new IntWritable(SIZE + 5), new Text())); } catch (Exception ex) { fail("unexpect ex in testGetBloomMapFile !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
/** * test {@link BloomMapFile.Reader} constructor with * IOException */ public void testIOExceptionInWriterConstructor() { Path dirNameSpy = spy(TEST_FILE); BloomMapFile.Reader reader = null; BloomMapFile.Writer writer = null; try { writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); writer.append(new IntWritable(1), new Text("123124142")); writer.close(); when(dirNameSpy.getFileSystem(conf)).thenThrow(new IOException()); reader = new BloomMapFile.Reader(dirNameSpy, conf, MapFile.Reader.comparator(new WritableComparator(IntWritable.class))); assertNull("testIOExceptionInWriterConstructor error !!!", reader.getBloomFilter()); } catch (Exception ex) { fail("unexpect ex in testIOExceptionInWriterConstructor !!!"); } finally { IOUtils.cleanup(null, writer, reader); } }
@SuppressWarnings("rawtypes") @Override public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) { try { Closeable rout; HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath); Class<? extends WritableComparable> keyWritableClass = configuration.getKeyType().getWritableClass(); Class<? extends WritableComparable> valueWritableClass = configuration.getValueType().getWritableClass(); rout = new BloomMapFile.Writer(hdfsInfo.getConf(), new Path(hdfsPath), org.apache.hadoop.io.MapFile.Writer.keyClass(keyWritableClass), org.apache.hadoop.io.MapFile.Writer.valueClass(valueWritableClass), org.apache.hadoop.io.MapFile.Writer.compression(configuration.getCompressionType(), configuration.getCompressionCodec().getCodec()), org.apache.hadoop.io.MapFile.Writer.progressable(new Progressable() { @Override public void progress() { } })); return rout; } catch (IOException ex) { throw new RuntimeCamelException(ex); } }
/** * test {@code BloomMapFile.delete()} method */ public void testDeleteFile() { BloomMapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); assertNotNull("testDeleteFile error !!!", writer); writer.close(); BloomMapFile.delete(fs, TEST_FILE.toString()); } catch (Exception ex) { fail("unexpect ex in testDeleteFile !!!"); } finally { IOUtils.cleanup(null, writer); } }
/** * test {@code BloomMapFile.delete()} method */ public void testDeleteFile() { BloomMapFile.Writer writer = null; try { FileSystem fs = FileSystem.getLocal(conf); writer = new BloomMapFile.Writer(conf, TEST_FILE, MapFile.Writer.keyClass(IntWritable.class), MapFile.Writer.valueClass(Text.class)); assertNotNull("testDeleteFile error !!!", writer); writer.close(); BloomMapFile.delete(fs, TEST_FILE.toString()); } catch (Exception ex) { fail("unexpect ex in testDeleteFile !!!"); } finally { IOUtils.cleanup(null, writer); } }