public void testMD5Filter() throws Exception { // set the filter class LOG.info("Testing MD5 Filter with frequency: 1000"); SequenceFileInputFilter.setFilterClass(job, SequenceFileInputFilter.MD5Filter.class); SequenceFileInputFilter.MD5Filter.setFrequency( job.getConfiguration(), 1000); // clean input dir fs.delete(inDir, true); // for a variety of lengths for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) { LOG.info("******Number of records: " + length); createSequenceFile(length); LOG.info("Accepted " + countRecords(0) + " records"); } // clean up fs.delete(inDir, true); }
/** Filtering method * If MD5(key) % frequency==0, return true; otherwise return false * @see Filter#accept(Object) */ public boolean accept(Object key) { try { long hashcode; if (key instanceof Text) { hashcode = MD5Hashcode((Text)key); } else if (key instanceof BytesWritable) { hashcode = MD5Hashcode((BytesWritable)key); } else { ByteBuffer bb; bb = Text.encode(key.toString()); hashcode = MD5Hashcode(bb.array(), 0, bb.limit()); } if (hashcode / frequency * frequency == hashcode) return true; } catch(Exception e) { LOG.warn(e); throw new RuntimeException(e); } return false; }
/** Filtering method * If MD5(key) % frequency==0, return true; otherwise return false * @see Filter#accept(Object) */ public boolean accept(Object key) { try { long hashcode; if (key instanceof Text) { hashcode = MD5Hashcode((Text)key); } else if (key instanceof BytesWritable) { hashcode = MD5Hashcode((BytesWritable)key); } else { ByteBuffer bb; bb = Text.encode(key.toString()); hashcode = MD5Hashcode(bb.array(), 0, bb.limit()); } if (hashcode / frequency * frequency == hashcode) return true; } catch(Exception e) { LOG.warn(e); throw new RuntimeException(e); } return false; }
/** Filtering method * If MD5(key) % frequency==0, return true; otherwise return false * @see Filter#accept(Object) */ public boolean accept(Object key) { try { long hashcode; if (key instanceof Text) { hashcode = MD5Hashcode((Text)key); } else if (key instanceof BytesWritable) { hashcode = MD5Hashcode((BytesWritable)key); } else { ByteBuffer bb; bb = Text.encode(key.toString()); hashcode = MD5Hashcode(bb.array(), 0, bb.limit()); } if (hashcode / frequency * frequency == hashcode) return true; } catch(Exception e) { LOG.warn(e); throw new RuntimeException(e); } return false; }
/** Filtering method * If MD5(key) % frequency==0, return true; otherwise return false * @see Filter#accept(Object) */ public boolean accept(Object key) { try { long hashcode; if (key instanceof Text) { hashcode = MD5Hashcode((Text)key); } else if (key instanceof BytesWritable) { hashcode = MD5Hashcode((BytesWritable)key); } else { ByteBuffer bb; bb = Text.encode(key.toString()); hashcode = MD5Hashcode(bb.array(), 0, bb.limit()); } if (hashcode / frequency * frequency == hashcode) return true; } catch(Exception e) { LOG.warn(e); throw new RuntimeException(e); } return false; }
/** Filtering method * If MD5(key) % frequency==0, return true; otherwise return false * @see Filter#accept(Object) */ public boolean accept(Object key) { try { long hashcode; if (key instanceof Text) { hashcode = MD5Hashcode((Text)key); } else if (key instanceof BytesWritable) { hashcode = MD5Hashcode((BytesWritable)key); } else { ByteBuffer bb; bb = Text.encode(key.toString()); hashcode = MD5Hashcode(bb.array(), 0, bb.limit()); } if (hashcode / frequency * frequency == hashcode) return true; } catch(Exception e) { LOG.warn(e); throw new RuntimeException(e); } return false; }
public void testMD5Filter() throws Exception { // set the filter class LOG.info("Testing MD5 Filter with frequency: 1000"); SequenceFileInputFilter.setFilterClass(job, SequenceFileInputFilter.MD5Filter.class); SequenceFileInputFilter.MD5Filter.setFrequency( job.getConfiguration(), 1000); // clean input dir fs.delete(inDir, true); // for a variety of lengths for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) { LOG.info("******Number of records: " + length); createSequenceFile(length); LOG.info("Accepted " + countRecords(0) + " records"); } // clean up fs.delete(inDir, true); }
private long MD5Hashcode(BytesWritable key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
private long MD5Hashcode(BytesWritable key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
private long MD5Hashcode(BytesWritable key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
private long MD5Hashcode(BytesWritable key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
private long MD5Hashcode(BytesWritable key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
private long MD5Hashcode(Text key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
public MD5Filter() { mf = new org.apache.hadoop.mapreduce.lib.input. SequenceFileInputFilter.MD5Filter(); }
private long MD5Hashcode(Text key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
private long MD5Hashcode(Text key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
private long MD5Hashcode(Text key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
private long MD5Hashcode(Text key) throws DigestException { return MD5Hashcode(key.getBytes(), 0, key.getLength()); }
public MD5Filter() { mf = new org.apache.hadoop.mapreduce.lib.input. SequenceFileInputFilter.MD5Filter(); }
public MD5Filter() { mf = new org.apache.hadoop.mapreduce.lib.input. SequenceFileInputFilter.MD5Filter(); }