public void testPercentFilter() throws Exception { LOG.info("Testing Percent Filter with frequency: 1000"); // set the filter class SequenceFileInputFilter.setFilterClass(job, SequenceFileInputFilter.PercentFilter.class); SequenceFileInputFilter.PercentFilter.setFrequency( job.getConfiguration(), 1000); // clean input dir fs.delete(inDir, true); // for a variety of lengths for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) { LOG.info("******Number of records: "+length); createSequenceFile(length); int count = countRecords(1); LOG.info("Accepted " + count + " records"); int expectedCount = length / 1000; if (expectedCount * 1000 != length) expectedCount++; assertEquals(count, expectedCount); } // clean up fs.delete(inDir, true); }
public void testPercentFilter() throws Exception { LOG.info("Testing Percent Filter with frequency: 1000"); // set the filter class SequenceFileInputFilter.setFilterClass(job, SequenceFileInputFilter.PercentFilter.class); SequenceFileInputFilter.PercentFilter.setFrequency( job.getConfiguration(), 1000); // clean input dir fs.delete(inDir, true); // for a variety of lengths for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) { LOG.info("******Number of records: "+length); createSequenceFile(length); int count = countRecords(1); LOG.info("Accepted " + count + " records"); int expectedCount = length / 1000; if (expectedCount * 1000 != length) expectedCount++; assertEquals(count, expectedCount); } // clean up fs.delete(inDir, true); }
public PercentFilter() { pf = new org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter(); }
public PercentFilter() { pf = new org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter(); }
public PercentFilter() { pf = new org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter(); }
public PercentFilter() { pf = new org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter(); }
/** set the frequency and stores it in conf * @param conf configuration * @param frequency filtering frequencey */ public static void setFrequency(Configuration conf, int frequency) { org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter.setFrequency(conf, frequency); }
/** set the frequency and stores it in conf * @param conf configuration * @param frequency filtering frequencey */ public static void setFrequency(Configuration conf, int frequency) { org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter.setFrequency(conf, frequency); }
/** set the frequency and stores it in conf * @param conf configuration * @param frequency filtering frequencey */ public static void setFrequency(Configuration conf, int frequency) { org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter.setFrequency(conf, frequency); }
/** set the frequency and stores it in conf * @param conf configuration * @param frequency filtering frequencey */ public static void setFrequency(Configuration conf, int frequency) { org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter.setFrequency(conf, frequency); }
/** configure the filter by checking the configuration * * @param conf configuration */ public void setConf(Configuration conf) { pf.setConf(conf); }
/** configure the filter by checking the configuration * * @param conf configuration */ public void setConf(Configuration conf) { pf.setConf(conf); }
/** Filtering method * If record# % frequency==0, return true; otherwise return false * @see org.apache.hadoop.mapred.SequenceFileInputFilter.Filter#accept(Object) */ public boolean accept(Object key) { return pf.accept(key); } }
/** Filtering method * If record# % frequency==0, return true; otherwise return false * @see org.apache.hadoop.mapred.SequenceFileInputFilter.Filter#accept(Object) */ public boolean accept(Object key) { return pf.accept(key); } }
/** Filtering method * If record# % frequency==0, return true; otherwise return false * @see org.apache.hadoop.mapred.SequenceFileInputFilter.Filter#accept(Object) */ public boolean accept(Object key) { return pf.accept(key); } }
/** configure the filter by checking the configuration * * @param conf configuration */ public void setConf(Configuration conf) { pf.setConf(conf); }
public PercentFilter() { pf = new org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter. PercentFilter(); }
/** configure the filter by checking the configuration * * @param conf configuration */ public void setConf(Configuration conf) { pf.setConf(conf); }
/** configure the filter by checking the configuration * * @param conf configuration */ public void setConf(Configuration conf) { pf.setConf(conf); }
/** Filtering method * If record# % frequency==0, return true; otherwise return false * @see org.apache.hadoop.mapred.SequenceFileInputFilter.Filter#accept(Object) */ public boolean accept(Object key) { return pf.accept(key); } }