@AfterClass public static void finish() throws Exception { RowKeyDistributorTestBase.runAfter = true; RowKeyDistributorTestBase.afterClass(); } }
@BeforeClass public static void init() throws Exception { RowKeyDistributorTestBase.beforeClass(); RowKeyDistributorTestBase.runBefore = false; RowKeyDistributorTestBase.runAfter = false; }
/** Test scan over the whole table. */ @Test public void testMapReduceUnbounded() throws IOException, InterruptedException, ClassNotFoundException { long origKeyPrefix = System.currentTimeMillis(); testMapReduceInternal(origKeyPrefix, new Scan(), 500, 500, 0, 999); }
/** Test scan without stop key. */ @Test public void testSimpleScanWithoutStopKey() throws IOException { long origKeyPrefix = System.currentTimeMillis(); int seekIntervalMinValue = 100; byte[] startKey = Bytes.toBytes(origKeyPrefix + seekIntervalMinValue); testSimpleScanInternal(origKeyPrefix, new Scan(startKey), 500, 500, 100, 999); }
private void testMapReduceInternal(long origKeyPrefix, Scan scan, int numValues, int startWithValue, int seekIntervalMinValue, int seekIntervalMaxValue) throws IOException, InterruptedException, ClassNotFoundException { int valuesCountInSeekInterval = writeTestData(origKeyPrefix, numValues, startWithValue, seekIntervalMinValue, seekIntervalMaxValue); // Reading data Configuration conf = new Configuration(testingUtility.getConfiguration()); conf.set("fs.defaultFS", "file:///"); conf.set("fs.default.name", "file:///"); conf.setInt("mapreduce.local.map.tasks.maximum", 16); conf.setInt("mapreduce.local.reduce.tasks.maximum", 16); Job job = Job.getInstance(conf, "testMapReduceInternal()-Job"); TableMapReduceUtil.initTableMapperJob(TABLE_NAME, scan, RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job); // Substituting standard TableInputFormat which was set in TableMapReduceUtil.initTableMapperJob(...) job.setInputFormatClass(WdTableInputFormat.class); keyDistributor.addInfo(job.getConfiguration()); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); boolean succeeded = job.waitForCompletion(true); Assert.assertTrue(succeeded); long mapInputRecords = job.getCounters().findCounter(RowCounterMapper.Counters.ROWS).getValue(); Assert.assertEquals(valuesCountInSeekInterval, mapInputRecords); // Need to kill the job after completion, after it could leave MRAppMaster running not terminated. // Not sure what causing this, but maybe problem in MiniYarnCluster job.killJob(); }
/** Test scan with start and stop key. */ @Test public void testSimpleScanBounded() throws IOException { long origKeyPrefix = System.currentTimeMillis(); int seekIntervalMinValue = 100; int seekIntervalMaxValue = 899; byte[] startKey = Bytes.toBytes(origKeyPrefix + seekIntervalMinValue); byte[] stopKey = Bytes.toBytes(origKeyPrefix + seekIntervalMaxValue + 1); Scan scan = new Scan(startKey, stopKey); testSimpleScanInternal(origKeyPrefix, scan, 500, 500, seekIntervalMinValue, seekIntervalMaxValue); }
private void testSimpleScanInternal(long origKeyPrefix, Scan scan, int numValues, int startWithValue, int seekIntervalMinValue, int seekIntervalMaxValue) throws IOException { int valuesCountInSeekInterval = writeTestData(origKeyPrefix, numValues, startWithValue, seekIntervalMinValue, seekIntervalMaxValue); // TODO: add some filters to the scan for better testing ResultScanner distributedScanner = DistributedScanner.create(hTable, scan, keyDistributor, Executors.newFixedThreadPool(2)); Result previous = null; int countMatched = 0; for (Result current : distributedScanner) { countMatched++; if (previous != null) { byte[] currentRowOrigKey = keyDistributor.getOriginalKey(current.getRow()); byte[] previousRowOrigKey = keyDistributor.getOriginalKey(previous.getRow()); Assert.assertTrue(Bytes.compareTo(currentRowOrigKey, previousRowOrigKey) >= 0); int currentValue = Bytes.toInt(current.getValue(CF, QUAL)); Assert.assertTrue(currentValue >= seekIntervalMinValue); Assert.assertTrue(currentValue <= seekIntervalMaxValue); } previous = current; } Assert.assertEquals(valuesCountInSeekInterval, countMatched); }
/** Test scan over the whole table. */ @Test public void testSimpleScanUnbounded() throws IOException { long origKeyPrefix = System.currentTimeMillis(); testSimpleScanInternal(origKeyPrefix, new Scan(), 500, 500, 0, 999); }
/** Test scan with start and stop key. */ @Test public void testMapReduceBounded() throws IOException, InterruptedException, ClassNotFoundException { long origKeyPrefix = System.currentTimeMillis(); int seekIntervalMinValue = 100; int seekIntervalMaxValue = 899; byte[] startKey = Bytes.toBytes(origKeyPrefix + seekIntervalMinValue); byte[] stopKey = Bytes.toBytes(origKeyPrefix + seekIntervalMaxValue + 1); Scan scan = new Scan(startKey, stopKey); testMapReduceInternal(origKeyPrefix, scan, 500, 500, seekIntervalMinValue, seekIntervalMaxValue); }