byte[][] splits = new RegionSplitter.UniformSplit().split(totalNumberOfRegions);
private static byte[][] getSplits(int numRegions) { RegionSplitter.UniformSplit split = new RegionSplitter.UniformSplit(); split.setFirstRow(Bytes.toBytes(0L)); split.setLastRow(Bytes.toBytes(Long.MAX_VALUE)); return split.split(numRegions); }
byte[][] splits = new RegionSplitter.UniformSplit().split(totalNumberOfRegions);
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, true, tableDir, new RegionSplitter.UniformSplit(), numSplitsPerRegion); } else {
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, jobConf, true, tableDir, new RegionSplitter.UniformSplit(), numSplitsPerRegion); } else {
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, jobConf, true, tableDir, new RegionSplitter.UniformSplit(), numSplitsPerRegion); } else {
UniformSplit splitter = new UniformSplit(); splitter.split(1); throw new AssertionError("Splitting into <2 regions should have thrown exception"); } catch (IllegalArgumentException e) { } byte[][] twoRegionsSplits = splitter.split(2); assertEquals(1, twoRegionsSplits.length); assertArrayEquals(twoRegionsSplits[0], new byte[] { (byte) 0x80, 0, 0, 0, 0, 0, 0, 0 }); byte[][] threeRegionsSplits = splitter.split(3); assertEquals(2, threeRegionsSplits.length); byte[] expectedSplit0 = new byte[] {0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55}; byte[] splitPoint = splitter.split(new byte[] {0x10}, new byte[] {0x30}); assertArrayEquals(new byte[] { 0x20 }, splitPoint); assertArrayEquals(lastRow, splitter.lastRow()); byte[] firstRow = ArrayUtils.EMPTY_BYTE_ARRAY; assertArrayEquals(firstRow, splitter.firstRow()); splitPoint = splitter.split(firstRow, new byte[] {0x20}); assertArrayEquals(splitPoint, new byte[] { 0x10 }); splitPoint = splitter.split(new byte[] {(byte)0xdf, xFF, xFF, xFF, xFF, xFF, xFF, xFF}, lastRow); assertArrayEquals(splitPoint, new byte[] { (byte) 0xef, xFF, xFF, xFF, xFF, xFF, xFF, xFF });
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, true, tableDir, new RegionSplitter.UniformSplit(), numSplitsPerRegion); } else {
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, false, tmpTableDir, new RegionSplitter.UniformSplit(), numSplitsPerRegion); } else {
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, false, tmpTableDir, new RegionSplitter.UniformSplit(), numSplitsPerRegion); } else {
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, false, tmpTableDir, new RegionSplitter.UniformSplit(), numSplitsPerRegion); } else {
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, false, tmpTableDir, new RegionSplitter.UniformSplit(), numSplitsPerRegion); } else {
@Test public void testUserInput() { SplitAlgorithm algo = new HexStringSplit(); assertFalse(splitFailsPrecondition(algo)); // default settings are fine assertFalse(splitFailsPrecondition(algo, "00", "AA")); // custom is fine assertTrue(splitFailsPrecondition(algo, "AA", "00")); // range error assertTrue(splitFailsPrecondition(algo, "AA", "AA")); // range error assertFalse(splitFailsPrecondition(algo, "0", "2", 3)); // should be fine assertFalse(splitFailsPrecondition(algo, "0", "A", 11)); // should be fine assertTrue(splitFailsPrecondition(algo, "0", "A", 12)); // too granular algo = new DecimalStringSplit(); assertFalse(splitFailsPrecondition(algo)); // default settings are fine assertFalse(splitFailsPrecondition(algo, "00", "99")); // custom is fine assertTrue(splitFailsPrecondition(algo, "99", "00")); // range error assertTrue(splitFailsPrecondition(algo, "99", "99")); // range error assertFalse(splitFailsPrecondition(algo, "0", "2", 3)); // should be fine assertFalse(splitFailsPrecondition(algo, "0", "9", 10)); // should be fine assertTrue(splitFailsPrecondition(algo, "0", "9", 11)); // too granular algo = new UniformSplit(); assertFalse(splitFailsPrecondition(algo)); // default settings are fine assertFalse(splitFailsPrecondition(algo, "\\x00", "\\xAA")); // custom is fine assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\x00")); // range error assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\xAA")); // range error assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x02", 3)); // should be fine assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 11)); // should be fine assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 12)); // should be fine }
@Test public void testWithMockedMapReduceWithNoStartRowStopRow() throws Exception { setupCluster(); String snapshotName = "testWithMockedMapReduceMultiRegion"; final TableName tableName = TableName.valueOf(name.getMethodName()); try { createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 10); Configuration conf = UTIL.getConfiguration(); conf.setBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_KEY, false); Job job = new Job(conf); Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName); // test scan without startRow and stopRow Scan scan2 = new Scan(); TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, scan2, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, false, tmpTableDir, new RegionSplitter.UniformSplit(), 5); verifyWithMockedMapReduce(job, 10, 50, HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); } finally { UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); tearDownCluster(); } }
@Test public void testWithMockedMapReduceWithSplitsPerRegion() throws Exception { setupCluster(); String snapshotName = "testWithMockedMapReduceMultiRegion"; final TableName tableName = TableName.valueOf(name.getMethodName()); try { createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 10); Configuration conf = UTIL.getConfiguration(); conf.setBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_KEY, false); Job job = new Job(conf); Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName); // test scan with startRow and stopRow Scan scan = new Scan(bbc, yya); TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, NullWritable.class, job, false, tmpTableDir, new RegionSplitter.UniformSplit(), 5); verifyWithMockedMapReduce(job, 10, 40, bbc, yya); } finally { UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); tearDownCluster(); } }
private byte[][] calculateRegionSplits(MongoURI uri, String tableName) throws Exception { DBCollection collection = uri.connectDB().getCollection(uri.getCollection()); long size = collection.getStats().getLong("size"); long regionSize = ConfigUtil.getPresplitTableRegionSize(_conf); int numRegions = (int) Math.min((size / regionSize) + 1, 4096); if (numRegions > 1) { log.info("Pre-splitting " + tableName + " into " + numRegions + " regions"); RegionSplitter.UniformSplit splitter = new RegionSplitter.UniformSplit(); return splitter.split(numRegions); } else { log.info("Not splitting " + tableName + ", because the data can fit into a single region"); return new byte[0][0]; } }
@Override public RegionSplitter.SplitAlgorithm createSplitter(int cardinality) { return new RegionSplitter.UniformSplit(); } };