/** * Split passed range. Expensive operation relatively. Uses BigInteger math. * Useful splitting ranges for MapReduce jobs. * @param a Beginning of range * @param b End of range * @param num Number of times to split range. Pass 1 if you want to split * the range in two; i.e. one split. * @return Array of dividing values */ public static byte [][] split(final byte [] a, final byte [] b, final int num) { return split(a, b, false, num); }
@Override public byte[] split(byte[] start, byte[] end) { return Bytes.split(start, end, 1)[1]; }
private byte[][] getSplitKeys(byte[] startKey, byte[] endKey, int numRegions) { if (numRegions < 3) { throw new IllegalArgumentException("Must create at least three regions"); } else if (Bytes.compareTo(startKey, endKey) >= 0) { throw new IllegalArgumentException("Start key must be smaller than end key"); } if (numRegions == 3) { return new byte[][] { startKey, endKey }; } byte[][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); if (splitKeys == null || splitKeys.length != numRegions - 1) { throw new IllegalArgumentException("Unable to split key range into enough regions"); } return splitKeys; }
@Override public void createTable(TableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException { if(numRegions < 3) { throw new IllegalArgumentException("Must create at least three regions"); } else if(Bytes.compareTo(startKey, endKey) >= 0) { throw new IllegalArgumentException("Start key must be smaller than end key"); } if (numRegions == 3) { createTable(desc, new byte[][]{startKey, endKey}); return; } byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); if(splitKeys == null || splitKeys.length != numRegions - 1) { throw new IllegalArgumentException("Unable to split key range into enough regions"); } createTable(desc, splitKeys); }
@Override public void createTable(TableDescriptor desc, byte [] startKey, byte [] endKey, int numRegions) throws IOException { if(numRegions < 3) { throw new IllegalArgumentException("Must create at least three regions"); } else if(Bytes.compareTo(startKey, endKey) >= 0) { throw new IllegalArgumentException("Start key must be smaller than end key"); } if (numRegions == 3) { createTable(desc, new byte[][]{startKey, endKey}); return; } byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); if(splitKeys == null || splitKeys.length != numRegions - 1) { throw new IllegalArgumentException("Unable to split key range into enough regions"); } createTable(desc, splitKeys); }
/** * Create region split keys between startkey and endKey * * @param startKey * @param endKey * @param numRegions the number of regions to be created. it has to be greater than 3. * @return */ public byte[][] getRegionSplitStartKeys(byte[] startKey, byte[] endKey, int numRegions){ assertTrue(numRegions>3); byte [][] tmpSplitKeys = Bytes.split(startKey, endKey, numRegions - 3); byte [][] result = new byte[tmpSplitKeys.length+1][]; System.arraycopy(tmpSplitKeys, 0, result, 1, tmpSplitKeys.length); result[0] = HConstants.EMPTY_BYTE_ARRAY; return result; }
/** * Create a table with multiple regions. * @param tableName * @param family * @param numRegions * @return A Table instance for the created table. * @throws IOException */ public Table createMultiRegionTable(TableName tableName, byte[] family, int numRegions) throws IOException { if (numRegions < 3) throw new IOException("Must create at least 3 regions"); byte[] startKey = Bytes.toBytes("aaaaa"); byte[] endKey = Bytes.toBytes("zzzzz"); byte[][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); return createTable(tableName, new byte[][] { family }, splitKeys); }
@Override public byte[][] split(byte[] start, byte[] end, int numSplits, boolean inclusive) { if (Arrays.equals(start, HConstants.EMPTY_BYTE_ARRAY)) { start = firstRowBytes; } if (Arrays.equals(end, HConstants.EMPTY_BYTE_ARRAY)) { end = lastRowBytes; } Preconditions.checkArgument( Bytes.compareTo(end, start) > 0, "last row (%s) is configured less than first row (%s)", Bytes.toStringBinary(end), Bytes.toStringBinary(start)); byte[][] splits = Bytes.split(start, end, true, numSplits - 1); Preconditions.checkState(splits != null, "Could not calculate input splits with given user input: " + this); if (inclusive) { return splits; } else { // remove endpoints, which are included in the splits list return Arrays.copyOfRange(splits, 1, splits.length - 1); } }
@Override public byte[][] split(int numRegions) { Preconditions.checkArgument( Bytes.compareTo(lastRowBytes, firstRowBytes) > 0, "last row (%s) is configured less than first row (%s)", Bytes.toStringBinary(lastRowBytes), Bytes.toStringBinary(firstRowBytes)); byte[][] splits = Bytes.split(firstRowBytes, lastRowBytes, true, numRegions - 1); Preconditions.checkState(splits != null, "Could not split region with given user input: " + this); // remove endpoints, which are included in the splits list return splits == null? null: Arrays.copyOfRange(splits, 1, splits.length - 1); }
public void testSplit2() throws Exception { // More split tests. byte [] lowest = Bytes.toBytes("http://A"); byte [] highest = Bytes.toBytes("http://z"); byte [] middle = Bytes.toBytes("http://]"); byte [][] parts = Bytes.split(lowest, highest, 1); for (int i = 0; i < parts.length; i++) { System.out.println(Bytes.toString(parts[i])); } assertEquals(3, parts.length); assertTrue(Bytes.equals(parts[1], middle)); }
@Override public int getPartition(final ImmutableBytesWritable key, final VALUE value, final int reduces) { if (reduces == 1) return 0; if (this.lastReduces != reduces) { this.splits = Bytes.split(this.startkey, this.endkey, reduces - 1); for (int i = 0; i < splits.length; i++) { LOG.info(Bytes.toStringBinary(splits[i])); } this.lastReduces = reduces; } int pos = Bytes.binarySearch(this.splits, key.get(), key.getOffset(), key.getLength()); // Below code is from hfile index search. if (pos < 0) { pos++; pos *= -1; if (pos == 0) { // falls before the beginning of the file. throw new RuntimeException("Key outside start/stop range: " + key.toString()); } pos--; } return pos; }
public void testSplit() throws Exception { byte [] lowest = Bytes.toBytes("AAA"); byte [] middle = Bytes.toBytes("CCC"); byte [] highest = Bytes.toBytes("EEE"); byte [][] parts = Bytes.split(lowest, highest, 1); for (int i = 0; i < parts.length; i++) { System.out.println(Bytes.toString(parts[i])); } assertEquals(3, parts.length); assertTrue(Bytes.equals(parts[1], middle)); // Now divide into three parts. Change highest so split is even. highest = Bytes.toBytes("DDD"); parts = Bytes.split(lowest, highest, 2); for (int i = 0; i < parts.length; i++) { System.out.println(Bytes.toString(parts[i])); } assertEquals(4, parts.length); // Assert that 3rd part is 'CCC'. assertTrue(Bytes.equals(parts[2], middle)); }
public void testSplit3() throws Exception { // Test invalid split cases byte [] low = { 1, 1, 1 }; byte [] high = { 1, 1, 3 }; // If swapped, should throw IAE try { Bytes.split(high, low, 1); assertTrue("Should not be able to split if low > high", false); } catch(IllegalArgumentException iae) { // Correct } // Single split should work byte [][] parts = Bytes.split(low, high, 1); for (int i = 0; i < parts.length; i++) { System.out.println("" + i + " -> " + Bytes.toStringBinary(parts[i])); } assertTrue("Returned split should have 3 parts but has " + parts.length, parts.length == 3); // If split more than once, use additional byte to split parts = Bytes.split(low, high, 2); assertTrue("Split with an additional byte", parts != null); assertEquals(parts.length, low.length + 1); // Split 0 times should throw IAE try { parts = Bytes.split(low, high, 0); assertTrue("Should not be able to split 0 times", false); } catch(IllegalArgumentException iae) { // Correct } }
byte[][] splitKeys = Bytes.split(startRow, endRow, true, n-1); for (int i = 0; i < splitKeys.length - 1; i++) {
private void splitRegion(final RegionInfo regionInfo) throws IOException { byte[][] splitPoints = Bytes.split(regionInfo.getStartKey(), regionInfo.getEndKey(), 1); admin.split(regionInfo.getTable(), splitPoints[1]); }
protected final void splitRegion(RegionInfo regionInfo) throws IOException { byte[][] splitPoints = Bytes.split(regionInfo.getStartKey(), regionInfo.getEndKey(), 1); admin.split(regionInfo.getTable(), splitPoints[1]); }
public static void main(String[] args) { // vv BytesSplit byte[][] splits = Bytes.split(Bytes.toBytes(0), Bytes.toBytes(100), 9); // co BytesSplit-1-9Splits The number defines the amount of splits performed. Splitting one region nine times results in ten parts. int n = 0; for (byte[] split : splits) { System.out.println("Split key[" + ++n + "]: " + Bytes.toInt(split)); } // ^^ BytesSplit } }
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(cf)).build()) .setReplicationScope(HConstants.REPLICATION_SCOPE_LOCAL).build(); UTIL.createTable(tableDescriptor, Bytes.split(Bytes.toBytes(1), Bytes.toBytes(256), 123)); assertTrue(UTIL.getAdmin().getRegions(tableName).size() > 0); for (RegionInfo region : UTIL.getAdmin().getRegions(tableName)) {
@Test public void testSplitBytes() { byte[] startRow = Bytes.toBytes("EA"); byte[] stopRow = Bytes.toBytes("EZ"); byte[][] splitPoints = Bytes.split(startRow, stopRow, 10); for (byte[] splitPoint : splitPoints) { assertTrue(Bytes.toStringBinary(splitPoint), Bytes.compareTo(startRow, splitPoint) <= 0); assertTrue(Bytes.toStringBinary(splitPoint), Bytes.compareTo(stopRow, splitPoint) >= 0); } }
@Test public void testSplitBytes() { byte[] startRow = Bytes.toBytes("EA"); byte[] stopRow = Bytes.toBytes("EZ"); byte[][] splitPoints = Bytes.split(startRow, stopRow, 10); for (byte[] splitPoint : splitPoints) { assertTrue(Bytes.toStringBinary(splitPoint), Bytes.compareTo(startRow, splitPoint) <= 0); assertTrue(Bytes.toStringBinary(splitPoint), Bytes.compareTo(stopRow, splitPoint) >= 0); } }