TableName tableName = ts.getTable(); String regionLocation = ts.getRegionLocation(); String encodedRegionName = ts.getEncodedRegionName(); long regionSize = ts.getLength(); byte[] startRow = ts.getStartRow(); byte[] endRow = ts.getEndRow(); new TableSplit(tableName, scan, splitKeys[i], splitKeys[i + 1], regionLocation, encodedRegionName, regionSize / n); res.add(tsplit);
throws IOException, InterruptedException { TableSplit tSplit = (TableSplit) split; LOG.info(MessageFormat.format("Input split length: {0} bytes.", tSplit.getLength())); if (tSplit.getTable() == null) { throw new IOException("Cannot create a record reader because of a" + " previous error. Please look at the previous logs lines from" Table table = connection.getTable(tSplit.getTable()); Scan sc = tSplit.getScan(); sc.setStartRow(tSplit.getStartRow()); sc.setStopRow(tSplit.getEndRow()); trr.setScan(sc); trr.setTable(table);
/** * Compares this split against the given one. * * @param split The split to compare to. * @return The result of the comparison. * @see java.lang.Comparable#compareTo(java.lang.Object) */ @Override public int compareTo(TableSplit split) { // If The table name of the two splits is the same then compare start row // otherwise compare based on table names int tableNameComparison = getTable().compareTo(split.getTable()); return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo( getStartRow(), split.getStartRow()); }
/** * Run MR job to test autobalance for setting number of mappers for TIF This does not run real MR * job */ protected void testAutobalanceNumOfSplit() throws IOException { // set up splits for testing List<InputSplit> splits = new ArrayList<>(5); int[] regionLen = { 10, 20, 20, 40, 60 }; for (int i = 0; i < 5; i++) { InputSplit split = new TableSplit(TABLE_NAME, new Scan(), Bytes.toBytes(i), Bytes.toBytes(i + 1), "", "", regionLen[i] * 1048576); splits.add(split); } TableInputFormat tif = new TableInputFormat(); List<InputSplit> res = tif.calculateAutoBalancedSplits(splits, 1073741824); assertEquals("Saw the wrong number of splits", 5, res.size()); TableSplit ts1 = (TableSplit) res.get(0); assertEquals("The first split end key should be", 2, Bytes.toInt(ts1.getEndRow())); TableSplit ts2 = (TableSplit) res.get(1); assertEquals("The second split regionsize should be", 20 * 1048576, ts2.getLength()); TableSplit ts3 = (TableSplit) res.get(2); assertEquals("The third split start key should be", 3, Bytes.toInt(ts3.getStartRow())); TableSplit ts4 = (TableSplit) res.get(4); assertNotEquals("The seventh split start key should not be", 4, Bytes.toInt(ts4.getStartRow())); } }
/** * TODO: use TableSnapshotRegionSplit HBASE-11555 is fixed. */ public HBaseSplit(InputSplit snapshotSplit, Path dummyPath) { super(dummyPath, 0, 0, (String[]) null); this.tableSplit = new TableSplit(); this.snapshotSplit = snapshotSplit; this.isTableSplit = false; }
/** * Create a new FijiTableSplit instance from an HBase TableSplit. * @param tableSplit the HBase TableSplit to clone. */ public FijiTableSplit(TableSplit tableSplit) { super(tableSplit.getTableName(), tableSplit.getStartRow(), tableSplit.getEndRow(), tableSplit.getRegionLocation()); }
LOG.info("Input split length: " + StringUtils.humanReadableInt(tSplit.getLength()) + " bytes."); final TableRecordReader trr = this.tableRecordReader != null ? this.tableRecordReader : new TableRecordReader(); Scan sc = new Scan(this.scan); sc.setStartRow(tSplit.getStartRow()); sc.setStopRow(tSplit.getEndRow()); trr.setScan(sc); trr.setTable(getTable());
/** {@inheritDoc} */ @Override protected void setup(Context context) throws IOException { try { super.setup(context); } catch (InterruptedException ie) { throw new IOException(ie); } if (context.getInputSplit() instanceof TableSplit) { TableSplit taskSplit = (TableSplit) context.getInputSplit(); LOG.info("Setting up map task on region [{} -- {}]", toHex(taskSplit.getStartRow()), toHex(taskSplit.getEndRow())); } }
@Override public RecordReader<ImmutableBytesWritable, Result> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { TableSplit tSplit = (TableSplit) split; if (tSplit.getTableName() == null) { throw new IOException("Cannot create a record reader because of a" + " previous error. Please look at the previous logs lines from" + " the task's full log for more details."); } ThemisTableRecordReader trr = this.themisTableRecordReader; // if no table record reader was provided use default if (trr == null) { trr = new ThemisTableRecordReader(); } Scan sc = tSplit.getScan(); sc.setStartRow(tSplit.getStartRow()); sc.setStopRow(tSplit.getEndRow()); trr.setScan(sc); trr.setConf(context.getConfiguration()); trr.setTableName(tSplit.getTableName()); trr.initialize(split, context); return trr; } }
/** * Length of region need to be properly serialized. * */ @Test public void testLengthIsSerialized() throws Exception { TableSplit split1 = new TableSplit(TableName.valueOf(name.getMethodName()), "row-start".getBytes(), "row-end".getBytes(), "location", 666); TableSplit deserialized = new TableSplit(TableName.valueOf(name.getMethodName()), "row-start2".getBytes(), "row-end2".getBytes(), "location1"); ReflectionUtils.copy(new Configuration(), split1, deserialized); Assert.assertEquals(666, deserialized.getLength()); }
@Test public void testHashCode() { TableSplit split1 = new TableSplit(TableName.valueOf(name.getMethodName()), "row-start".getBytes(), "row-end".getBytes(), "location"); TableSplit split2 = new TableSplit(TableName.valueOf(name.getMethodName()), "row-start".getBytes(), "row-end".getBytes(), "location"); assertEquals (split1, split2); assertTrue (split1.hashCode() == split2.hashCode()); HashSet<TableSplit> set = new HashSet<>(2); set.add(split1); set.add(split2); assertTrue(set.size() == 1); }
@Override protected void setup(Context context) throws IOException, InterruptedException { targetBatchSize = context.getConfiguration() .getLong(HASH_BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE); hasher = new ResultHasher(); TableSplit split = (TableSplit) context.getInputSplit(); hasher.startBatch(new ImmutableBytesWritable(split.getStartRow())); }
.getEndKey(); } else { endRow = ((TableSplit) tableSplit).getEndRow();
reader = new GraphRecordReader(); Scan sc = tSplit.getScan(); log.debug("SCAN: " + sc.toString()); if (sc.getFilter() != null) { sc.setStartRow(tSplit.getStartRow()); sc.setStopRow(tSplit.getEndRow()); reader.setScan(sc); reader.setTable(table);
@Test public void testToString() { TableSplit split = new TableSplit(TableName.valueOf(name.getMethodName()), "row-start".getBytes(), "row-end".getBytes(), "location"); String str = + "end row: row-end, region location: location, " + "encoded region name: )"; Assert.assertEquals(str, split.toString()); new TableSplit(TableName.valueOf(name.getMethodName()), null, "row-start".getBytes(), "row-end".getBytes(), "location", "encoded-region-name", 1000L); str = + "end row: row-end, region location: location, " + "encoded region name: encoded-region-name)"; Assert.assertEquals(str, split.toString()); split = new TableSplit((TableName) null, null, null, null); str = "HBase table split(table name: null, scan: , start row: null, " + "end row: null, region location: null, " + "encoded region name: )"; Assert.assertEquals(str, split.toString()); split = new TableSplit((TableName) null, null, null, null, null, null, 1000L); str = "HBase table split(table name: null, scan: , start row: null, " + "end row: null, region location: null, "
tableSplit = new TableSplit( tableSplit.getTableName(), startRow, stopRow, tableSplit.getRegionLocation());
/** * Compares this split against the given one. * * @param split The split to compare to. * @return The result of the comparison. * @see java.lang.Comparable#compareTo(java.lang.Object) */ @Override public int compareTo(TableSplit split) { // If The table name of the two splits is the same then compare start row // otherwise compare based on table names int tableNameComparison = Bytes.compareTo(getTableName(), split.getTableName()); return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo( getStartRow(), split.getStartRow()); }
@Override public long getLength() { long val = 0; try { val = isTableSplit ? tableSplit.getLength() : snapshotSplit.getLength(); } finally { return val; } }
@Override public void setup(Context context) { tableName = ((TableSplit)context.getInputSplit()).getTableName(); }
@Override public List<InputSplit> getSplits(JobContext context) throws IOException { List<InputSplit> source = getAggregatedSplits(context); if (!partitionByRegionServer) { return source; } // Partition by regionserver Multimap<String, TableSplit> partitioned = ArrayListMultimap.<String, TableSplit>create(); for (InputSplit split : source) { TableSplit cast = (TableSplit) split; String rs = cast.getRegionLocation(); partitioned.put(rs, cast); }