/** * Compares this split against the given one. * * @param split The split to compare to. * @return The result of the comparison. * @see java.lang.Comparable#compareTo(java.lang.Object) */ @Override public int compareTo(TableSplit split) { // If The table name of the two splits is the same then compare start row // otherwise compare based on table names int tableNameComparison = getTable().compareTo(split.getTable()); return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo( getStartRow(), split.getStartRow()); }
@Override protected void setup(Context context) throws IOException, InterruptedException { targetBatchSize = context.getConfiguration() .getLong(HASH_BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE); hasher = new ResultHasher(); TableSplit split = (TableSplit) context.getInputSplit(); hasher.startBatch(new ImmutableBytesWritable(split.getStartRow())); }
@Override protected void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); sourceHashDir = new Path(conf.get(SOURCE_HASH_DIR_CONF_KEY)); sourceConnection = openConnection(conf, SOURCE_ZK_CLUSTER_CONF_KEY, null); targetConnection = openConnection(conf, TARGET_ZK_CLUSTER_CONF_KEY, TableOutputFormat.OUTPUT_CONF_PREFIX); sourceTable = openTable(sourceConnection, conf, SOURCE_TABLE_CONF_KEY); targetTable = openTable(targetConnection, conf, TARGET_TABLE_CONF_KEY); dryRun = conf.getBoolean(DRY_RUN_CONF_KEY, false); doDeletes = conf.getBoolean(DO_DELETES_CONF_KEY, true); doPuts = conf.getBoolean(DO_PUTS_CONF_KEY, true); sourceTableHash = HashTable.TableHash.read(conf, sourceHashDir); LOG.info("Read source hash manifest: " + sourceTableHash); LOG.info("Read " + sourceTableHash.partitions.size() + " partition keys"); TableSplit split = (TableSplit) context.getInputSplit(); ImmutableBytesWritable splitStartKey = new ImmutableBytesWritable(split.getStartRow()); sourceHashReader = sourceTableHash.newReader(conf, splitStartKey); findNextKeyHashPair(); // create a hasher, but don't start it right away // instead, find the first hash batch at or after the start row // and skip any rows that come before. they will be caught by the previous task targetHasher = new HashTable.ResultHasher(); }
byte[] splitStartKey = ts.getStartRow(); byte[] splitEndKey = ts.getEndRow(); int j = i + 1; long nextRegionSize = nextRegion.getLength(); if (totalSize + nextRegionSize <= averageRegionSize && Bytes.equals(splitEndKey, nextRegion.getStartRow())) { totalSize = totalSize + nextRegionSize; splitEndKey = nextRegion.getEndRow();
String encodedRegionName = ts.getEncodedRegionName(); long regionSize = ts.getLength(); byte[] startRow = ts.getStartRow(); byte[] endRow = ts.getEndRow();
/** * Run MR job to test autobalance for setting number of mappers for TIF This does not run real MR * job */ protected void testAutobalanceNumOfSplit() throws IOException { // set up splits for testing List<InputSplit> splits = new ArrayList<>(5); int[] regionLen = { 10, 20, 20, 40, 60 }; for (int i = 0; i < 5; i++) { InputSplit split = new TableSplit(TABLE_NAME, new Scan(), Bytes.toBytes(i), Bytes.toBytes(i + 1), "", "", regionLen[i] * 1048576); splits.add(split); } TableInputFormat tif = new TableInputFormat(); List<InputSplit> res = tif.calculateAutoBalancedSplits(splits, 1073741824); assertEquals("Saw the wrong number of splits", 5, res.size()); TableSplit ts1 = (TableSplit) res.get(0); assertEquals("The first split end key should be", 2, Bytes.toInt(ts1.getEndRow())); TableSplit ts2 = (TableSplit) res.get(1); assertEquals("The second split regionsize should be", 20 * 1048576, ts2.getLength()); TableSplit ts3 = (TableSplit) res.get(2); assertEquals("The third split start key should be", 3, Bytes.toInt(ts3.getStartRow())); TableSplit ts4 = (TableSplit) res.get(4); assertNotEquals("The seventh split start key should not be", 4, Bytes.toInt(ts4.getStartRow())); } }
this.tableRecordReader != null ? this.tableRecordReader : new TableRecordReader(); Scan sc = new Scan(this.scan); sc.setStartRow(tSplit.getStartRow()); sc.setStopRow(tSplit.getEndRow()); trr.setScan(sc);
sc.setStartRow(tSplit.getStartRow()); sc.setStopRow(tSplit.getEndRow()); trr.setScan(sc);
/** * Create a new FijiTableSplit instance from an HBase TableSplit. * @param tableSplit the HBase TableSplit to clone. * @param regionStartKey the starting key of the region associated with this split. */ public FijiTableSplit(TableSplit tableSplit, byte[] regionStartKey) { super(tableSplit.getTableName(), tableSplit.getStartRow(), tableSplit.getEndRow(), tableSplit.getRegionLocation()); checkNotNull(regionStartKey); mRegionStartKey = regionStartKey; }
private InputSplit[] convertSplits(List<org.apache.hadoop.mapreduce.InputSplit> splits) { InputSplit[] converted = new InputSplit[splits.size()]; for (int i = 0; i < splits.size(); i++) { org.apache.hadoop.hbase.mapreduce.TableSplit tableSplit = (org.apache.hadoop.hbase.mapreduce.TableSplit) splits.get(i); TableSplit newTableSplit = new TableSplit(tableSplit.getTableName(), tableSplit.getStartRow(), tableSplit.getEndRow(), tableSplit.getRegionLocation()); converted[i] = newTableSplit; } return converted; }
/** * Compares this split against the given one. * * @param split The split to compare to. * @return The result of the comparison. * @see java.lang.Comparable#compareTo(java.lang.Object) */ @Override public int compareTo(TableSplit split) { // If The table name of the two splits is the same then compare start row // otherwise compare based on table names int tableNameComparison = getTable().compareTo(split.getTable()); return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo( getStartRow(), split.getStartRow()); }
/** * Compares this split against the given one. * * @param split The split to compare to. * @return The result of the comparison. * @see java.lang.Comparable#compareTo(java.lang.Object) */ @Override public int compareTo(TableSplit split) { // If The table name of the two splits is the same then compare start row // otherwise compare based on table names int tableNameComparison = getTable().compareTo(split.getTable()); return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo( getStartRow(), split.getStartRow()); }
@Override protected void setup(Context context) throws IOException, InterruptedException { targetBatchSize = context.getConfiguration() .getLong(HASH_BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE); hasher = new ResultHasher(); TableSplit split = (TableSplit) context.getInputSplit(); hasher.startBatch(new ImmutableBytesWritable(split.getStartRow())); }
/** * Compares this split against the given one. * * @param split The split to compare to. * @return The result of the comparison. * @see java.lang.Comparable#compareTo(java.lang.Object) */ @Override public int compareTo(TableSplit split) { // If The table name of the two splits is the same then compare start row // otherwise compare based on table names int tableNameComparison = getTable().compareTo(split.getTable()); return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo( getStartRow(), split.getStartRow()); }
/** * Compares this split against the given one. * * @param split The split to compare to. * @return The result of the comparison. * @see java.lang.Comparable#compareTo(java.lang.Object) */ @Override public int compareTo(TableSplit split) { // If The table name of the two splits is the same then compare start row // otherwise compare based on table names int tableNameComparison = Bytes.compareTo(getTableName(), split.getTableName()); return tableNameComparison != 0 ? tableNameComparison : Bytes.compareTo( getStartRow(), split.getStartRow()); }
@Override protected void setup(Context context) throws IOException, InterruptedException { targetBatchSize = context.getConfiguration() .getLong(HASH_BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE); hasher = new ResultHasher(); TableSplit split = (TableSplit) context.getInputSplit(); hasher.startBatch(new ImmutableBytesWritable(split.getStartRow())); }
@Override protected void setup(Context context) throws IOException, InterruptedException { targetBatchSize = context.getConfiguration() .getLong(HASH_BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE); hasher = new ResultHasher(); TableSplit split = (TableSplit) context.getInputSplit(); hasher.startBatch(new ImmutableBytesWritable(split.getStartRow())); }
/** * Create a new FijiTableSplit instance from an HBase TableSplit. * @param tableSplit the HBase TableSplit to clone. */ public FijiTableSplit(TableSplit tableSplit) { super(tableSplit.getTableName(), tableSplit.getStartRow(), tableSplit.getEndRow(), tableSplit.getRegionLocation()); }
/** {@inheritDoc} */ @Override protected void setup(Context context) throws IOException { try { super.setup(context); } catch (InterruptedException ie) { throw new IOException(ie); } if (context.getInputSplit() instanceof TableSplit) { TableSplit taskSplit = (TableSplit) context.getInputSplit(); LOG.info("Setting up map task on region [{} -- {}]", toHex(taskSplit.getStartRow()), toHex(taskSplit.getEndRow())); } }
/** {@inheritDoc} */ @Override protected void cleanup(Context context) throws IOException { if (context.getInputSplit() instanceof TableSplit) { TableSplit taskSplit = (TableSplit) context.getInputSplit(); LOG.info("Cleaning up task on region [{} -- {}]", toHex(taskSplit.getStartRow()), toHex(taskSplit.getEndRow())); } try { super.cleanup(context); } catch (InterruptedException ie) { throw new IOException(ie); } }