while (sourceCell != null || targetCell != null) { int cellKeyComparison = compareCellKeysWithinRow(sourceCell, targetCell); if (cellKeyComparison < 0) { if (LOG.isDebugEnabled()) {
while(nextSourceRow != null || nextTargetRow != null) { boolean rowMatched; int rowComparison = compareRowKeys(nextSourceRow, nextTargetRow); if (rowComparison < 0) { if (LOG.isInfoEnabled()) { rowMatched = syncRowCells(context, nextSourceRow, sourceCells, EMPTY_CELL_SCANNER); rowMatched = syncRowCells(context, nextTargetRow, EMPTY_CELL_SCANNER, targetCells); rowMatched = syncRowCells(context, nextSourceRow, sourceCells, targetCells); nextSourceRow = sourceCells.nextRow(); nextTargetRow = targetCells.nextRow();
@Override protected void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); sourceHashDir = new Path(conf.get(SOURCE_HASH_DIR_CONF_KEY)); sourceConnection = openConnection(conf, SOURCE_ZK_CLUSTER_CONF_KEY, null); targetConnection = openConnection(conf, TARGET_ZK_CLUSTER_CONF_KEY, TableOutputFormat.OUTPUT_CONF_PREFIX); sourceTable = openTable(sourceConnection, conf, SOURCE_TABLE_CONF_KEY); targetTable = openTable(targetConnection, conf, TARGET_TABLE_CONF_KEY); dryRun = conf.getBoolean(DRY_RUN_CONF_KEY, false); doDeletes = conf.getBoolean(DO_DELETES_CONF_KEY, true); doPuts = conf.getBoolean(DO_PUTS_CONF_KEY, true); sourceTableHash = HashTable.TableHash.read(conf, sourceHashDir); LOG.info("Read source hash manifest: " + sourceTableHash); LOG.info("Read " + sourceTableHash.partitions.size() + " partition keys"); TableSplit split = (TableSplit) context.getInputSplit(); ImmutableBytesWritable splitStartKey = new ImmutableBytesWritable(split.getStartRow()); sourceHashReader = sourceTableHash.newReader(conf, splitStartKey); findNextKeyHashPair(); // create a hasher, but don't start it right away // instead, find the first hash batch at or after the start row // and skip any rows that come before. they will be caught by the previous task targetHasher = new HashTable.ResultHasher(); }
while (sourceCell != null || targetCell != null) { int cellKeyComparison = compareCellKeysWithinRow(sourceCell, targetCell); if (cellKeyComparison < 0) { if (LOG.isDebugEnabled()) {
while (sourceCell != null || targetCell != null) { int cellKeyComparison = compareCellKeysWithinRow(sourceCell, targetCell); if (cellKeyComparison < 0) { if (LOG.isDebugEnabled()) {
while (sourceCell != null || targetCell != null) { int cellKeyComparison = compareCellKeysWithinRow(sourceCell, targetCell); if (cellKeyComparison < 0) { if (LOG.isDebugEnabled()) {
while(nextSourceRow != null || nextTargetRow != null) { boolean rowMatched; int rowComparison = compareRowKeys(nextSourceRow, nextTargetRow); if (rowComparison < 0) { if (LOG.isInfoEnabled()) { rowMatched = syncRowCells(context, nextSourceRow, sourceCells, EMPTY_CELL_SCANNER); rowMatched = syncRowCells(context, nextTargetRow, EMPTY_CELL_SCANNER, targetCells); rowMatched = syncRowCells(context, nextSourceRow, sourceCells, targetCells); nextSourceRow = sourceCells.nextRow(); nextTargetRow = targetCells.nextRow();
while(nextSourceRow != null || nextTargetRow != null) { boolean rowMatched; int rowComparison = compareRowKeys(nextSourceRow, nextTargetRow); if (rowComparison < 0) { if (LOG.isInfoEnabled()) { rowMatched = syncRowCells(context, nextSourceRow, sourceCells, EMPTY_CELL_SCANNER); rowMatched = syncRowCells(context, nextTargetRow, EMPTY_CELL_SCANNER, targetCells); rowMatched = syncRowCells(context, nextSourceRow, sourceCells, targetCells); nextSourceRow = sourceCells.nextRow(); nextTargetRow = targetCells.nextRow();
while(nextSourceRow != null || nextTargetRow != null) { boolean rowMatched; int rowComparison = compareRowKeys(nextSourceRow, nextTargetRow); if (rowComparison < 0) { if (LOG.isInfoEnabled()) { rowMatched = syncRowCells(context, nextSourceRow, sourceCells, EMPTY_CELL_SCANNER); rowMatched = syncRowCells(context, nextTargetRow, EMPTY_CELL_SCANNER, targetCells); rowMatched = syncRowCells(context, nextSourceRow, sourceCells, targetCells); nextSourceRow = sourceCells.nextRow(); nextTargetRow = targetCells.nextRow();
moveToNextBatch(context); finishBatchAndCompareHashes(context);
moveToNextBatch(context); finishBatchAndCompareHashes(context);
moveToNextBatch(context); finishBatchAndCompareHashes(context);
/** * Finish the currently open hash batch. * Compare the target hash to the given source hash. * If they do not match, then sync the covered key range. */ private void finishBatchAndCompareHashes(Context context) throws IOException, InterruptedException { targetHasher.finishBatch(); context.getCounter(Counter.BATCHES).increment(1); if (targetHasher.getBatchSize() == 0) { context.getCounter(Counter.EMPTY_BATCHES).increment(1); } ImmutableBytesWritable targetHash = targetHasher.getBatchHash(); if (targetHash.equals(currentSourceHash)) { context.getCounter(Counter.HASHES_MATCHED).increment(1); } else { context.getCounter(Counter.HASHES_NOT_MATCHED).increment(1); ImmutableBytesWritable stopRow = nextSourceKey == null ? new ImmutableBytesWritable(sourceTableHash.stopRow) : nextSourceKey; if (LOG.isDebugEnabled()) { LOG.debug("Hash mismatch. Key range: " + toHex(targetHasher.getBatchStartKey()) + " to " + toHex(stopRow) + " sourceHash: " + toHex(currentSourceHash) + " targetHash: " + toHex(targetHash)); } syncRange(context, targetHasher.getBatchStartKey(), stopRow); } } private static String toHex(ImmutableBytesWritable bytes) {
@Override protected void cleanup(Context context) throws IOException, InterruptedException { if (mapperException == null) { try { finishRemainingHashRanges(context); } catch (Throwable t) { mapperException = t; } } try { sourceTable.close(); targetTable.close(); sourceConnection.close(); targetConnection.close(); } catch (Throwable t) { if (mapperException == null) { mapperException = t; } else { LOG.error("Suppressing exception from closing tables", t); } } // propagate first exception if (mapperException != null) { Throwables.propagateIfInstanceOf(mapperException, IOException.class); Throwables.propagateIfInstanceOf(mapperException, InterruptedException.class); Throwables.propagate(mapperException); } }
/** * Finish the currently open hash batch. * Compare the target hash to the given source hash. * If they do not match, then sync the covered key range. */ private void finishBatchAndCompareHashes(Context context) throws IOException, InterruptedException { targetHasher.finishBatch(); context.getCounter(Counter.BATCHES).increment(1); if (targetHasher.getBatchSize() == 0) { context.getCounter(Counter.EMPTY_BATCHES).increment(1); } ImmutableBytesWritable targetHash = targetHasher.getBatchHash(); if (targetHash.equals(currentSourceHash)) { context.getCounter(Counter.HASHES_MATCHED).increment(1); } else { context.getCounter(Counter.HASHES_NOT_MATCHED).increment(1); ImmutableBytesWritable stopRow = nextSourceKey == null ? new ImmutableBytesWritable(sourceTableHash.stopRow) : nextSourceKey; if (LOG.isDebugEnabled()) { LOG.debug("Hash mismatch. Key range: " + toHex(targetHasher.getBatchStartKey()) + " to " + toHex(stopRow) + " sourceHash: " + toHex(currentSourceHash) + " targetHash: " + toHex(targetHash)); } syncRange(context, targetHasher.getBatchStartKey(), stopRow); } } private static String toHex(ImmutableBytesWritable bytes) {
if (mapperException == null) { try { finishRemainingHashRanges(context); } catch (Throwable t) { mapperException = t;
@Override protected void cleanup(Context context) throws IOException, InterruptedException { if (mapperException == null) { try { finishRemainingHashRanges(context); } catch (Throwable t) { mapperException = t; } } try { sourceTable.close(); targetTable.close(); sourceConnection.close(); targetConnection.close(); } catch (Throwable t) { if (mapperException == null) { mapperException = t; } else { LOG.error("Suppressing exception from closing tables", t); } } // propagate first exception if (mapperException != null) { Throwables.propagateIfInstanceOf(mapperException, IOException.class); Throwables.propagateIfInstanceOf(mapperException, InterruptedException.class); Throwables.propagate(mapperException); } }
@Override protected void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); sourceHashDir = new Path(conf.get(SOURCE_HASH_DIR_CONF_KEY)); sourceConnection = openConnection(conf, SOURCE_ZK_CLUSTER_CONF_KEY, null); targetConnection = openConnection(conf, TARGET_ZK_CLUSTER_CONF_KEY, TableOutputFormat.OUTPUT_CONF_PREFIX); sourceTable = openTable(sourceConnection, conf, SOURCE_TABLE_CONF_KEY); targetTable = openTable(targetConnection, conf, TARGET_TABLE_CONF_KEY); dryRun = conf.getBoolean(SOURCE_TABLE_CONF_KEY, false); sourceTableHash = HashTable.TableHash.read(conf, sourceHashDir); LOG.info("Read source hash manifest: " + sourceTableHash); LOG.info("Read " + sourceTableHash.partitions.size() + " partition keys"); TableSplit split = (TableSplit) context.getInputSplit(); ImmutableBytesWritable splitStartKey = new ImmutableBytesWritable(split.getStartRow()); sourceHashReader = sourceTableHash.newReader(conf, splitStartKey); findNextKeyHashPair(); // create a hasher, but don't start it right away // instead, find the first hash batch at or after the start row // and skip any rows that come before. they will be caught by the previous task targetHasher = new HashTable.ResultHasher(); }
@Override protected void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); sourceHashDir = new Path(conf.get(SOURCE_HASH_DIR_CONF_KEY)); sourceConnection = openConnection(conf, SOURCE_ZK_CLUSTER_CONF_KEY, null); targetConnection = openConnection(conf, TARGET_ZK_CLUSTER_CONF_KEY, TableOutputFormat.OUTPUT_CONF_PREFIX); sourceTable = openTable(sourceConnection, conf, SOURCE_TABLE_CONF_KEY); targetTable = openTable(targetConnection, conf, TARGET_TABLE_CONF_KEY); dryRun = conf.getBoolean(SOURCE_TABLE_CONF_KEY, false); sourceTableHash = HashTable.TableHash.read(conf, sourceHashDir); LOG.info("Read source hash manifest: " + sourceTableHash); LOG.info("Read " + sourceTableHash.partitions.size() + " partition keys"); TableSplit split = (TableSplit) context.getInputSplit(); ImmutableBytesWritable splitStartKey = new ImmutableBytesWritable(split.getStartRow()); sourceHashReader = sourceTableHash.newReader(conf, splitStartKey); findNextKeyHashPair(); // create a hasher, but don't start it right away // instead, find the first hash batch at or after the start row // and skip any rows that come before. they will be caught by the previous task targetHasher = new HashTable.ResultHasher(); }