byte [] row = Bytes.toBytes(values[0]); byte [] family = Bytes.toBytes(values[1]); byte [] qualifier = Bytes.toBytes(values[2]); byte [] value = Bytes.toBytes(values[3]); Put put = new Put(row); put.addColumn(family, qualifier, value); context.write(new ImmutableBytesWritable(row), put); } catch (InterruptedException e) { LOG.error("Interrupted emitting put", e); context.setStatus("Emitting Put " + count);
@Override public void map(ImmutableBytesWritable key, Result result, Context context) throws IOException { List<Long> tsList = new ArrayList<>(); for (Cell kv : result.listCells()) { tsList.add(kv.getTimestamp()); } List<Put> puts = new ArrayList<>(); for (Long ts : tsList) { Put put = new Put(key.get()); put.setDurability(Durability.SKIP_WAL); put.addColumn(FAMILY_NAME, COLUMN_NAME, ts, Bytes.toBytes(true)); puts.add(put); } table.put(puts); }
if (key == null) key = new ImmutableBytesWritable(); if (value == null) value = new Result(); try { try { value = this.scanner.next(); if (value != null && value.isStale()) numStale++; if (logScannerActivity) { rowcount ++; if (value != null && value.isStale()) numStale++; numRestarts++; if (value != null && value.size() > 0) { key.set(value.getRow()); lastSuccessfulRow = key.get(); return true; if (value != null && value.isCursor()) { key.set(value.getCursor().getRow()); lastSuccessfulRow = key.get(); return true; LOG.info(ioe.toString(), ioe); String lastRow = lastSuccessfulRow == null ? "null" : Bytes.toStringBinary(lastSuccessfulRow); LOG.info("lastSuccessfulRow=" + lastRow);
public boolean nextKeyValue() throws IOException { result = scanner.next(); if (result == null) { //we are done return false; } if (this.row == null) { this.row = new ImmutableBytesWritable(); } this.row.set(result.getRow()); return true; }
private static void testExpression(LiteralExpression literal, double expected) throws SQLException { List<Expression> expressions = Lists.newArrayList((Expression) literal); Expression cbrtFunction = new CbrtFunction(expressions); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); cbrtFunction.evaluate(null, ptr); Double result = (Double) cbrtFunction.getDataType().toObject(ptr, cbrtFunction.getSortOrder()); assertTrue(Math.abs(result.doubleValue() - expected) <= 1e-9); }
private Object executeFunction(Expression inputArg) throws SQLException { ExternalSqlTypeIdFunction externalIdFunction = new ExternalSqlTypeIdFunction(Lists.newArrayList(inputArg)); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); assertTrue(externalIdFunction.evaluate(null, ptr)); return PInteger.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength(), PInteger.INSTANCE, inputArg.getSortOrder()); } }
/** * Implements mapper logic for use across APIs. */ protected static Put map(ImmutableBytesWritable key, Result value) throws IOException { if (value.size() != 1) { throw new IOException("There should only be one input column"); } Map<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> cf = value.getMap(); if(!cf.containsKey(INPUT_FAMILY)) { throw new IOException("Wrong input columns. Missing: '" + Bytes.toString(INPUT_FAMILY) + "'."); } // Get the original value and reverse it String originalValue = Bytes.toString(value.getValue(INPUT_FAMILY, INPUT_FAMILY)); StringBuilder newValue = new StringBuilder(originalValue); newValue.reverse(); // Now set the value to be collected Put outval = new Put(key.get()); outval.addColumn(OUTPUT_FAMILY, null, Bytes.toBytes(newValue.toString())); return outval; }
@Override public void map(ImmutableBytesWritable row, Result result, OutputCollector<ImmutableBytesWritable, Put> outCollector, Reporter reporter) throws IOException { String rowKey = Bytes.toString(result.getRow()); final ImmutableBytesWritable pKey = new ImmutableBytesWritable( Bytes.toBytes(PRESIDENT_PATTERN)); final ImmutableBytesWritable aKey = new ImmutableBytesWritable( Bytes.toBytes(ACTOR_PATTERN)); ImmutableBytesWritable outKey = null; if (rowKey.startsWith(PRESIDENT_PATTERN)) { outKey = pKey; } else if (rowKey.startsWith(ACTOR_PATTERN)) { outKey = aKey; } else { throw new AssertionError("unexpected rowKey"); } String name = Bytes.toString(result.getValue(COLUMN_FAMILY, COLUMN_QUALIFIER)); outCollector.collect(outKey, new Put(Bytes.toBytes("rowKey2")) .addColumn(COLUMN_FAMILY, COLUMN_QUALIFIER, Bytes.toBytes(name))); } }
public void map(ImmutableBytesWritable row, Result columns, Context context) throws IOException { context.getCounter(Counters.ROWS).increment(1); String value = null; try { Put put = new Put(row.get()); for (Cell cell : columns.listCells()) { context.getCounter(Counters.COLS).increment(1); value = Bytes.toStringBinary(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); JSONObject json = (JSONObject) parser.parse(value); for (Object key : json.keySet()) { Object val = json.get(key); put.addColumn(columnFamily, Bytes.toBytes(key.toString()), Bytes.toBytes(val.toString())); context.write(row, put); context.getCounter(Counters.VALID).increment(1); } catch (Exception e) { e.printStackTrace(); System.err.println("Error: " + e.getMessage() + ", Row: " + Bytes.toStringBinary(row.get()) + ", JSON: " + value); context.getCounter(Counters.ERROR).increment(1);
assertEquals(numHashFiles - 1, tableHash.partitions.size()); for (ImmutableBytesWritable bytes : tableHash.partitions) { LOG.debug("partition: " + Bytes.toInt(bytes.get())); .put(-1, new ImmutableBytesWritable(Bytes.fromHex("714cb10a9e3b5569852980edd8c6ca2f"))) .put(5, new ImmutableBytesWritable(Bytes.fromHex("28d961d9252ce8f8d44a07b38d3e1d96"))) .put(10, new ImmutableBytesWritable(Bytes.fromHex("f6bbc4a224d8fd929b783a92599eaffa"))) .put(15, new ImmutableBytesWritable(Bytes.fromHex("522deb5d97f73a414ecc11457be46881"))) .put(20, new ImmutableBytesWritable(Bytes.fromHex("b026f2611aaa46f7110116d807545352"))) .put(25, new ImmutableBytesWritable(Bytes.fromHex("39ffc1a3094aa12a2e90ffd9cef2ce93"))) .put(30, new ImmutableBytesWritable(Bytes.fromHex("f6b4d75727ce9a30ac29e4f08f601666"))) .put(35, new ImmutableBytesWritable(Bytes.fromHex("422e2d2f1eb79a8f02171a705a42c090"))) .put(40, new ImmutableBytesWritable(Bytes.fromHex("559ad61c900fffefea0a15abf8a97bc3"))) .put(45, new ImmutableBytesWritable(Bytes.fromHex("23019084513eca41cee436b2a29611cb"))) .put(50, new ImmutableBytesWritable(Bytes.fromHex("b40467d222ddb4949b142fe145ee9edc"))) .put(55, new ImmutableBytesWritable(Bytes.fromHex("372bf89fcd8ca4b7ab3c1add9d07f7e4"))) .put(60, new ImmutableBytesWritable(Bytes.fromHex("69ae0585e6255de27dce974e332b8f8b"))) .put(65, new ImmutableBytesWritable(Bytes.fromHex("8029610044297aad0abdbecd485d8e59"))) .put(70, new ImmutableBytesWritable(Bytes.fromHex("de5f784f7f78987b6e57ecfd81c8646f"))) .put(75, new ImmutableBytesWritable(Bytes.fromHex("1cd757cc4e1715c8c3b1c24447a1ec56"))) .put(80, new ImmutableBytesWritable(Bytes.fromHex("f9a53aacfeb6142b08066615e7038095"))) .put(85, new ImmutableBytesWritable(Bytes.fromHex("89b872b7e639df32d3276b33928c0c91"))) .put(90, new ImmutableBytesWritable(Bytes.fromHex("45eeac0646d46a474ea0484175faed38"))) .put(95, new ImmutableBytesWritable(Bytes.fromHex("f57c447e32a08f4bf1abb2892839ac56"))) .build(); ImmutableBytesWritable key = new ImmutableBytesWritable(); ImmutableBytesWritable hash = new ImmutableBytesWritable(); while(reader.next(key, hash)) {
public void map(LongWritable offset, Text value, Context context) throws IOException { byte[] family = Bytes.toBytes("FAM"); final byte[][] qualifiers = { Bytes.toBytes("A"), Bytes.toBytes("B") }; new ImmutableBytesWritable(Bytes.toBytes(valueTokens[0])); Put put = new Put(rowKey.copyBytes()); put.setDurability(Durability.SKIP_WAL); String newValue = prefix + Integer.parseInt(suffix) * 3; KeyValue kv = new KeyValue(rowKey.copyBytes(), family, qualifiers[i-1], Bytes.toBytes(newValue)); put.add(kv); context.write(rowKey, put); } catch (InterruptedException e) { e.printStackTrace();
public void makeAssertions(ImmutableBytesWritable key, Result value) throws IOException { if (value.size() != 1) { throw new IOException("There should only be one input column"); } Map<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> cf = value.getMap(); if (!cf.containsKey(INPUT_FAMILY)) { throw new IOException("Wrong input columns. Missing: '" + Bytes.toString(INPUT_FAMILY) + "'."); } String val = Bytes.toStringBinary(value.getValue(INPUT_FAMILY, null)); LOG.debug("map: key -> " + Bytes.toStringBinary(key.get()) + ", value -> " + val); } }
LOG.debug("Target missing cell: " + sourceCell); context.getCounter(Counter.TARGETMISSINGCELLS).increment(1); matchingRow = false; put = new Put(rowKey); put.add(sourceCell); LOG.debug("Source missing cell: " + targetCell); CellUtil.cloneQualifier(targetCell), targetCell.getTimestamp()); put = new Put(rowKey); put.add(sourceCell); if (put != null && put.size() >= sourceTableHash.scanBatch) { context.write(new ImmutableBytesWritable(rowKey), put); put = null; context.write(new ImmutableBytesWritable(rowKey), delete); delete = null; context.write(new ImmutableBytesWritable(rowKey), put); context.write(new ImmutableBytesWritable(rowKey), delete);
protected static void verifyRowFromMap(ImmutableBytesWritable key, Result result) throws IOException { byte[] row = key.get(); CellScanner scanner = result.cellScanner(); while (scanner.advance()) { Cell cell = scanner.current(); //assert that all Cells in the Result have the same key Assert.assertEquals(0, Bytes.compareTo(row, 0, row.length, cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); } for (int j = 0; j < FAMILIES.length; j++) { byte[] actual = result.getValue(FAMILIES[j], FAMILIES[j]); Assert.assertArrayEquals("Row in snapshot does not match, expected:" + Bytes.toString(row) + " ,actual:" + Bytes.toString(actual), row, actual); } }
@Override protected void map(ImmutableBytesWritable rowKey, Result result, Context context) throws IOException, InterruptedException { for(java.util.Map.Entry<byte[], ImmutableBytesWritable> index : indexes.entrySet()) { byte[] qualifier = index.getKey(); ImmutableBytesWritable tableName = index.getValue(); byte[] value = result.getValue(family, qualifier); if (value != null) { // original: row 123 attribute:phone 555-1212 // index: row 555-1212 INDEX:ROW 123 Put put = new Put(value); put.addColumn(INDEX_COLUMN, INDEX_QUALIFIER, rowKey.get()); context.write(tableName, put); } } }
lineBytes, value.getLength()); ImmutableBytesWritable rowKey = new ImmutableBytesWritable(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength()); tags.add(new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(ttl))); Put put = new Put(rowKey.copyBytes()); for (int i = 0; i < parsed.getColumnCount(); i++) { if (i == parser.getRowKeyColumnIndex() || i == parser.getTimestampKeyColumnIndex() context.write(rowKey, put); } catch (ImportTsv.TsvParser.BadTsvLineException | IllegalArgumentException | InvalidLabelException badLine) {
SecureWriter out = new SecureWriter(conf, userProvider, userToken, opts)) { scanner = cp.checkScannerOpen(scan); ImmutableBytesWritable key = new ImmutableBytesWritable(); long rowCount = 0; long cellCount = 0; if (Bytes.compareTo(firstCell.getRowArray(), firstCell.getRowOffset(), firstCell.getRowLength(), cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()) != 0) { throw new IOException("Why the RegionScanner#nextRaw returns the data of different" + " rows?? first row=" + Bytes.toHex(firstCell.getRowArray(), firstCell.getRowOffset(), firstCell.getRowLength()) + ", current row=" + Bytes.toHex(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); results.add(Result.create(cells)); cells.clear(); cp.postScannerNext(scanner, results, scan.getBatch(), hasMore); key.set(r.getRow()); out.append(key, r); ++rowCount; cellCount += r.size();
@Override protected void map(LongWritable key, LongWritable value, Context context) throws IOException, InterruptedException { long chainId = value.get(); LOG.info("Starting mapper with chainId:" + chainId); byte[] chainIdArray = Bytes.toBytes(chainId); long currentRow = 0; long chainLength = context.getConfiguration().getLong(CHAIN_LENGTH_KEY, CHAIN_LENGTH); long nextRow = getNextRow(0, chainLength); for (long i = 0; i < chainLength; i++) { byte[] rk = Bytes.toBytes(currentRow); // Next link in the chain. KeyValue linkKv = new KeyValue(rk, CHAIN_FAM, chainIdArray, Bytes.toBytes(nextRow)); // What link in the chain this is. KeyValue sortKv = new KeyValue(rk, SORT_FAM, chainIdArray, Bytes.toBytes(i)); // Added data so that large stores are created. KeyValue dataKv = new KeyValue(rk, DATA_FAM, chainIdArray, Bytes.toBytes(RandomStringUtils.randomAlphabetic(50)) ); // Emit the key values. context.write(new ImmutableBytesWritable(rk), linkKv); context.write(new ImmutableBytesWritable(rk), sortKv); context.write(new ImmutableBytesWritable(rk), dataKv); // Move to the next row. currentRow = nextRow; nextRow = getNextRow(i+1, chainLength); } }