/** * Constructor * @param splitRow This is row we are splitting around. * @param fr */ Reference(final byte [] splitRow, final Range fr) { this.splitkey = splitRow == null? null: KeyValueUtil.createFirstOnRow(splitRow).getKey(); this.region = fr; }
currentKey = currentKV.getKey(); if (previousKey != null) { for (int i = 0; i < previousKey.length && i < currentKey.length &&
@Override public void doReduce(RowKeyWritable row, Iterable<KeyValue> kvs, Context context) throws java.io.IOException, InterruptedException { for (KeyValue kv : kvs) { immutableBytesWritable.set(kv.getKey()); context.write(immutableBytesWritable, kv); } } }
@Override public Tuple2<ImmutableBytesWritable, KeyValue> call( Tuple2<RowKeyWritable, KeyValue> rowKeyWritableKeyValueTuple2) throws Exception { return new Tuple2<>(new ImmutableBytesWritable(rowKeyWritableKeyValueTuple2._2.getKey()), rowKeyWritableKeyValueTuple2._2); } }).saveAsNewAPIHadoopDataset(job.getConfiguration());
@Override public Tuple2<RowKeyWritable, KeyValue> call(Tuple2<Text, Text> textTextTuple2) throws Exception { KeyValue outputValue = keyValueCreators.get(0).create(textTextTuple2._1, textTextTuple2._2.getBytes(), 0, textTextTuple2._2.getLength()); return new Tuple2<>(new RowKeyWritable(outputValue.createKeyOnly(false).getKey()), outputValue); } });
@Override public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException { KeyValue outputValue; int n = keyValueCreators.size(); if (n == 1 && keyValueCreators.get(0).isFullCopy) { // shortcut for simple full copy outputValue = keyValueCreators.get(0).create(key, value.getBytes(), 0, value.getLength()); rowKeyWritable.set(outputValue.createKeyOnly(false).getKey()); context.write(rowKeyWritable, outputValue); } else { // normal (complex) case that distributes measures to multiple HBase columns inputCodec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), inputMeasures); for (int i = 0; i < n; i++) { outputValue = keyValueCreators.get(i).create(key, inputMeasures); rowKeyWritable.set(outputValue.createKeyOnly(false).getKey()); context.write(rowKeyWritable, outputValue); } } }
@Override public Iterator<Tuple2<RowKeyWritable, KeyValue>> call(Tuple2<Text, Text> textTextTuple2) throws Exception { List<Tuple2<RowKeyWritable, KeyValue>> result = Lists.newArrayListWithExpectedSize(cfNum); Object[] inputMeasures = new Object[cubeDesc.getMeasures().size()]; inputCodec.decode(ByteBuffer.wrap(textTextTuple2._2.getBytes(), 0, textTextTuple2._2.getLength()), inputMeasures); for (int i = 0; i < cfNum; i++) { KeyValue outputValue = keyValueCreators.get(i).create(textTextTuple2._1, inputMeasures); result.add(new Tuple2<>(new RowKeyWritable(outputValue.createKeyOnly(false).getKey()), outputValue)); } return result.iterator(); } });
private byte[] getSomeKey(int rowId) { KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(), Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put); return kv.getKey(); }
new RowKeyWritable(KeyValueUtil.createFirstOnRow(splits.get(i), 9223372036854775807L).createKeyOnly(false).getKey()), NullWritable.get());
Type.Maximum).getKey();
private int readAndCheckbytes(HFileScanner scanner, int start, int n) throws IOException { String value = "value"; int i = start; for (; i < (start + n); i++) { ByteBuffer key = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); ByteBuffer val = scanner.getValue(); String keyStr = String.format(localFormatter, Integer.valueOf(i)); String valStr = value + keyStr; KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"), Bytes.toBytes("qual"), Bytes.toBytes(valStr)); byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, Bytes.toBytes(key).length).getKey(); assertTrue("bytes for keys do not match " + keyStr + " " + Bytes.toString(Bytes.toBytes(key)), Arrays.equals(kv.getKey(), keyBytes)); byte [] valBytes = Bytes.toBytes(val); assertTrue("bytes for vals do not match " + valStr + " " + Bytes.toString(valBytes), Arrays.equals(Bytes.toBytes(valStr), valBytes)); if (!scanner.next()) { break; } } assertEquals(i, start + n - 1); return (start + n); }
@Test public void testCompareCellWithKey() throws Exception { KeyValue kv1 = new KeyValue(row1, fam1, qual1, val); KeyValue kv2 = new KeyValue(row2, fam1, qual1, val); assertTrue( (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) < 0); kv1 = new KeyValue(row1, fam2, qual1, val); kv2 = new KeyValue(row1, fam1, qual1, val); assertTrue( (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) > 0); kv1 = new KeyValue(row1, fam1, qual1, 1L, val); kv2 = new KeyValue(row1, fam1, qual1, 2L, val); assertTrue( (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) > 0); kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put); kv2 = new KeyValue(row1, fam1, qual1, 1L, Type.Maximum); assertTrue( (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) > 0); kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put); kv2 = new KeyValue(row1, fam1, qual1, 1L, Type.Put); assertTrue( (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) == 0); }
fs.getFileStatus(path).getLen(), new CacheConfig(conf), conf); reader.loadFileInfo(); KeySampler kSampler = new KeySampler(rng, ((KeyValue) reader.getFirstKey().get()).getKey(), ((KeyValue) reader.getLastKey().get()).getKey(), keyLenGen); HFileScanner scanner = reader.getScanner(false, USE_PREAD); BytesWritable key = new BytesWritable(); KeyValue kv = new KeyValue(k, CF, QUAL); if (scanner.seekTo(kv) >= 0) { ByteBuffer bbkey = ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()); ByteBuffer bbval = scanner.getValue(); totalBytes += bbkey.limit();
scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))) == 0); ByteBuffer readKey = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), Bytes.toBytes(readKey)));
byte[] k = new KeyValue(RandomKeyValueUtil.randomOrderedKey(rand, i * 16 + j), family, qualifier, EnvironmentEdgeManager.currentTime(), KeyValue.Type.Put).getKey(); keys.add(k); if (j == 8) {
@Test public void testCreateKey() { byte[] row = "myRow".getBytes(); byte[] qualifier = "myQualifier".getBytes(); // Mimic what Storefile.createBloomKeyValue() does byte[] rowKey = KeyValueUtil.createFirstOnRow(row, 0, row.length, new byte[0], 0, 0, row, 0, 0).getKey(); byte[] rowColKey = KeyValueUtil.createFirstOnRow(row, 0, row.length, new byte[0], 0, 0, qualifier, 0, qualifier.length).getKey(); KeyValue rowKV = KeyValueUtil.createKeyValueFromKey(rowKey); KeyValue rowColKV = KeyValueUtil.createKeyValueFromKey(rowColKey); assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp()); assertEquals(Bytes.toStringBinary(rowKV.getRowArray(), rowKV.getRowOffset(), rowKV.getRowLength()), Bytes.toStringBinary(rowColKV.getRowArray(), rowColKV.getRowOffset(), rowColKV.getRowLength())); assertEquals(0, rowKV.getQualifierLength()); } }
private void checkSeekingConsistency(List<DataBlockEncoder.EncodedSeeker> encodedSeekers, boolean seekBefore, Cell keyValue) { Cell expectedKeyValue = null; ByteBuffer expectedKey = null; ByteBuffer expectedValue = null; for (DataBlockEncoder.EncodedSeeker seeker : encodedSeekers) { seeker.seekToKeyInBlock(keyValue, seekBefore); seeker.rewind(); Cell actualKeyValue = seeker.getCell(); ByteBuffer actualKey = null; actualKey = ByteBuffer.wrap(((KeyValue) seeker.getKey()).getKey()); ByteBuffer actualValue = seeker.getValueShallowCopy(); if (expectedKeyValue != null) { assertTrue(CellUtil.equals(expectedKeyValue, actualKeyValue)); } else { expectedKeyValue = actualKeyValue; } if (expectedKey != null) { assertEquals(expectedKey, actualKey); } else { expectedKey = actualKey; } if (expectedValue != null) { assertEquals(expectedValue, actualValue); } else { expectedValue = actualValue; } } }
new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTime(), RandomKeyValueUtil.randomValue(rand)); byte[] k = kv.getKey(); writer.append(kv); keys[i] = k; reader.getTrailer().getNumDataIndexLevels()); assertTrue(Bytes.equals(keys[0], ((KeyValue)reader.getFirstKey().get()).getKey())); assertTrue(Bytes.equals(keys[NUM_KV - 1], ((KeyValue)reader.getLastKey().get()).getKey())); LOG.info("Last key: " + Bytes.toStringBinary(keys[NUM_KV - 1])); checkSeekTo(keys, scanner, i); checkKeyValue("i=" + i, keys[i], values[i], ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()), scanner.getValue()); checkSeekTo(keys, scanner, i); checkKeyValue("i=" + i, keys[i], values[i], ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()), scanner.getValue());
OnheapDecodedCell c1 = new OnheapDecodedCell(kv1.getKey(), kv1.getRowLength(), kv1.getFamilyOffset() - KeyValue.ROW_OFFSET, kv1.getFamilyLength(), kv1.getQualifierOffset() - KeyValue.ROW_OFFSET, kv1.getQualifierLength(), KeyValue kv2 = new KeyValue(Bytes.toBytes("r2"), Bytes.toBytes("f"), Bytes.toBytes("2"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("2")); OnheapDecodedCell c2 = new OnheapDecodedCell(kv2.getKey(), kv2.getRowLength(), kv2.getFamilyOffset() - KeyValue.ROW_OFFSET, kv2.getFamilyLength(), kv2.getQualifierOffset() - KeyValue.ROW_OFFSET, kv2.getQualifierLength(), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("3")); BufferedDataBlockEncoder.OffheapDecodedExtendedCell c3 = new BufferedDataBlockEncoder.OffheapDecodedExtendedCell(ByteBuffer.wrap(kv2.getKey()), kv2.getRowLength(), kv2.getFamilyOffset() - KeyValue.ROW_OFFSET, kv2.getFamilyLength(), kv2.getQualifierOffset() - KeyValue.ROW_OFFSET, kv2.getQualifierLength(),
Cell kv = null; for (boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) { ByteBuffer bb = ByteBuffer.wrap(((KeyValue) s.getKey()).getKey()); kv = KeyValueUtil.createKeyValueFromKey(bb); if (first) {