protected static String getTimestampString(final KeyValue kv) { return kv.getTimestamp() + ""; }
/** * * @return Timestamp */ @Override public long getTimestamp() { return getTimestamp(getKeyLength()); }
@Override public long getTimestamp() { return this.kv.getTimestamp(); }
/********************* toString ************************************/ public static String toStringWithPadding(final Collection<? extends KeyValue> kvs, final boolean includeMeta) { int maxRowStringLength = 0; int maxFamilyStringLength = 0; int maxQualifierStringLength = 0; int maxTimestampLength = 0; for (KeyValue kv : kvs) { maxRowStringLength = Math.max(maxRowStringLength, getRowString(kv).length()); maxFamilyStringLength = Math.max(maxFamilyStringLength, getFamilyString(kv).length()); maxQualifierStringLength = Math.max(maxQualifierStringLength, getQualifierString(kv) .length()); maxTimestampLength = Math.max(maxTimestampLength, Long.valueOf(kv.getTimestamp()).toString() .length()); } StringBuilder sb = new StringBuilder(); for (KeyValue kv : kvs) { if (sb.length() > 0) { sb.append("\n"); } String row = toStringWithPadding(kv, maxRowStringLength, maxFamilyStringLength, maxQualifierStringLength, maxTimestampLength, includeMeta); sb.append(row); } return sb.toString(); }
/** * {@inheritDoc} */ @Override public void put(Put put) throws IOException { byte[] row = put.getRow(); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR)); for (byte[] family : put.getFamilyMap().keySet()) { if (columnFamilies.contains(new String(family, StandardCharsets.UTF_8)) == false) { throw new RuntimeException("Not Exists columnFamily : " + new String(family, StandardCharsets.UTF_8)); } NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR)); for (KeyValue kv : put.getFamilyMap().get(family)) { kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis())); byte[] qualifier = kv.getQualifier(); NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>()); qualifierData.put(kv.getTimestamp(), kv.getValue()); } } }
private static boolean matchesQuery(KeyValue kv, Set<String> qualSet, int maxVersions, Map<String, Long> lastDelTimeMap) { Long lastDelTS = lastDelTimeMap.get(getRowQualStr(kv)); long ts = kv.getTimestamp(); return qualSet.contains(qualStr(kv)) && ts >= TIMESTAMPS[TIMESTAMPS.length - maxVersions] && (lastDelTS == null || ts > lastDelTS); }
/** * Decrement the timestamp. For tests (currently wasteful) * * Remember timestamps are sorted reverse chronologically. * @param in * @return previous key */ public static KeyValue previousKey(final KeyValue in) { return createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in), CellUtil.cloneQualifier(in), in.getTimestamp() - 1); }
/** * Produces a string map for this key/value pair. Useful for programmatic use * and manipulation of the data stored in an WALKey, for example, printing * as JSON. Values are left out due to their tendency to be large. If needed, * they can be added manually. * * @return the Map<String,?> containing data from this key */ public Map<String, Object> toStringMap() { Map<String, Object> stringMap = new HashMap<>(); stringMap.put("row", Bytes.toStringBinary(getRowArray(), getRowOffset(), getRowLength())); stringMap.put("family", Bytes.toStringBinary(getFamilyArray(), getFamilyOffset(), getFamilyLength())); stringMap.put("qualifier", Bytes.toStringBinary(getQualifierArray(), getQualifierOffset(), getQualifierLength())); stringMap.put("timestamp", getTimestamp()); stringMap.put("vlen", getValueLength()); Iterator<Tag> tags = getTags(); if (tags != null) { List<String> tagsString = new ArrayList<String>(); while (tags.hasNext()) { tagsString.add(tags.next().toString()); } stringMap.put("tag", tagsString); } return stringMap; }
/** * Tests that getTimestamp() does always return the proper timestamp, even after updating it. * See HBASE-6265. */ @Test public void testGetTimestamp() { KeyValue kv = new KeyValue(Bytes.toBytes("myRow"), Bytes.toBytes("myCF"), Bytes.toBytes("myQualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("myValue")); long time1 = kv.getTimestamp(); kv.updateLatestStamp(Bytes.toBytes(12345L)); long time2 = kv.getTimestamp(); assertEquals(HConstants.LATEST_TIMESTAMP, time1); assertEquals(12345L, time2); }
protected static String toStringWithPadding(final KeyValue kv, final int maxRowLength, int maxFamilyLength, int maxQualifierLength, int maxTimestampLength, boolean includeMeta) { String leadingLengths = ""; String familyLength = kv.getFamilyLength() + " "; if (includeMeta) { leadingLengths += Strings.padFront(kv.getKeyLength() + "", '0', 4); leadingLengths += " "; leadingLengths += Strings.padFront(kv.getValueLength() + "", '0', 4); leadingLengths += " "; leadingLengths += Strings.padFront(kv.getRowLength() + "", '0', 2); leadingLengths += " "; } int spacesAfterRow = maxRowLength - getRowString(kv).length() + 2; int spacesAfterFamily = maxFamilyLength - getFamilyString(kv).length() + 2; int spacesAfterQualifier = maxQualifierLength - getQualifierString(kv).length() + 1; int spacesAfterTimestamp = maxTimestampLength - Long.valueOf(kv.getTimestamp()).toString().length() + 1; return leadingLengths + getRowString(kv) + StringUtils.repeat(' ', spacesAfterRow) + familyLength + getFamilyString(kv) + StringUtils.repeat(' ', spacesAfterFamily) + getQualifierString(kv) + StringUtils.repeat(' ', spacesAfterQualifier) + getTimestampString(kv) + StringUtils.repeat(' ', spacesAfterTimestamp) + getTypeString(kv) + " " + getValueString(kv); }
public void verifyKvs(KeyValue[][] kvss, boolean allFiles, List<Long> boundaries) { if (allFiles) { assertEquals(kvss.length, writers.size()); } int skippedWriters = 0; for (int i = 0; i < kvss.length; ++i) { KeyValue[] kvs = kvss[i]; if (kvs != null) { Writer w = writers.get(i - skippedWriters); assertEquals(kvs.length, w.kvs.size()); for (int j = 0; j < kvs.length; ++j) { assertTrue(kvs[j].getTimestamp() >= boundaries.get(i)); assertTrue(kvs[j].getTimestamp() < boundaries.get(i + 1)); assertEquals(kvs[j], w.kvs.get(j)); } } else { assertFalse(allFiles); ++skippedWriters; } } }
try { for (KeyValue k: set) { assertTrue(count++ == k.getTimestamp()); count = 0; for (KeyValue k: set) { assertTrue(count++ == k.getTimestamp());
private long addRowsByKeysDataSize(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); MemStoreSizing memstoreSizing = new NonThreadSafeMemStoreSizing(); for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); hmc.add(kv, memstoreSizing); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); } MemStoreSize mss = memstoreSizing.getMemStoreSize(); regionServicesForStores.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); return mss.getDataSize(); }
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys, byte[] val) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += Segment.getCellLength(kv); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0, cellsCount); return totalLen; }
private void updateMutationAddingTags(final Mutation m) { byte[] attribute = m.getAttribute("visibility"); byte[] cf = null; List<Cell> updatedCells = new ArrayList<>(); if (attribute != null) { for (List<? extends Cell> edits : m.getFamilyCellMap().values()) { for (Cell cell : edits) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (cf == null) { cf = CellUtil.cloneFamily(kv); } Tag tag = new ArrayBackedTag((byte) 1, attribute); List<Tag> tagList = new ArrayList<>(); tagList.add(tag); KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(), CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0, kv.getQualifierLength(), kv.getTimestamp(), KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0, kv.getValueLength(), tagList); ((List<Cell>) updatedCells).add(newKV); } } m.getFamilyCellMap().remove(cf); // Update the family map m.getFamilyCellMap().put(cf, updatedCells); } }
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += Segment.getCellLength(kv); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0, hmc.getActive().getCellsCount() - cellsCount); return totalLen; }
@Override public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e, final Put put, final WALEdit edit, final Durability durability) throws IOException { byte[] attribute = put.getAttribute("visibility"); byte[] cf = null; List<Cell> updatedCells = new ArrayList<>(); if (attribute != null) { for (List<? extends Cell> edits : put.getFamilyCellMap().values()) { for (Cell cell : edits) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (cf == null) { cf = CellUtil.cloneFamily(kv); } Tag tag = new ArrayBackedTag(TAG_TYPE, attribute); List<Tag> tagList = new ArrayList<>(1); tagList.add(tag); KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(), CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0, kv.getQualifierLength(), kv.getTimestamp(), KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0, kv.getValueLength(), tagList); ((List<Cell>) updatedCells).add(newKV); } } put.getFamilyCellMap().remove(cf); // Update the family map put.getFamilyCellMap().put(cf, updatedCells); } } }
private void internalRun() throws IOException { for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) { MultiVersionConcurrencyControl.WriteEntry w = mvcc.begin(); // Insert the sequence value (i) byte[] v = Bytes.toBytes(i); KeyValue kv = new KeyValue(row, f, q1, i, v); kv.setSequenceId(w.getWriteNumber()); memstore.add(kv, null); mvcc.completeAndWait(w); // Assert that we can read back KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0); s.seek(kv); Cell ret = s.next(); assertNotNull("Didnt find own write at all", ret); assertEquals("Didnt read own writes", kv.getTimestamp(), ret.getTimestamp()); } } }
@Test public void testCreateKey() { byte[] row = "myRow".getBytes(); byte[] qualifier = "myQualifier".getBytes(); // Mimic what Storefile.createBloomKeyValue() does byte[] rowKey = KeyValueUtil.createFirstOnRow(row, 0, row.length, new byte[0], 0, 0, row, 0, 0).getKey(); byte[] rowColKey = KeyValueUtil.createFirstOnRow(row, 0, row.length, new byte[0], 0, 0, qualifier, 0, qualifier.length).getKey(); KeyValue rowKV = KeyValueUtil.createKeyValueFromKey(rowKey); KeyValue rowColKV = KeyValueUtil.createKeyValueFromKey(rowColKey); assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp()); assertEquals(Bytes.toStringBinary(rowKV.getRowArray(), rowKV.getRowOffset(), rowKV.getRowLength()), Bytes.toStringBinary(rowColKV.getRowArray(), rowColKV.getRowOffset(), rowColKV.getRowLength())); assertEquals(0, rowKV.getQualifierLength()); } }
KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(), KeyValue.Type.DeleteColumn); kvDel.setSequenceId(w.getWriteNumber());