/** * Removes all entries from the cache and the SortedIndex, regardless of their state. * * @throws IllegalStateException If the StreamSegmentReadIndex is not closed. */ private void removeAllEntries() { // A bit unusual, but we want to make sure we do not call this while the index is active. Preconditions.checkState(this.closed, "Cannot call removeAllEntries unless the ReadIndex is closed."); int count; synchronized (this.lock) { this.indexEntries.forEach(entry -> { if (entry.isDataEntry()) { CacheKey key = getCacheKey(entry); this.cache.remove(key); } }); count = this.indexEntries.size(); this.indexEntries.clear(); } log.info("{}: Cleared all cache entries ({}).", this.traceObjectId, count); }
private ArrayList<Long> populate(SortedIndex<TestEntry> index, int itemCount, int maxKey) { Random rnd = new Random(0); val keys = new ArrayList<Long>(); for (int i = 0; i < itemCount; i++) { // Generate a unique key. long key; do { key = rnd.nextInt(maxKey); } while (index.get(key) != null); keys.add(key); index.put(new TestEntry(key)); } return keys; }
private void readCeiling(SortedIndex<TestEntry> rbt, int count) { for (int i = 0; i < count; i++) { rbt.getCeiling(i); } }
/** * Gets a copy of all the ReadIndexEntries in this Index that are not RedirectReadIndices. All returned * entries have their offsets adjusted by the given amount. * * @param offsetAdjustment The amount to adjust the offset by. */ private List<MergedIndexEntry> getAllEntries(long offsetAdjustment) { Exceptions.checkArgument(offsetAdjustment >= 0, "offsetAdjustment", "offsetAdjustment must be a non-negative number."); synchronized (this.lock) { List<MergedIndexEntry> result = new ArrayList<>(this.indexEntries.size()); this.indexEntries.forEach(entry -> { if (entry.isDataEntry()) { result.add(new MergedIndexEntry(entry.getStreamSegmentOffset() + offsetAdjustment, this.metadata.getId(), (CacheIndexEntry) entry)); } }); return result; } }
/** * Tests the clear() method. */ @Test public void testClear() { val index = createIndex(); val keys = populate(index); Assert.assertNotNull("Unexpected return value for getFirst() on non-empty index.", index.getFirst()); Assert.assertNotNull("Unexpected return value for getLast() on non-empty index.", index.getLast()); index.clear(); Assert.assertEquals("Unexpected size of empty index.", 0, index.size()); Assert.assertNull("Unexpected return value for getFirst() on empty index.", index.getFirst()); Assert.assertNull("Unexpected return value for getLast() on empty index.", index.getLast()); for (long key : keys) { Assert.assertNull("Unexpected value for get() on empty index.", index.get(key)); Assert.assertNull("Unexpected value for getCeiling() on empty index.", index.getCeiling(key)); } }
/** * Tests the forEach() method. */ @Test public void testForEach() { // Create an index and populate it. val index = createIndex(); val validKeys = populate(index); // Extract the keys using forEach - they should be ordered naturally. val actualKeys = new ArrayList<Long>(); index.forEach(e -> actualKeys.add(e.key())); // Order the inserted keys using the same comparator we used for the index. validKeys.sort(KEY_COMPARATOR); // Verify that modifying the index while looping through it does throw an exception. AssertExtensions.assertThrows( "forEach did not throw when a new item was added during enumeration.", () -> index.forEach(e -> index.put(new TestEntry(index.size()))), ex -> ex instanceof ConcurrentModificationException); AssertExtensions.assertThrows( "forEach did not throw when an item was removed during enumeration.", () -> index.forEach(e -> index.remove(e.key())), ex -> ex instanceof ConcurrentModificationException); AssertExtensions.assertThrows( "forEach did not throw when the index was cleared during enumeration.", () -> index.forEach(e -> index.clear()), ex -> ex instanceof ConcurrentModificationException); }
/** * Tests the remove(), size(), get(), getFirst(), getLast() methods. */ @Test public void testRemove() { val index = createIndex(); val keys = populate(index); // Remove the items, in order. keys.sort(KEY_COMPARATOR); val keysToRemove = new LinkedList<Long>(keys); int expectedSize = index.size(); while (keysToRemove.size() > 0) { // Remove either the first or the last key - this helps test getFirst/getLast properly. long key = expectedSize % 2 == 0 ? keysToRemove.removeLast() : keysToRemove.removeFirst(); val entry = index.get(key); val removedEntry = index.remove(key); expectedSize--; Assert.assertEquals("Unexpected removed entry for key " + key, entry, removedEntry); Assert.assertEquals("Unexpected size after removing key " + key, expectedSize, index.size()); Assert.assertNull("Entry was not removed for key " + key, index.get(key)); if (expectedSize == 0) { Assert.assertNull("Unexpected value from getFirst() when index is empty.", index.getFirst()); Assert.assertNull("Unexpected value from getLast() when index is empty.", index.getLast()); } else { Assert.assertEquals("Unexpected value from getFirst() after removing key " + key, (long) keysToRemove.getFirst(), index.getFirst().key()); Assert.assertEquals("Unexpected value from getLast() after removing key " + key, (long) keysToRemove.getLast(), index.getLast().key()); } } }
do { key = rnd.nextInt(); } while (index.get(key) != null); index.put(entry); Assert.assertEquals("Unexpected size.", i + 1, index.size()); Assert.assertEquals("Unexpected value from getFirst() after " + index.size() + " insertions.", firstEntry, index.getFirst()); Assert.assertEquals("Unexpected value from getLast() after " + index.size() + " insertions.", lastEntry, index.getLast()); val oldEntry = index.get(key); val entry = new TestEntry(key); val overriddenEntry = index.put(entry); val reRetrievedEntry = index.get(key); Assert.assertEquals("Unexpected overridden entry for key " + key, oldEntry, overriddenEntry); Assert.assertEquals("New entry was not placed in the index for key " + key, entry, reRetrievedEntry); Assert.assertEquals("Unexpected size when overriding entry.", ITEM_COUNT, index.size());
/** * Tests various operations on already sorted input. */ @Test public void testSortedInput() { val index = createIndex(); for (int key = 0; key < ITEM_COUNT; key++) { index.put(new TestEntry(key)); } //Get + GetCeiling. for (int key = 0; key < ITEM_COUNT; key++) { Assert.assertEquals("Unexpected value from get() for key " + key, key, (long) index.get(key).key()); Assert.assertEquals("Unexpected value from getCeiling() for key " + key, key, (long) index.getCeiling(key).key()); } // Remove + get. for (long key = 0; key < ITEM_COUNT; key++) { long removedKey = index.remove(key).key(); Assert.assertEquals("Unexpected value from remove(). ", key, removedKey); Assert.assertNull("Unexpected value from get() for removed key " + key, index.get(key)); if (key == ITEM_COUNT - 1) { Assert.assertNull("Unexpected value from getCeiling() for removed key " + key, index.getCeiling(key)); } else { Assert.assertEquals("Unexpected value from getCeiling() for removed key " + key, key + 1, (long) index.getCeiling(key).key()); } } }
ReadIndexEntry indexEntry = this.indexEntries.get(pendingMerge.getMergeOffset()); assert indexEntry != null && !indexEntry.isDataEntry() : String.format("pendingMergers points to a ReadIndexEntry that does not exist or is of the wrong type. sourceStreamSegmentId = %d, offset = %d, treeEntry = %s.", this.indexEntries.remove(pendingMerge.getMergeOffset()); this.pendingMergers.remove(sourceMetadata.getId()); sourceEntries.forEach(this::addToIndex);
ReadIndexEntry indexEntry = this.indexEntries.getFloor(startOffset); if (indexEntry == null || startOffset > indexEntry.getLastStreamSegmentOffset() || !indexEntry.isDataEntry()) {
private void readLast(SortedIndex<TestEntry> rbt, int count) { for (int i = 0; i < count; i++) { rbt.getLast(); } }
private void insert(SortedIndex<TestEntry> rbt, int count) { for (int i = 0; i < count; i++) { rbt.put(new TestEntry(i)); } }
private void readExact(SortedIndex<TestEntry> rbt, int count) { for (int i = 0; i < count; i++) { rbt.get(i); } }
/** * Tests the getFloor() method. */ @Test public void testGetFloor() { final int itemCount = 1000; final int maxKey = itemCount * 10; // Create an index and populate sparsely. val index = createIndex(); val validKeys = populate(index, itemCount, maxKey); validKeys.sort(KEY_REVERSE_COMPARATOR); val validKeysIterator = validKeys.iterator(); Long expectedValue = (long) Integer.MAX_VALUE; for (long testKey = maxKey; testKey >= 0; testKey--) { // Since both testKey and validKeysIterator increase with natural ordering, finding the next expected value // is a straightforward call to the iterator next() method. while (expectedValue != null && testKey < expectedValue) { if (validKeysIterator.hasNext()) { expectedValue = validKeysIterator.next(); } else { expectedValue = null; } } val ceilingEntry = index.getFloor(testKey); Long actualValue = ceilingEntry != null ? ceilingEntry.key() : null; Assert.assertEquals("Unexpected value for getCeiling for key " + testKey, expectedValue, actualValue); } }
lastEntry = this.indexEntries.getLast();
@GuardedBy("lock") private ReadIndexEntry addToIndex(ReadIndexEntry entry) { // Insert the new entry and figure out if an old entry was overwritten. ReadIndexEntry oldEntry = this.indexEntries.put(entry); if (entry.isDataEntry()) { if (entry instanceof MergedIndexEntry) { // This entry has already existed in the cache for a while; do not change its generation. this.summary.add(entry.getLength(), entry.getGeneration()); } else { // Update the Stats with the entry's length, and set the entry's generation as well. int generation = this.summary.add(entry.getLength()); entry.setGeneration(generation); } } if (oldEntry != null && oldEntry.isDataEntry()) { // Need to eject the old entry's data from the Cache Stats. this.summary.remove(oldEntry.getLength(), oldEntry.getGeneration()); } return oldEntry; }
/** * Returns a CacheReadResultEntry that matches the specified search parameters, but only if the data is readily available * in the cache and if there is an index entry that starts at that location.. As opposed from getSingleReadResultEntry(), * this method will return null if the requested data is not available. * * @param resultStartOffset The Offset within the StreamSegment where to start returning data from. * @param maxLength The maximum number of bytes to return. * @return A CacheReadResultEntry representing the data to return. */ private CacheReadResultEntry getSingleMemoryReadResultEntry(long resultStartOffset, int maxLength) { Exceptions.checkNotClosed(this.closed, this); if (maxLength > 0 && checkReadAvailability(resultStartOffset, false) == ReadAvailability.Available) { // Look up an entry in the index that contains our requested start offset. synchronized (this.lock) { ReadIndexEntry indexEntry = this.indexEntries.get(resultStartOffset); if (indexEntry != null && indexEntry.isDataEntry()) { // We found an entry; return a result for it. return createMemoryRead(indexEntry, resultStartOffset, maxLength, true); } } } // Nothing could be found in the cache at the given offset. return null; }
/** * Returns the length from the given offset until the beginning of the next index entry. If no such entry exists, or * if the length is greater than maxLength, then maxLength is returned. * * @param startOffset The offset to search from. * @param maxLength The maximum allowed length. * @return The result. */ @GuardedBy("lock") private int getLengthUntilNextEntry(long startOffset, int maxLength) { ReadIndexEntry ceilingEntry = this.indexEntries.getCeiling(startOffset); if (ceilingEntry != null) { maxLength = (int) Math.min(maxLength, ceilingEntry.getStreamSegmentOffset() - startOffset); } return maxLength; }