private ByteArraySegment serializeValue(Long value) { if (value == null || value == Attributes.NULL_ATTRIBUTE_VALUE) { // Deletion. return null; } byte[] result = new byte[VALUE_LENGTH]; BitConverter.writeLong(result, 0, value); return new ByteArraySegment(result); }
/** * Compares two non-null ByteArraySegments of the same length using lexicographic bitwise comparison. * * @param b1 First instance. * @param b2 Second instance. * @return -1 if b1 should be before b2, 0 if b1 equals b2 and 1 if b1 should be after b2. */ int compare(ByteArraySegment b1, ByteArraySegment b2) { assert b1.getLength() == b2.getLength(); return compare(b1.array(), b1.arrayOffset(), b2.array(), b2.arrayOffset(), b1.getLength()); }
@Override @SneakyThrows(IOException.class) public byte[] toBytes() { return SERIALIZER.serialize(this).getCopy(); }
/** * Creates a new instance of the BTreePage class wrapping the given Data Items (no header or footer). * * @param config Page Configuration. * @param count Number of items in data. * @param data A ByteArraySegment containing a list of Key-Value pairs to include. The contents of this ByteArraySegment * will be copied into a new buffer, so changes to this BTreePage will not affect it. */ private BTreePage(Config config, int count, ByteArraySegment data) { this(config, new ByteArraySegment(new byte[DATA_OFFSET + data.getLength() + FOOTER_LENGTH]), false); Preconditions.checkArgument(count * config.entryLength == data.getLength(), "Unexpected data length given the count."); formatHeaderAndFooter(count, ID_GENERATOR.nextInt()); this.data.copyFrom(data, 0, data.getLength()); }
@Override public String toString() { if (getLength() > 128) { return String.format("Length = %s", getLength()); } else { return String.format("{%s}", IntStream.range(arrayOffset(), arrayOffset() + getLength()).boxed() .map(i -> Byte.toString(this.array[i])) .collect(Collectors.joining(","))); } }
/** * Tests the basic functionality of get() and set() methods. */ @Test public void testGetSet() { final int incrementValue = 7; final int decrementValue = 11; final byte[] buffer = createFormattedBuffer(); final int halfOffset = buffer.length / 2; ByteArraySegment s1 = new ByteArraySegment(buffer, 0, halfOffset); ByteArraySegment s2 = new ByteArraySegment(buffer, halfOffset, buffer.length - halfOffset); //s1 - increment by 7 for (int i = 0; i < s1.getLength(); i++) { Assert.assertEquals("Unexpected value for initial get in first half of buffer at index " + i, i, s1.get(i)); byte newValue = (byte) (s1.get(i) + incrementValue); s1.set(i, newValue); Assert.assertEquals("Unexpected value for updated get in first half of buffer at index " + i, newValue, s1.get(i)); Assert.assertEquals("Unexpected value for the base buffer (first half) at index " + i, newValue, buffer[i]); } //s2 - decrement by 11 for (int i = 0; i < s2.getLength(); i++) { Assert.assertEquals("Unexpected value for initial get in second half of buffer at index " + i, i + halfOffset, s2.get(i)); byte newValue = (byte) (s2.get(i) - decrementValue); s2.set(i, newValue); Assert.assertEquals("Unexpected value for updated get in second half of buffer at index " + i, newValue, s2.get(i)); Assert.assertEquals("Unexpected value for the base buffer (second half) at index " + (i - halfOffset), newValue, buffer[i + halfOffset]); } }
private void serializeHandle(RollingSegmentHandle handle) throws StreamSegmentException { ByteArraySegment handleData = HandleSerializer.serialize(handle); try { this.baseStorage.write(handle.getHeaderHandle(), 0, handleData.getReader(), handleData.getLength()); handle.setHeaderLength(handleData.getLength()); log.debug("Header for '{}' fully serialized to '{}'.", handle.getSegmentName(), handle.getHeaderHandle().getSegmentName()); } catch (BadOffsetException ex) { // If we get BadOffsetException when writing the Handle, it means it was modified externally. throw new StorageNotPrimaryException(handle.getSegmentName(), ex); } }
/** * Tests the ability for the ByteArraySegment to create sub-segments. */ @Test public void testSubSegment() { final byte[] buffer = createFormattedBuffer(); ByteArraySegment segment = new ByteArraySegment(buffer); // As long as the size of the segment > 1, choose a segment half the length of the current one. // If current iteration is odd, choose the upper segment. If it is even, choose the lower one. int iteration = 0; int startOffset = 0; while (segment.getLength() > 1) { iteration++; // Check correctness. for (int i = 0; i < segment.getLength(); i++) { Assert.assertEquals(String.format("Unexpected value at offset %d for subsegment (O=%d, L=%d), iteration %d.", i, startOffset, segment.getLength(), iteration), buffer[i + startOffset], segment.get(i)); } // Pick a new size and create a new subsegment. if (iteration % 2 == 0) { // Upper half for even iterations. startOffset = startOffset + segment.getLength() / 2; segment = segment.subSegment(segment.getLength() / 2, segment.getLength() - segment.getLength() / 2); } else { // Lower half for odd iterations. segment = segment.subSegment(0, segment.getLength() / 2); } } }
private List<ByteArraySegment> copy(List<ByteArraySegment> source) { return source.stream() .map(a -> new ByteArraySegment(Arrays.copyOf(a.array(), a.array().length), a.arrayOffset(), a.getLength())) .collect(Collectors.toList()); }
/** * Tests the functionality of writeTo and readFrom. */ @Test public void testWriteToReadFrom() throws IOException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); int count = 10; ArrayList<ByteArraySegment> sourceSegments = new ArrayList<>(); for (int i = 0; i < count; i++) { ByteArraySegment s = new ByteArraySegment(createFormattedBuffer()); sourceSegments.add(s); s.writeTo(outputStream); } InputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); for (int i = 0; i < count; i++) { ByteArraySegment s = sourceSegments.get(i); ByteArraySegment t = new ByteArraySegment(new byte[s.getLength()]); t.readFrom(inputStream); Assert.assertEquals("Source and target lengths differ.", s.getLength(), t.getLength()); for (int j = 0; j < s.getLength(); j++) { if (t.get(j) != s.get(j)) { Assert.fail(String.format("Source at target differ at index %d.", j)); } } } }
/** * Tests the behavior of the ByteArraySegment in read-only mode. */ @Test public void testReadOnly() throws Exception { final byte[] buffer = createFormattedBuffer(); ByteArraySegment segment = new ByteArraySegment(buffer, 0, buffer.length, true); // Check the isReadonly flag Assert.assertTrue("Unexpected value for isReadOnly() for read-only segment.", segment.isReadOnly()); Assert.assertFalse("Unexpected value for isReadOnly() for non-read-only segment.", new ByteArraySegment(buffer).isReadOnly()); // Verify that "mutator" methods do not work. checkReadOnlyException("copyFrom", () -> segment.copyFrom(new ByteArraySegment(new byte[10], 0, 10), 0, 10)); checkReadOnlyException("getWriter", segment::getWriter); checkReadOnlyException("set", () -> segment.set(0, (byte) 0)); // Check to see that, even though we did get an exception, the buffer was not modified. for (int i = 0; i < buffer.length; i++) { Assert.assertEquals("One of the 'mutator' methods modified the buffer at index " + i, i, buffer[i]); } // Check that a subsegment is also read-only. Assert.assertTrue("Unexpected value for isReadOnly() for read-only sub-segment.", segment.subSegment(0, 1).isReadOnly()); Assert.assertTrue("Unexpected value for isReadOnly() for read-only sub-segment from non-read-only segment (when attempting to create a non-read-only segment).", segment.subSegment(0, 1, false).isReadOnly()); Assert.assertTrue("Unexpected value for isReadOnly() for read-only sub-segment from non-read-only segment.", new ByteArraySegment(buffer).subSegment(0, 1, true).isReadOnly()); }
CompletableFuture<ByteArraySegment> read(long offset, int length, Duration timeout) { return CompletableFuture.supplyAsync(() -> { synchronized (this.data) { if (this.checkOffsets.get()) { // We want to make sure that we actually read pages that we wrote, and not from arbitrary locations // in the data source. Preconditions.checkArgument(this.offsets.isEmpty() || this.offsets.getOrDefault(offset, false), "Offset not registered or already obsolete: " + offset); } return new ByteArraySegment(this.data.getData().subSegment((int) offset, length).getCopy()); } }, executorService()); }
if (this.contents.getLength() <= this.config.getMaxPageSize()) { int maxDataLength = (this.config.getMaxPageSize() - this.header.getLength() - this.footer.getLength()) / this.config.entryLength * this.config.entryLength; int remainingPageCount = (int) Math.ceil((double) this.data.getLength() / maxDataLength); ByteArraySegment splitPageData = this.data.subSegment(readIndex, itemsPerPage * this.config.entryLength); result.add(new BTreePage(this.config, itemsPerPage, splitPageData)); readIndex += splitPageData.getLength(); remainingPageCount--; remainingItems -= itemsPerPage; assert readIndex == this.data.getLength() : "did not copy everything"; return result;
/** * Sets the Value at the given position. * * @param pos The Position to set the value at. * @param value A ByteArraySegment representing the value to set. */ private void setValueAtPosition(int pos, ByteArraySegment value) { Preconditions.checkElementIndex(pos, getCount(), "pos must be non-negative and smaller than the number of items."); Preconditions.checkArgument(value.getLength() == this.config.valueLength, "Given value has incorrect length."); this.data.copyFrom(value, pos * this.config.entryLength + this.config.keyLength, value.getLength()); }
/** * Tests the functionality of getReader (the ability to return an InputStream from a sub-segment of the main buffer). */ @Test public void testGetReader() throws IOException { final byte[] buffer = createFormattedBuffer(); ByteArraySegment segment = new ByteArraySegment(buffer); for (int offset = 0; offset < buffer.length / 2; offset++) { int length = buffer.length - offset * 2; byte[] readBuffer = new byte[length]; try (InputStream stream = segment.getReader(offset, length)) { int readBytes = StreamHelpers.readAll(stream, readBuffer, 0, readBuffer.length); Assert.assertEquals("Unexpected number of bytes read from the InputStream at offset " + offset, length, readBytes); } for (int i = 0; i < length; i++) { Assert.assertEquals("Unexpected value at index " + i + " after reading from offset " + offset, segment.get(i + offset), readBuffer[i]); } } }
/** * Creates a new instance of a DataFrame. * * @param source The ByteArraySegment to wrap. */ DataFrame(ByteArraySegment source) { Preconditions.checkArgument(!source.isReadOnly(), "Cannot create a WriteFrame for a readonly source."); this.data = source; this.writeEntryStartIndex = -1; this.sealed = source.isReadOnly(); this.writePosition = this.sealed ? -1 : 0; //We want to use the DataFrame for at least 1 byte of data. int sourceLength = this.data.getLength(); Exceptions.checkArgument(sourceLength > FrameHeader.SERIALIZATION_LENGTH, "data", "Insufficient array length. Byte array must have a length of at least %d.", FrameHeader.SERIALIZATION_LENGTH + 1); this.header = new WriteFrameHeader(CURRENT_VERSION, this.data.subSegment(0, FrameHeader.SERIALIZATION_LENGTH)); this.contents = this.data.subSegment(FrameHeader.SERIALIZATION_LENGTH, sourceLength - FrameHeader.SERIALIZATION_LENGTH); }
/** * Determines whether the given ByteArraySegment represents an Index Page * * @param pageContents The ByteArraySegment to check. * @return True if Index Page, False if Leaf page. * @throws IllegalDataFormatException If the given contents is not a valid BTreePage format. */ static boolean isIndexPage(@NonNull ByteArraySegment pageContents) { // Check ID match. int headerId = BitConverter.readInt(pageContents, ID_OFFSET); int footerId = BitConverter.readInt(pageContents, pageContents.getLength() - FOOTER_LENGTH); if (headerId != footerId) { throw new IllegalDataFormatException("Invalid Page Format (id mismatch). HeaderId=%s, FooterId=%s.", headerId, footerId); } int flags = pageContents.get(FLAGS_OFFSET); return (flags & FLAG_INDEX_PAGE) == FLAG_INDEX_PAGE; }
/** * Tests the functionality of copyFrom. */ @Test public void testCopyFrom() { final byte[] sourceBuffer = createFormattedBuffer(); final int targetOffset = 11; final byte[] targetBuffer = new byte[sourceBuffer.length + targetOffset]; final int copyLength = sourceBuffer.length - 7; ByteArraySegment source = new ByteArraySegment(sourceBuffer); ByteArraySegment target = new ByteArraySegment(targetBuffer); // Copy second part. target.copyFrom(source, targetOffset, copyLength); for (int i = 0; i < targetBuffer.length; i++) { int expectedValue = i < targetOffset || i >= targetOffset + copyLength ? 0 : i - targetOffset; Assert.assertEquals("Unexpected value after copyFrom (1) in segment at offset " + i, expectedValue, target.get(i)); Assert.assertEquals("Unexpected value after copyFrom (1) in base buffer at offset " + i, expectedValue, targetBuffer[i]); } // Test copyFrom with source offset. Arrays.fill(targetBuffer, (byte) 0); final int sourceOffset = 3; target.copyFrom(source, sourceOffset, targetOffset, copyLength); for (int i = 0; i < targetBuffer.length; i++) { int expectedValue = i < targetOffset || i >= targetOffset + copyLength ? 0 : (i - targetOffset + sourceOffset); Assert.assertEquals("Unexpected value after copyFrom (2) in segment at offset " + i, expectedValue, target.get(i)); Assert.assertEquals("Unexpected value after copyFrom (2) in base buffer at offset " + i, expectedValue, targetBuffer[i]); } }
/** * Tests the functionality of getWriter (the ability to return an OutputStream that can be used to write to the main buffer). */ @Test public void testGetWriter() throws IOException { final byte[] buffer = new byte[Byte.MAX_VALUE]; ByteArraySegment segment = new ByteArraySegment(buffer); try (OutputStream writer = segment.getWriter()) { for (int i = 0; i < buffer.length; i++) { writer.write(i); } } for (int i = 0; i < buffer.length; i++) { Assert.assertEquals("Unexpected value in segment at index " + i, i, segment.get(i)); Assert.assertEquals("Unexpected value in source buffer at index " + i, i, buffer[i]); } }
private PageEntry generateEntry(byte keyByte, byte valueByte) { val key = new ByteArraySegment(new byte[KEY_LENGTH]); Arrays.fill(key.array(), keyByte); val newValue = new ByteArraySegment(new byte[VALUE_LENGTH]); Arrays.fill(newValue.array(), valueByte); return new PageEntry(key, newValue); }