private SegmentProperties doGetStreamSegmentInfo(String streamSegmentName) throws IOException { long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); PosixFileAttributes attrs = Files.readAttributes(Paths.get(config.getRoot(), streamSegmentName), PosixFileAttributes.class); StreamSegmentInformation information = StreamSegmentInformation.builder() .name(streamSegmentName) .length(attrs.size()) .sealed(!(attrs.permissions().contains(OWNER_WRITE))) .lastModified(new ImmutableDate(attrs.creationTime().toMillis())) .build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName); return information; }
private ArrayList<Operation> populate(MemoryStateUpdater updater, int segmentCount, int operationCountPerType) throws DataCorruptionException { ArrayList<Operation> operations = new ArrayList<>(); long offset = 0; for (int i = 0; i < segmentCount; i++) { for (int j = 0; j < operationCountPerType; j++) { StreamSegmentMapOperation mapOp = new StreamSegmentMapOperation( StreamSegmentInformation.builder().name("a").length( i * j).build()); mapOp.setStreamSegmentId(i); operations.add(mapOp); StreamSegmentAppendOperation appendOp = new StreamSegmentAppendOperation(i, Integer.toString(i).getBytes(), null); appendOp.setStreamSegmentOffset(offset); offset += appendOp.getData().length; operations.add(appendOp); operations.add(new MergeSegmentOperation(i, j)); } } for (int i = 0; i < operations.size(); i++) { operations.get(i).setSequenceNumber(i); } updater.process(operations.iterator()); return operations; }
@Test(timeout = 20000) public void testReadSegmentTruncated() { // Set up PravegaRequestProcessor instance to execute read segment request against String streamSegmentName = "testReadSegment"; int readLength = 1000; StreamSegmentStore store = mock(StreamSegmentStore.class); ServerConnection connection = mock(ServerConnection.class); PravegaRequestProcessor processor = new PravegaRequestProcessor(store, mock(TableStore.class), connection); TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.Truncated, 0, readLength); List<ReadResultEntry> results = new ArrayList<>(); results.add(entry1); CompletableFuture<ReadResult> readResult = new CompletableFuture<>(); readResult.complete(new TestReadResult(0, readLength, results)); when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult); StreamSegmentInformation info = StreamSegmentInformation.builder() .name(streamSegmentName) .length(1234) .startOffset(123) .build(); when(store.getStreamSegmentInfo(streamSegmentName, PravegaRequestProcessor.TIMEOUT)) .thenReturn(CompletableFuture.completedFuture(info)); // Execute and Verify readSegment calling stack in connection and store is executed as design. processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, "")); verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT); verify(store).getStreamSegmentInfo(streamSegmentName, PravegaRequestProcessor.TIMEOUT); verify(connection).send(new WireCommands.SegmentIsTruncated(0, streamSegmentName, info.getStartOffset(), "")); verifyNoMoreInteractions(connection); verifyNoMoreInteractions(store); }
.length(123) .sealed(true) .attributes(toAttributes(createAttributeUpdates(ATTRIBUTE_COUNT)))
.length(getSegmentLength.apply(segmentName)) .startOffset(getSegmentStartOffset.apply(segmentName)) .sealed(i % 2 == 0)
val si1 = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(SEGMENT_LENGTH).startOffset(0).sealed(true).build(); val rr1 = StreamSegmentStorageReader.read(si1, 0, SEGMENT_LENGTH, SEGMENT_APPEND_SIZE - 1, s); val firstEntry1 = rr1.next(); val si2 = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(SEGMENT_LENGTH).startOffset(0).sealed(true).build(); val rr2 = StreamSegmentStorageReader.read(si2, 0, SEGMENT_LENGTH, SEGMENT_APPEND_SIZE - 1, s);
/** * Reads a range of bytes from a Segment in Storage. * * @param handle A SegmentHandle pointing to the Segment to read from. * @param startOffset The first offset within the Segment to read from. * @param maxReadLength The maximum number of bytes to read. * @param readBlockSize The maximum number of bytes to read at once (the returned StreamSegmentReadResult will be * broken down into Entries smaller than or equal to this size). * @param storage A ReadOnlyStorage to execute the reads against. * @return A StreamSegmentReadResult that can be used to process the data. This will be made up of ReadResultEntries * of the following types: Storage, Truncated or EndOfSegment. */ public static StreamSegmentReadResult read(SegmentHandle handle, long startOffset, int maxReadLength, int readBlockSize, ReadOnlyStorage storage) { Exceptions.checkArgument(startOffset >= 0, "startOffset", "startOffset must be a non-negative number."); Exceptions.checkArgument(maxReadLength >= 0, "maxReadLength", "maxReadLength must be a non-negative number."); Preconditions.checkNotNull(handle, "handle"); Preconditions.checkNotNull(storage, "storage"); String traceId = String.format("Read[%s]", handle.getSegmentName()); // Build a SegmentInfo using the information we are given. If startOffset or length are incorrect, the underlying // ReadOnlyStorage will throw appropriate exceptions at the caller. StreamSegmentInformation segmentInfo = StreamSegmentInformation.builder().name(handle.getSegmentName()) .startOffset(startOffset) .length(startOffset + maxReadLength) .build(); return new StreamSegmentReadResult(startOffset, maxReadLength, new SegmentReader(segmentInfo, handle, readBlockSize, storage), traceId); }
@Override public SegmentProperties getStreamSegmentInfo(String streamSegmentName) throws StreamSegmentException { ensureInitializedAndNotClosed(); long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); try { return HDFS_RETRY.run(() -> { FileStatus last = findStatusForSegment(streamSegmentName, true); boolean isSealed = isSealed(last.getPath()); StreamSegmentInformation result = StreamSegmentInformation.builder().name(streamSegmentName).length(last.getLen()).sealed(isSealed).build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName, result); return result; }); } catch (IOException e) { throw HDFSExceptionHelpers.convertException(streamSegmentName, e); } catch (RetriesExhaustedException e) { throw HDFSExceptionHelpers.convertException(streamSegmentName, e.getCause()); } }
/** * Tests the read() method on a truncated segment. */ @Test public void testTruncatedSegment() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(SEGMENT_APPEND_SIZE) .sealed(false).build(); // 1. Read a truncated offset. val truncatedResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(truncatedResult, si, 0, 0, writtenData); // 2. Read a truncated offset. val nonTruncatedResult = StreamSegmentStorageReader.read(si, SEGMENT_APPEND_SIZE, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(nonTruncatedResult, si, SEGMENT_APPEND_SIZE, writtenData.length - SEGMENT_APPEND_SIZE, writtenData); }
/** * Creates a new {@link StreamSegmentInformationBuilder} with information already populated from the given SegmentProperties. * * @param base The SegmentProperties to use as a base. * @return The Builder. */ public static StreamSegmentInformationBuilder from(SegmentProperties base) { return StreamSegmentInformation.builder() .name(base.getName()) .startOffset(base.getStartOffset()) .length(base.getLength()) .sealed(base.isSealed()) .deleted(base.isDeleted()) .lastModified(base.getLastModified()) .attributes(base.getAttributes()); }
@Override protected StreamSegmentMapOperation createOperation(Random random) { long length = MathHelpers.abs(random.nextLong()); val op = new StreamSegmentMapOperation(StreamSegmentInformation .builder() .name(super.getStreamSegmentName(random.nextLong())) .startOffset(length / 2) .length(length) .sealed(random.nextBoolean()) .deleted(random.nextBoolean()) .attributes(createAttributes(10)) .build()); op.markPinned(); return op; }
private StreamSegmentInformation doGetStreamSegmentInfo(String streamSegmentName) { long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); S3ObjectMetadata result = client.getObjectMetadata(config.getBucket(), config.getRoot() + streamSegmentName); AccessControlList acls = client.getObjectAcl(config.getBucket(), config.getRoot() + streamSegmentName); boolean canWrite = acls.getGrants().stream().anyMatch(grant -> grant.getPermission().compareTo(Permission.WRITE) >= 0); StreamSegmentInformation information = StreamSegmentInformation.builder() .name(streamSegmentName) .length(result.getContentLength()) .sealed(!canWrite) .lastModified(new ImmutableDate(result.getLastModified().toInstant().toEpochMilli())) .build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName); return information; }
private SegmentProperties createSegmentProperty(String streamSegmentName, UUID txnId) { Map<UUID, Long> attributes = new HashMap<>(); attributes.put(Attributes.EVENT_COUNT, 10L); attributes.put(Attributes.CREATION_TIME, System.currentTimeMillis()); return StreamSegmentInformation.builder() .name(txnId == null ? streamSegmentName + "#." : streamSegmentName + "#transaction." + txnId) .sealed(true) .deleted(false) .lastModified(null) .startOffset(0) .length(100) .attributes(attributes) .build(); }
/** * Tests the read() method on a non-truncated, non-sealed segment. */ @Test public void testNormalRead() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(0).sealed(false).build(); // We try to read 1 byte beyond the end of the segment to verify the reading ends correctly. val readResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(readResult, si, 0, writtenData.length, writtenData); }
/** * Tests the read() method on a Sealed Segment. */ @Test public void testSealedSegment() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(0).sealed(true).build(); // We try to read 1 byte beyond the end of the segment to verify the reading ends correctly. val readResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(readResult, si, 0, writtenData.length, writtenData); }
private StreamSegmentMapOperation createMap(String name) { return new StreamSegmentMapOperation(StreamSegmentInformation.builder() .name(name) .length(SEGMENT_LENGTH) .startOffset(SEGMENT_LENGTH / 2) .sealed(true) .attributes(createAttributes()) .build()); }
private void read00(RevisionDataInput input, SegmentInfo.SegmentInfoBuilder builder) throws IOException { builder.segmentId(input.readLong()); val infoBuilder = StreamSegmentInformation .builder() .name(input.readUTF()) .length(input.readLong()) .startOffset(input.readLong()) .sealed(input.readBoolean()); infoBuilder.attributes(input.readMap(RevisionDataInput::readUUID, RevisionDataInput::readLong)); builder.properties(infoBuilder.build()); } }
SegmentProperties getInfo() { synchronized (this.lock) { return StreamSegmentInformation.builder().name(this.name).length(this.length).sealed(this.sealed).build(); } }
private StreamSegmentMapOperation createTransactionMap(String name) { return new StreamSegmentMapOperation( StreamSegmentInformation.builder() .name(name) .length(SEALED_SOURCE_LENGTH) .sealed(true) .attributes(createAttributes()) .build()); }
@Override public SegmentProperties getStreamSegmentInfo(String segmentName) throws StreamSegmentException { val handle = (RollingSegmentHandle) openRead(segmentName); return StreamSegmentInformation .builder() .name(handle.getSegmentName()) .sealed(handle.isSealed()) .length(handle.length()) .build(); }