private SegmentProperties doGetStreamSegmentInfo(String streamSegmentName) throws IOException { long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); PosixFileAttributes attrs = Files.readAttributes(Paths.get(config.getRoot(), streamSegmentName), PosixFileAttributes.class); StreamSegmentInformation information = StreamSegmentInformation.builder() .name(streamSegmentName) .length(attrs.size()) .sealed(!(attrs.permissions().contains(OWNER_WRITE))) .lastModified(new ImmutableDate(attrs.creationTime().toMillis())) .build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName); return information; }
private StreamSegmentInformation doGetStreamSegmentInfo(String streamSegmentName) { long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); S3ObjectMetadata result = client.getObjectMetadata(config.getBucket(), config.getRoot() + streamSegmentName); AccessControlList acls = client.getObjectAcl(config.getBucket(), config.getRoot() + streamSegmentName); boolean canWrite = acls.getGrants().stream().anyMatch(grant -> grant.getPermission().compareTo(Permission.WRITE) >= 0); StreamSegmentInformation information = StreamSegmentInformation.builder() .name(streamSegmentName) .length(result.getContentLength()) .sealed(!canWrite) .lastModified(new ImmutableDate(result.getLastModified().toInstant().toEpochMilli())) .build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName); return information; }
private SegmentProperties createSegmentProperty(String streamSegmentName, UUID txnId) { Map<UUID, Long> attributes = new HashMap<>(); attributes.put(Attributes.EVENT_COUNT, 10L); attributes.put(Attributes.CREATION_TIME, System.currentTimeMillis()); return StreamSegmentInformation.builder() .name(txnId == null ? streamSegmentName + "#." : streamSegmentName + "#transaction." + txnId) .sealed(true) .deleted(false) .lastModified(null) .startOffset(0) .length(100) .attributes(attributes) .build(); }
final String segmentName = "Segment"; final long segmentId = 123; final SegmentProperties storageInfo = StreamSegmentInformation.builder().name(segmentName).length(123).sealed(true).build(); final long metadataLength = storageInfo.getLength() + 1;
.name(segmentName) .length(123) .sealed(true) .attributes(toAttributes(createAttributeUpdates(ATTRIBUTE_COUNT))) .build();
.length(getSegmentLength.apply(segmentName)) .startOffset(getSegmentStartOffset.apply(segmentName)) .sealed(i % 2 == 0) .attributes(toAttributes(createAttributeUpdates(ATTRIBUTE_COUNT))) .build();
val si1 = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(SEGMENT_LENGTH).startOffset(0).sealed(true).build(); val rr1 = StreamSegmentStorageReader.read(si1, 0, SEGMENT_LENGTH, SEGMENT_APPEND_SIZE - 1, s); val firstEntry1 = rr1.next(); val si2 = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(SEGMENT_LENGTH).startOffset(0).sealed(true).build(); val rr2 = StreamSegmentStorageReader.read(si2, 0, SEGMENT_LENGTH, SEGMENT_APPEND_SIZE - 1, s);
@Override public SegmentProperties getStreamSegmentInfo(String streamSegmentName) throws StreamSegmentException { ensureInitializedAndNotClosed(); long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); try { return HDFS_RETRY.run(() -> { FileStatus last = findStatusForSegment(streamSegmentName, true); boolean isSealed = isSealed(last.getPath()); StreamSegmentInformation result = StreamSegmentInformation.builder().name(streamSegmentName).length(last.getLen()).sealed(isSealed).build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName, result); return result; }); } catch (IOException e) { throw HDFSExceptionHelpers.convertException(streamSegmentName, e); } catch (RetriesExhaustedException e) { throw HDFSExceptionHelpers.convertException(streamSegmentName, e.getCause()); } }
/** * Tests the read() method on a truncated segment. */ @Test public void testTruncatedSegment() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(SEGMENT_APPEND_SIZE) .sealed(false).build(); // 1. Read a truncated offset. val truncatedResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(truncatedResult, si, 0, 0, writtenData); // 2. Read a truncated offset. val nonTruncatedResult = StreamSegmentStorageReader.read(si, SEGMENT_APPEND_SIZE, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(nonTruncatedResult, si, SEGMENT_APPEND_SIZE, writtenData.length - SEGMENT_APPEND_SIZE, writtenData); }
@Override protected StreamSegmentMapOperation createOperation(Random random) { long length = MathHelpers.abs(random.nextLong()); val op = new StreamSegmentMapOperation(StreamSegmentInformation .builder() .name(super.getStreamSegmentName(random.nextLong())) .startOffset(length / 2) .length(length) .sealed(random.nextBoolean()) .deleted(random.nextBoolean()) .attributes(createAttributes(10)) .build()); op.markPinned(); return op; }
/** * Creates a new {@link StreamSegmentInformationBuilder} with information already populated from the given SegmentProperties. * * @param base The SegmentProperties to use as a base. * @return The Builder. */ public static StreamSegmentInformationBuilder from(SegmentProperties base) { return StreamSegmentInformation.builder() .name(base.getName()) .startOffset(base.getStartOffset()) .length(base.getLength()) .sealed(base.isSealed()) .deleted(base.isDeleted()) .lastModified(base.getLastModified()) .attributes(base.getAttributes()); }
/** * Tests the read() method on a non-truncated, non-sealed segment. */ @Test public void testNormalRead() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(0).sealed(false).build(); // We try to read 1 byte beyond the end of the segment to verify the reading ends correctly. val readResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(readResult, si, 0, writtenData.length, writtenData); }
/** * Tests the read() method on a Sealed Segment. */ @Test public void testSealedSegment() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(0).sealed(true).build(); // We try to read 1 byte beyond the end of the segment to verify the reading ends correctly. val readResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(readResult, si, 0, writtenData.length, writtenData); }
private StreamSegmentMapOperation createMap(String name) { return new StreamSegmentMapOperation(StreamSegmentInformation.builder() .name(name) .length(SEGMENT_LENGTH) .startOffset(SEGMENT_LENGTH / 2) .sealed(true) .attributes(createAttributes()) .build()); }
private void read00(RevisionDataInput input, SegmentInfo.SegmentInfoBuilder builder) throws IOException { builder.segmentId(input.readLong()); val infoBuilder = StreamSegmentInformation .builder() .name(input.readUTF()) .length(input.readLong()) .startOffset(input.readLong()) .sealed(input.readBoolean()); infoBuilder.attributes(input.readMap(RevisionDataInput::readUUID, RevisionDataInput::readLong)); builder.properties(infoBuilder.build()); } }
SegmentProperties getInfo() { synchronized (this.lock) { return StreamSegmentInformation.builder().name(this.name).length(this.length).sealed(this.sealed).build(); } }
private StreamSegmentMapOperation createTransactionMap(String name) { return new StreamSegmentMapOperation( StreamSegmentInformation.builder() .name(name) .length(SEALED_SOURCE_LENGTH) .sealed(true) .attributes(createAttributes()) .build()); }
@Override public SegmentProperties getStreamSegmentInfo(String segmentName) throws StreamSegmentException { val handle = (RollingSegmentHandle) openRead(segmentName); return StreamSegmentInformation .builder() .name(handle.getSegmentName()) .sealed(handle.isSealed()) .length(handle.length()) .build(); }