@Override public CompletableFuture<SegmentProperties> getStreamSegmentInfo(String streamSegmentName, Duration timeout) { if (this.segments.contains(streamSegmentName)) { return CompletableFuture.completedFuture(StreamSegmentInformation.builder().name(streamSegmentName).build()); } else { return Futures.failedFuture(new StreamSegmentNotExistsException(streamSegmentName)); } }
static SegmentInfo newSegment(String name, Collection<AttributeUpdate> attributeUpdates) { val infoBuilder = StreamSegmentInformation .builder() .name(name); if (attributeUpdates != null) { infoBuilder.attributes(attributeUpdates.stream().collect(Collectors.toMap(AttributeUpdate::getAttributeId, AttributeUpdate::getValue))); } return builder() .segmentId(ContainerMetadata.NO_STREAM_SEGMENT_ID) .properties(infoBuilder.build()) .build(); }
SegmentProperties getInfo() { synchronized (this.lock) { return StreamSegmentInformation.builder().name(this.name).length(this.length).sealed(this.sealed).build(); } }
private SegmentProperties doGetStreamSegmentInfo(String streamSegmentName) throws IOException { long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); PosixFileAttributes attrs = Files.readAttributes(Paths.get(config.getRoot(), streamSegmentName), PosixFileAttributes.class); StreamSegmentInformation information = StreamSegmentInformation.builder() .name(streamSegmentName) .length(attrs.size()) .sealed(!(attrs.permissions().contains(OWNER_WRITE))) .lastModified(new ImmutableDate(attrs.creationTime().toMillis())) .build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName); return information; }
/** * Creates a number of StreamSegments in the given Metadata and OperationLog. */ Set<Long> createStreamSegmentsWithOperations(int streamSegmentCount, OperationLog durableLog) { val operations = new ArrayList<StreamSegmentMapOperation>(); val futures = new ArrayList<CompletableFuture<Void>>(); for (int i = 0; i < streamSegmentCount; i++) { val op = new StreamSegmentMapOperation(StreamSegmentInformation.builder().name(getStreamSegmentName(i)).build()); operations.add(op); futures.add(durableLog.add(op, TIMEOUT)); } Futures.allOf(futures).join(); return operations.stream().map(StreamSegmentMapOperation::getStreamSegmentId).collect(Collectors.toSet()); }
@Override public SegmentProperties getStreamSegmentInfo(String segmentName) throws StreamSegmentException { val handle = (RollingSegmentHandle) openRead(segmentName); return StreamSegmentInformation .builder() .name(handle.getSegmentName()) .sealed(handle.isSealed()) .length(handle.length()) .build(); }
private StreamSegmentInformation doGetStreamSegmentInfo(String streamSegmentName) { long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); S3ObjectMetadata result = client.getObjectMetadata(config.getBucket(), config.getRoot() + streamSegmentName); AccessControlList acls = client.getObjectAcl(config.getBucket(), config.getRoot() + streamSegmentName); boolean canWrite = acls.getGrants().stream().anyMatch(grant -> grant.getPermission().compareTo(Permission.WRITE) >= 0); StreamSegmentInformation information = StreamSegmentInformation.builder() .name(streamSegmentName) .length(result.getContentLength()) .sealed(!canWrite) .lastModified(new ImmutableDate(result.getLastModified().toInstant().toEpochMilli())) .build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName); return information; }
private SegmentProperties createSegmentProperty(String streamSegmentName, UUID txnId) { Map<UUID, Long> attributes = new HashMap<>(); attributes.put(Attributes.EVENT_COUNT, 10L); attributes.put(Attributes.CREATION_TIME, System.currentTimeMillis()); return StreamSegmentInformation.builder() .name(txnId == null ? streamSegmentName + "#." : streamSegmentName + "#transaction." + txnId) .sealed(true) .deleted(false) .lastModified(null) .startOffset(0) .length(100) .attributes(attributes) .build(); }
@Override public SegmentProperties getStreamSegmentInfo(String streamSegmentName) throws StreamSegmentException { ensureInitializedAndNotClosed(); long traceId = LoggerHelpers.traceEnter(log, "getStreamSegmentInfo", streamSegmentName); try { return HDFS_RETRY.run(() -> { FileStatus last = findStatusForSegment(streamSegmentName, true); boolean isSealed = isSealed(last.getPath()); StreamSegmentInformation result = StreamSegmentInformation.builder().name(streamSegmentName).length(last.getLen()).sealed(isSealed).build(); LoggerHelpers.traceLeave(log, "getStreamSegmentInfo", traceId, streamSegmentName, result); return result; }); } catch (IOException e) { throw HDFSExceptionHelpers.convertException(streamSegmentName, e); } catch (RetriesExhaustedException e) { throw HDFSExceptionHelpers.convertException(streamSegmentName, e.getCause()); } }
/** * Creates a number of Transaction Segments in the given Metadata and OperationLog. */ AbstractMap<Long, Long> createTransactionsWithOperations(Set<Long> streamSegmentIds, int transactionsPerStreamSegment, ContainerMetadata containerMetadata, OperationLog durableLog) { val result = new HashMap<Long, Long>(); for (long streamSegmentId : streamSegmentIds) { String streamSegmentName = containerMetadata.getStreamSegmentMetadata(streamSegmentId).getName(); for (int i = 0; i < transactionsPerStreamSegment; i++) { String transactionName = StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, UUID.randomUUID()); StreamSegmentMapOperation op = new StreamSegmentMapOperation(StreamSegmentInformation.builder().name(transactionName).build()); durableLog.add(op, TIMEOUT).join(); result.put(op.getStreamSegmentId(), streamSegmentId); } } return result; }
@Override protected StreamSegmentMapOperation createOperation(Random random) { long length = MathHelpers.abs(random.nextLong()); val op = new StreamSegmentMapOperation(StreamSegmentInformation .builder() .name(super.getStreamSegmentName(random.nextLong())) .startOffset(length / 2) .length(length) .sealed(random.nextBoolean()) .deleted(random.nextBoolean()) .attributes(createAttributes(10)) .build()); op.markPinned(); return op; }
private void read00(RevisionDataInput input, SegmentInfo.SegmentInfoBuilder builder) throws IOException { builder.segmentId(input.readLong()); val infoBuilder = StreamSegmentInformation .builder() .name(input.readUTF()) .length(input.readLong()) .startOffset(input.readLong()) .sealed(input.readBoolean()); infoBuilder.attributes(input.readMap(RevisionDataInput::readUUID, RevisionDataInput::readLong)); builder.properties(infoBuilder.build()); } }
private StreamSegmentMapOperation createTransactionMap(String name) { return new StreamSegmentMapOperation( StreamSegmentInformation.builder() .name(name) .length(SEALED_SOURCE_LENGTH) .sealed(true) .attributes(createAttributes()) .build()); }
/** * Creates a new {@link StreamSegmentInformationBuilder} with information already populated from the given SegmentProperties. * * @param base The SegmentProperties to use as a base. * @return The Builder. */ public static StreamSegmentInformationBuilder from(SegmentProperties base) { return StreamSegmentInformation.builder() .name(base.getName()) .startOffset(base.getStartOffset()) .length(base.getLength()) .sealed(base.isSealed()) .deleted(base.isDeleted()) .lastModified(base.getLastModified()) .attributes(base.getAttributes()); }
private StreamSegmentMapOperation createMap(String name) { return new StreamSegmentMapOperation(StreamSegmentInformation.builder() .name(name) .length(SEGMENT_LENGTH) .startOffset(SEGMENT_LENGTH / 2) .sealed(true) .attributes(createAttributes()) .build()); }
private long mapSegment(OperationMetadataUpdater updater, UpdateableContainerMetadata referenceMetadata) throws Exception { String segmentName = "Segment_" + updater.nextOperationSequenceNumber(); val mapOp = new StreamSegmentMapOperation(StreamSegmentInformation.builder().name(segmentName).build()); process(mapOp, updater); if (referenceMetadata != null) { val rsm = referenceMetadata.mapStreamSegmentId(segmentName, mapOp.getStreamSegmentId()); rsm.setLength(0); rsm.setStorageLength(0); } return mapOp.getStreamSegmentId(); }
private long mapTransaction(long parentSegmentId, OperationMetadataUpdater updater, UpdateableContainerMetadata referenceMetadata) throws Exception { String segmentName = "Transaction_" + updater.nextOperationSequenceNumber(); val mapOp = new StreamSegmentMapOperation(StreamSegmentInformation .builder() .name(segmentName) .attributes(Collections.singletonMap(PARENT_ID, parentSegmentId)) .build()); process(mapOp, updater); if (referenceMetadata != null) { val rsm = referenceMetadata.mapStreamSegmentId(segmentName, mapOp.getStreamSegmentId()); rsm.setLength(0); rsm.setStorageLength(0); rsm.updateAttributes(mapOp.getAttributes()); } return mapOp.getStreamSegmentId(); }
/** * Tests the read() method on a Sealed Segment. */ @Test public void testSealedSegment() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(0).sealed(true).build(); // We try to read 1 byte beyond the end of the segment to verify the reading ends correctly. val readResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(readResult, si, 0, writtenData.length, writtenData); }
/** * Tests the read() method on a non-truncated, non-sealed segment. */ @Test public void testNormalRead() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(0).sealed(false).build(); // We try to read 1 byte beyond the end of the segment to verify the reading ends correctly. val readResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(readResult, si, 0, writtenData.length, writtenData); }
/** * Tests the read() method on a truncated segment. */ @Test public void testTruncatedSegment() throws Exception { @Cleanup val s = createStorage(); val writtenData = populate(s); val si = StreamSegmentInformation.builder().name(SEGMENT_NAME).length(writtenData.length).startOffset(SEGMENT_APPEND_SIZE) .sealed(false).build(); // 1. Read a truncated offset. val truncatedResult = StreamSegmentStorageReader.read(si, 0, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(truncatedResult, si, 0, 0, writtenData); // 2. Read a truncated offset. val nonTruncatedResult = StreamSegmentStorageReader.read(si, SEGMENT_APPEND_SIZE, writtenData.length + 1, SEGMENT_APPEND_SIZE - 1, s); verifyReadResult(nonTruncatedResult, si, SEGMENT_APPEND_SIZE, writtenData.length - SEGMENT_APPEND_SIZE, writtenData); }