@GuardedBy("lock") @SneakyThrows(IOException.class) private void writeInternal(long startOffset, InputStream data, int length) throws BadOffsetException, StreamSegmentSealedException { Exceptions.checkArgument(length >= 0, "length", "bad length"); if (startOffset != this.length) { throw new BadOffsetException(this.name, this.length, startOffset); } if (this.sealed) { throw new StreamSegmentSealedException(this.name); } long offset = startOffset; ensureAllocated(offset, length); int writtenBytes = 0; while (writtenBytes < length) { OffsetLocation ol = getOffsetLocation(offset); int readBytes = data.read(this.data.get(ol.bufferSequence), ol.bufferOffset, Math.min(length - writtenBytes, BUFFER_SIZE - ol.bufferOffset)); if (readBytes < 0) { throw new IOException("reached end of stream while still expecting data"); } writtenBytes += readBytes; offset += readBytes; } this.length = Math.max(this.length, startOffset + length); }
void concat(StreamSegmentData other, long offset) throws BadOffsetException, StreamSegmentSealedException { synchronized (this.context.syncRoot) { // In order to do a proper concat, we need to lock on both the source and the target segments. But since // there's always a possibility of two concurrent calls to concat with swapped arguments, there is a chance // this could deadlock in certain scenarios. One way to avoid that is to ensure that only one call to concat() // can be in progress at any given time (for any instance of InMemoryStorage), thus the need to synchronize // on SyncContext.syncRoot. synchronized (other.lock) { Preconditions.checkState(other.sealed, "Cannot concat segment '%s' into '%s' because it is not sealed.", other.name, this.name); other.checkOpened(); synchronized (this.lock) { checkOpened(); if (offset != this.length) { throw new BadOffsetException(this.name, this.length, offset); } long bytesCopied = 0; int currentBlockIndex = 0; while (bytesCopied < other.length) { byte[] currentBlock = other.data.get(currentBlockIndex); int length = (int) Math.min(currentBlock.length, other.length - bytesCopied); ByteArrayInputStream bis = new ByteArrayInputStream(currentBlock, 0, length); writeInternal(this.length, bis, length); bytesCopied += length; currentBlockIndex++; } } } } }
/** * Pre-processes a StreamSegmentTruncateOperation. * * @param operation The Operation. * @throws BadOffsetException If the operation's Offset is not between the current StartOffset and current * EndOffset (SegmentLength - 1). */ void preProcessOperation(StreamSegmentTruncateOperation operation) throws BadOffsetException { ensureSegmentId(operation); if (operation.getStreamSegmentOffset() < this.startOffset || operation.getStreamSegmentOffset() > this.length) { String msg = String.format("Truncation Offset must be at least %d and at most %d, given %d.", this.startOffset, this.length, operation.getStreamSegmentOffset()); throw new BadOffsetException(this.name, this.startOffset, operation.getStreamSegmentOffset(), msg); } }
private void ensureOffset(RollingSegmentHandle handle, long offset) throws StreamSegmentException { if (offset != handle.length()) { // Force-refresh the handle to make sure it is still in sync with reality. Make sure we open a read handle // so that we don't force any sort of fencing during this process. val refreshedHandle = openHandle(handle.getSegmentName(), true); handle.refresh(refreshedHandle); log.debug("Handle refreshed: {}.", handle); if (offset != handle.length()) { // Still in disagreement; throw exception. throw new BadOffsetException(handle.getSegmentName(), handle.length(), offset); } } }
throw new BadOffsetException(handle.getSegmentName(), fileSize, offset); } else { long totalBytesWritten = 0;
@Override public void concat(SegmentHandle target, long offset, String sourceSegment) throws StreamSegmentException { ensureInitializedAndNotClosed(); long traceId = LoggerHelpers.traceEnter(log, "concat", target, offset, sourceSegment); target = asWritableHandle(target); // Check for target offset and whether it is sealed. FileStatus fileStatus = null; try { fileStatus = findStatusForSegment(target.getSegmentName(), true); if (isSealed(fileStatus.getPath())) { throw new StreamSegmentSealedException(target.getSegmentName()); } else if (getEpoch(fileStatus) > this.epoch) { throw new StorageNotPrimaryException(target.getSegmentName()); } else if (fileStatus.getLen() != offset) { throw new BadOffsetException(target.getSegmentName(), fileStatus.getLen(), offset); } FileStatus sourceFile = findStatusForSegment(sourceSegment, true); Preconditions.checkState(isSealed(sourceFile.getPath()), "Cannot concat segment '%s' into '%s' because it is not sealed.", sourceSegment, target.getSegmentName()); // Concat source file into target. this.fileSystem.concat(fileStatus.getPath(), new Path[]{sourceFile.getPath()}); } catch (IOException ex) { throw HDFSExceptionHelpers.convertException(sourceSegment, ex); } LoggerHelpers.traceLeave(log, "concat", traceId, target, offset, sourceSegment); }
throw new BadOffsetException(handle.getSegmentName(), status.getLen(), offset); } else if (stream.getPos() != offset) { throw new BadOffsetException(handle.getSegmentName(), status.getLen(), offset);
throw new BadOffsetException(this.name, this.length, operationOffset);
processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data), null)); result = Futures.failedFuture(new BadOffsetException(streamSegmentName, data.length, 0)); when(store.append(streamSegmentName, 0, data, updateEventNumber(clientId, 2, 1, 1), AppendProcessor.TIMEOUT)).thenReturn(result);
private Void doWrite(SegmentHandle handle, long offset, InputStream data, int length) throws StreamSegmentException { Preconditions.checkArgument(!handle.isReadOnly(), "handle must not be read-only."); long traceId = LoggerHelpers.traceEnter(log, "write", handle.getSegmentName(), offset, length); SegmentProperties si = doGetStreamSegmentInfo(handle.getSegmentName()); if (si.isSealed()) { throw new StreamSegmentSealedException(handle.getSegmentName()); } if (si.getLength() != offset) { throw new BadOffsetException(handle.getSegmentName(), si.getLength(), offset); } client.putObject(this.config.getBucket(), this.config.getRoot() + handle.getSegmentName(), Range.fromOffsetLength(offset, length), data); LoggerHelpers.traceLeave(log, "write", traceId); return null; }