/** * maximum batch size based on pipe and must fit batches count * * @param pipe Pipe source * @param mustFit how many batches must fit on the pipe * @return max batch size */ public static <S extends MessageSchema<S>> int computeMaxBatchSize(Pipe<S> pipe, int mustFit) { assert(mustFit>=1); int maxBatchFromBytes = pipe.maxVarLen==0?Integer.MAX_VALUE:(pipe.sizeOfBlobRing/pipe.maxVarLen)/mustFit; int maxBatchFromPrimary = (pipe.sizeOfSlabRing/FieldReferenceOffsetManager.maxFragmentSize(from(pipe)))/mustFit; return Math.min(maxBatchFromBytes, maxBatchFromPrimary); }
private void validate(T messageSchema, int minimumFragmentsOnRing, int maximumLenghOfVariableLengthFields) { //Do not change this constant, it is assumed by Pipe roll over masks and flags if (blobBits>30) { throw new UnsupportedOperationException("Unable to support blob data larger than 1GB Reduce either the data size or count of desired message msgs:"+ minimumFragmentsOnRing+" varLen:"+maximumLenghOfVariableLengthFields+" schema: "+messageSchema+ " slabBits: "+slabBits+" maxFragSize: "+FieldReferenceOffsetManager.maxFragmentSize(MessageSchema.from(messageSchema))); } if (slabBits>30) { throw new UnsupportedOperationException("Unable to support slab data larger than 1GB, Reduce the count of desired message msgs:"+ minimumFragmentsOnRing+" varLen:"+maximumLenghOfVariableLengthFields+" schema: "+messageSchema); } }
private void validate(T messageSchema, int minimumFragmentsOnRing, int maximumLenghOfVariableLengthFields) { //Do not change this constant, it is assumed by Pipe roll over masks and flags if (blobBits>30) { throw new UnsupportedOperationException("Unable to support blob data larger than 1GB Reduce either the data size or count of desired message msgs:"+ minimumFragmentsOnRing+" varLen:"+maximumLenghOfVariableLengthFields+" schema: "+messageSchema+ " slabBits: "+slabBits+" maxFragSize: "+FieldReferenceOffsetManager.maxFragmentSize(MessageSchema.from(messageSchema))); } if (slabBits>30) { throw new UnsupportedOperationException("Unable to support slab data larger than 1GB, Reduce the count of desired message msgs:"+ minimumFragmentsOnRing+" varLen:"+maximumLenghOfVariableLengthFields+" schema: "+messageSchema); } }
private void validate(T messageSchema, int minimumFragmentsOnRing, int maximumLenghOfVariableLengthFields) { //Do not change this constant, it is assumed by Pipe roll over masks and flags if (blobBits>30) { throw new UnsupportedOperationException("Unable to support blob data larger than 1GB Reduce either the data size or count of desired message msgs:"+ minimumFragmentsOnRing+" varLen:"+maximumLenghOfVariableLengthFields+" schema: "+messageSchema+ " slabBits: "+slabBits+" maxFragSize: "+FieldReferenceOffsetManager.maxFragmentSize(MessageSchema.from(messageSchema))); } if (slabBits>30) { throw new UnsupportedOperationException("Unable to support slab data larger than 1GB, Reduce the count of desired message msgs:"+ minimumFragmentsOnRing+" varLen:"+maximumLenghOfVariableLengthFields+" schema: "+messageSchema); } }
/** * maximum batch size based on pipe and must fit batches count * * @param pipe Pipe source * @param mustFit how many batches must fit on the pipe * @return max batch size */ public static <S extends MessageSchema<S>> int computeMaxBatchSize(Pipe<S> pipe, int mustFit) { assert(mustFit>=1); int maxBatchFromBytes = pipe.maxVarLen==0?Integer.MAX_VALUE:(pipe.sizeOfBlobRing/pipe.maxVarLen)/mustFit; int maxBatchFromPrimary = (pipe.sizeOfSlabRing/FieldReferenceOffsetManager.maxFragmentSize(from(pipe)))/mustFit; return Math.min(maxBatchFromBytes, maxBatchFromPrimary); }
/** * * @param messageCount number of messages */ public boolean hasRoomFor(int messageCount) { return null==msgCommandChannel.goPipe || Pipe.hasRoomForWrite(msgCommandChannel.goPipe, FieldReferenceOffsetManager.maxFragmentSize(Pipe.from(msgCommandChannel.goPipe))*messageCount); }
/** * * @param messageCount number to be multiplied by msgCommandChannel.httpRequest * @return has room */ public boolean hasRoomFor(int messageCount) { assert(msgCommandChannel.httpRequest!=null) : "Client side HTTP Request must be enabled"; return Pipe.hasRoomForWrite(msgCommandChannel.httpRequest, FieldReferenceOffsetManager.maxFragmentSize( Pipe.from(msgCommandChannel.httpRequest))*messageCount); }
public StreamingVisitorWriter(Pipe outputRing, StreamingWriteVisitor visitor) { this.visitor = visitor; this.outputRing = outputRing; this.from = Pipe.from(outputRing); this.maxFragmentSize = FieldReferenceOffsetManager.maxFragmentSize(this.from); this.navState = new LowLevelStateManager(from); }
public StreamingVisitorWriter(Pipe outputRing, StreamingWriteVisitor visitor) { this.visitor = visitor; this.outputRing = outputRing; this.from = Pipe.from(outputRing); this.maxFragmentSize = FieldReferenceOffsetManager.maxFragmentSize(this.from); this.navState = new LowLevelStateManager(from); }
public StreamingVisitorWriter(Pipe outputRing, StreamingWriteVisitor visitor) { this.visitor = visitor; this.outputRing = outputRing; this.from = Pipe.from(outputRing); this.maxFragmentSize = FieldReferenceOffsetManager.maxFragmentSize(this.from); this.navState = new LowLevelStateManager(from); }
@Override public void startup() { tailCache = Pipe.tailPosition(input); maxFrag = FieldReferenceOffsetManager.maxFragmentSize(Pipe.from(input)); }
public PipeConfig(T messageSchema, int minimumFragmentsOnRing, int maximumLenghOfVariableLengthFields, byte[] byteConst) { this.maximumLenghOfVariableLengthFields = maximumLenghOfVariableLengthFields; int biggestFragment = FieldReferenceOffsetManager.maxFragmentSize(MessageSchema.from(messageSchema)); int primaryMinSize = minimumFragmentsOnRing * biggestFragment; this.slabBits = (byte)(32 - Integer.numberOfLeadingZeros(primaryMinSize - 1)); int maxVarFieldsInRingAtOnce = FieldReferenceOffsetManager.maxVarLenFieldsPerPrimaryRingSize(MessageSchema.from(messageSchema), 1<<slabBits); int secondaryMinSize = maxVarFieldsInRingAtOnce * maximumLenghOfVariableLengthFields; this.blobBits = ((0==maximumLenghOfVariableLengthFields) | (0==maxVarFieldsInRingAtOnce))? (byte)0 : (byte)(32 - Integer.numberOfLeadingZeros(secondaryMinSize - 1)); this.byteConst = byteConst; this.schema = messageSchema; validate(messageSchema, minimumFragmentsOnRing, maximumLenghOfVariableLengthFields ); }
public PipeConfig(T messageSchema, int minimumFragmentsOnRing, int maximumLenghOfVariableLengthFields, byte[] byteConst) { int biggestFragment = FieldReferenceOffsetManager.maxFragmentSize(MessageSchema.from(messageSchema)); int primaryMinSize = minimumFragmentsOnRing*biggestFragment; this.slabBits = (byte)(32 - Integer.numberOfLeadingZeros(primaryMinSize - 1)); int maxVarFieldsInRingAtOnce = FieldReferenceOffsetManager.maxVarLenFieldsPerPrimaryRingSize(MessageSchema.from(messageSchema), 1<<slabBits); int secondaryMinSize = maxVarFieldsInRingAtOnce * maximumLenghOfVariableLengthFields; this.blobBits = ((0==maximumLenghOfVariableLengthFields) | (0==maxVarFieldsInRingAtOnce))? (byte)0 : (byte)(32 - Integer.numberOfLeadingZeros(secondaryMinSize - 1)); this.byteConst = byteConst; this.schema = messageSchema; validate(messageSchema, minimumFragmentsOnRing, maximumLenghOfVariableLengthFields ); }
public PipeConfig(T messageSchema, int minimumFragmentsOnRing, int maximumLenghOfVariableLengthFields, byte[] byteConst) { this.maximumLenghOfVariableLengthFields = maximumLenghOfVariableLengthFields; int biggestFragment = FieldReferenceOffsetManager.maxFragmentSize(MessageSchema.from(messageSchema)); int primaryMinSize = minimumFragmentsOnRing * biggestFragment; this.slabBits = (byte)(32 - Integer.numberOfLeadingZeros(primaryMinSize - 1)); int maxVarFieldsInRingAtOnce = FieldReferenceOffsetManager.maxVarLenFieldsPerPrimaryRingSize(MessageSchema.from(messageSchema), 1<<slabBits); int secondaryMinSize = maxVarFieldsInRingAtOnce * maximumLenghOfVariableLengthFields; this.blobBits = ((0==maximumLenghOfVariableLengthFields) | (0==maxVarFieldsInRingAtOnce))? (byte)0 : (byte)(32 - Integer.numberOfLeadingZeros(secondaryMinSize - 1)); this.byteConst = byteConst; this.schema = messageSchema; validate(messageSchema, minimumFragmentsOnRing, maximumLenghOfVariableLengthFields ); }
/** * Checks the specified pipe to see if there is room to write * @param pipe pipe to examine * @param <S> MessageSchema to extend * @return <code>true</code> if pipe has room to write else <code>false</code> */ public static <S extends MessageSchema<S>> boolean hasRoomForWrite(Pipe<S> pipe) { assert(null != pipe.slabRing) : "Pipe must be init before use"; assert(null != pipe.llRead) : "Expected pipe to be setup for low level use."; assert(Pipe.singleThreadPerPipeWrite(pipe.id)); long temp = pipe.llRead.llwConfirmedPosition+FieldReferenceOffsetManager.maxFragmentSize(Pipe.from(pipe)); return roomToLowLevelWrite(pipe, temp); }
public boolean goHasRoomFor(int messageCount) { assert(null==goPipe || Pipe.isInit(goPipe)) : "not init yet"; return (null==goPipe || Pipe.hasRoomForWrite(goPipe, FieldReferenceOffsetManager.maxFragmentSize(Pipe.from(goPipe))*messageCount)); }
/** * Checks the specified pipe to see if there is room to write * @param pipe pipe to examine * @param <S> MessageSchema to extend * @return <code>true</code> if pipe has room to write else <code>false</code> */ public static <S extends MessageSchema<S>> boolean hasRoomForWrite(Pipe<S> pipe) { assert(null != pipe.slabRing) : "Pipe must be init before use"; assert(null != pipe.llRead) : "Expected pipe to be setup for low level use."; assert(Pipe.singleThreadPerPipeWrite(pipe.id)); return roomToLowLevelWrite(pipe, pipe.llRead.llwConfirmedPosition+FieldReferenceOffsetManager.maxFragmentSize(Pipe.from(pipe))); }
/** * Check to see if specified pipe has room to write * @param pipe to be checked * @return <code>true</code> if pipe has room else <code>false</code> */ public static boolean hasRoomForWrite(Pipe pipe) { assert(Pipe.singleThreadPerPipeWrite(pipe.id)); assert(pipe!=null); assert(Pipe.isInit(pipe)); assert(pipe.usingHighLevelAPI); return StackStateWalker.hasRoomForFragmentOfSizeX(pipe, pipe.ringWalker.nextWorkingHead - (pipe.sizeOfSlabRing - FieldReferenceOffsetManager.maxFragmentSize( Pipe.from(pipe)))); }
/** * Check to see if specified pipe has room to write * @param pipe to be checked * @return <code>true</code> if pipe has room else <code>false</code> */ public static boolean hasRoomForWrite(Pipe pipe) { assert(Pipe.singleThreadPerPipeWrite(pipe.id)); assert(pipe!=null); assert(Pipe.isInit(pipe)); assert(pipe.usingHighLevelAPI); return StackStateWalker.hasRoomForFragmentOfSizeX(pipe, pipe.ringWalker.nextWorkingHead - (pipe.sizeOfSlabRing - FieldReferenceOffsetManager.maxFragmentSize( Pipe.from(pipe)))); }
/** * Check to see if specified pipe has room to write * @param pipe to be checked * @return <code>true</code> if pipe has room else <code>false</code> */ public static boolean hasRoomForWrite(Pipe pipe) { assert(Pipe.singleThreadPerPipeWrite(pipe.id)); assert(pipe!=null); assert(Pipe.isInit(pipe)); assert(pipe.usingHighLevelAPI); return StackStateWalker.hasRoomForFragmentOfSizeX(pipe, pipe.ringWalker.nextWorkingHead - (pipe.sizeOfSlabRing - FieldReferenceOffsetManager.maxFragmentSize( Pipe.from(pipe)))); }