public boolean canAppend(Journal.WriteCommand write) { int newSize = size + write.location.getSize(); if (newSize >= maxWriteBatchSize) { return false; } return true; }
private Location getNextInitializedLocation(Location location) throws IOException { Location mayNotBeInitialized = journal.getNextLocation(location); if (location.getSize() == NOT_SET && mayNotBeInitialized != null && mayNotBeInitialized.getSize() != NOT_SET) { // need to init size and type to skip return journal.getNextLocation(mayNotBeInitialized); } else { return mayNotBeInitialized; } }
public boolean canAppend(Journal.WriteCommand write) { int newSize = size + write.location.getSize(); if (newSize >= maxWriteBatchSize || offset+newSize > journal.getMaxFileLength() ) { return false; } return true; }
@Override protected synchronized void onLoad(ListNode<String, Location> node, Transaction tx) { try { Iterator<Entry<String, Location>> i = node.iterator(tx); while (i.hasNext()) { messageSize.addSize(i.next().getValue().getSize()); } } catch (IOException e) { LOG.warn("could not increment message size", e); } }
@Override public void writePayload(Location object, DataOutput dataOut) throws IOException { dataOut.writeInt(object.getDataFileId()); dataOut.writeInt(object.getOffset()); dataOut.writeInt(object.getSize()); }
@Override public void onRemove(Entry<String, Location> removed) { super.onRemove(removed); if (removed != null) { messageSize.addSize(-removed.getValue().getSize()); } }
@Override public synchronized Location addFirst(Transaction tx, String key, Location value) throws IOException { Location location = super.addFirst(tx, key, value); messageSize.addSize(value.getSize()); return location; }
@Override public synchronized Location add(Transaction tx, String key, Location value) throws IOException { Location location = super.add(tx, key, value); messageSize.addSize(value.getSize()); return location; }
public ByteSequence readRecord(Location location) throws IOException { if (!location.isValid()) { throw new IOException("Invalid location: " + location); } Journal.WriteCommand asyncWrite = inflightWrites.get(new Journal.WriteKey(location)); if (asyncWrite != null) { return asyncWrite.data; } try { if (location.getSize() == Location.NOT_SET) { file.seek(location.getOffset()); location.setSize(file.readInt()); location.setType(file.readByte()); } else { file.seek(location.getOffset() + Journal.RECORD_HEAD_SPACE); } if ((long)location.getOffset() + location.getSize() > dataFile.length) { throw new IOException("Invalid location size: " + location + ", size: " + location.getSize()); } byte[] data = new byte[location.getSize() - Journal.RECORD_HEAD_SPACE]; file.readFully(data); return new ByteSequence(data, 0, data.length); } catch (RuntimeException e) { throw new IOException("Invalid location: " + location + " : " + e, e); } }
private void removeAckLocationsForSub(KahaSubscriptionCommand command, Transaction tx, StoredDestination sd, String subscriptionKey) throws IOException { if (!sd.ackPositions.isEmpty(tx)) { SequenceSet sequences = sd.ackPositions.remove(tx, subscriptionKey); if (sequences == null || sequences.isEmpty()) { return; } ArrayList<Long> unreferenced = new ArrayList<>(); for(Long sequenceId : sequences) { if(!isSequenceReferenced(tx, sd, sequenceId)) { unreferenced.add(sequenceId); } } for(Long sequenceId : unreferenced) { // Find all the entries that need to get deleted. ArrayList<Entry<Long, MessageKeys>> deletes = new ArrayList<>(); sd.orderIndex.getDeleteList(tx, deletes, sequenceId); // Do the actual deletes. for (Entry<Long, MessageKeys> entry : deletes) { sd.locationIndex.remove(tx, entry.getValue().location); sd.messageIdIndex.remove(tx, entry.getValue().messageId); sd.orderIndex.remove(tx, entry.getKey()); decrementAndSubSizeToStoreStat(tx, command.getDestination(), entry.getValue().location.getSize()); } } } }
public void updateRecord(Location location, ByteSequence data, boolean sync) throws IOException { file.seek(location.getOffset() + Journal.RECORD_HEAD_SPACE); int size = Math.min(data.getLength(), location.getSize()); file.write(data.getData(), data.getOffset(), size); if (sync) { file.sync(); } }
protected long getStoredMessageSize(Transaction tx, StoredDestination sd, String subscriptionKey) throws IOException { long locationSize = 0; if (sd.ackPositions != null) { //grab the messages attached to this subscription SequenceSet messageSequences = sd.ackPositions.get(tx, subscriptionKey); if (messageSequences != null && !messageSequences.isEmpty()) { final Sequence head = messageSequences.getHead(); //get an iterator over the order index starting at the first unacked message //and go over each message to add up the size Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx, new MessageOrderCursor(head.getFirst())); final boolean contiguousRange = messageSequences.size() == 1; while (iterator.hasNext()) { Entry<Long, MessageKeys> entry = iterator.next(); //Verify sequence contains the key //if contiguous we just add all starting with the first but if not //we need to check if the id is part of the range - could happen if individual ack mode was used if (contiguousRange || messageSequences.contains(entry.getKey())) { locationSize += entry.getValue().location.getSize(); } } } } return locationSize; }
public void append(Journal.WriteCommand write) throws IOException { this.writes.addLast(write); write.location.setDataFileId(dataFile.getDataFileId()); write.location.setOffset(offset+size); int s = write.location.getSize(); size += s; dataFile.incrementLength(s); journal.addToTotalLength(s); } }
public void append(Journal.WriteCommand write) throws IOException { this.writes.addLast(write); write.location.setDataFileId(dataFile.getDataFileId()); write.location.setOffset(offset + size); int s = write.location.getSize(); size += s; dataFile.incrementLength(s); journal.addToTotalLength(s); } }
); sd.locationIndex.put(tx, location, id); incrementAndAddSizeToStoreStat(tx, command.getDestination(), location.getSize()); decrementAndSubSizeToStoreStat(tx, command.getDestination(), previousKeys.location.getSize()); if (enableSubscriptionStatistics && sd.ackPositions != null && location.getSize() != previousKeys.location.getSize()) { Iterator<Entry<String, SequenceSet>> iter = sd.ackPositions.iterator(tx); while (iter.hasNext()) { Entry<String, SequenceSet> e = iter.next(); if (e.getValue().contains(id)) { incrementAndAddSizeToStoreStat(key(command.getDestination()), e.getKey(), location.getSize()); decrementAndSubSizeToStoreStat(key(command.getDestination()), e.getKey(), previousKeys.location.getSize());
public void readLocationDetails(Location location) throws IOException { Journal.WriteCommand asyncWrite = inflightWrites.get(new Journal.WriteKey(location)); if (asyncWrite != null) { location.setSize(asyncWrite.location.getSize()); location.setType(asyncWrite.location.getType()); } else { file.seek(location.getOffset()); location.setSize(file.readInt()); location.setType(file.readByte()); } }
private boolean appendToBuffer(Journal.WriteCommand write, DataByteArrayOutputStream buff) throws IOException { buff.writeInt(write.location.getSize()); buff.writeByte(write.location.getType()); buff.write(write.data.getData(), write.data.getOffset(), write.data.getLength()); return write.sync | (syncOnComplete && write.onComplete != null); } }
private void addAckLocationForNewMessage(Transaction tx, KahaDestination kahaDest, StoredDestination sd, Long messageSequence) throws IOException { for(String subscriptionKey : sd.subscriptionCache) { SequenceSet sequences = sd.ackPositions.get(tx, subscriptionKey); if (sequences == null) { sequences = new SequenceSet(); sequences.add(new Sequence(messageSequence, messageSequence + 1)); sd.ackPositions.add(tx, subscriptionKey, sequences); } else { sequences.add(new Sequence(messageSequence, messageSequence + 1)); sd.ackPositions.put(tx, subscriptionKey, sequences); } MessageKeys key = sd.orderIndex.get(tx, messageSequence); incrementAndAddSizeToStoreStat(kahaDest, subscriptionKey, key.location.getSize()); } }
previous = sd.messageIdIndex.put(tx, command.getMessageId(), id); if (previous == null) { incrementAndAddSizeToStoreStat(tx, command.getDestination(), location.getSize()); sd.orderIndex.put(tx, priority, id, new MessageKeys(command.getMessageId(), location)); if (sd.subscriptions != null && !sd.subscriptions.isEmpty(tx)) {