public QueueSize toQueueSize() { return new QueueSize(activeQueueCount + swappedCount + unacknowledgedCount, activeQueueBytes + swappedBytes + unacknowledgedBytes); }
public QueueSize activeQueueSize() { return new QueueSize(activeQueueCount, activeQueueBytes); }
public QueueSize swapQueueSize() { return new QueueSize(swappedCount, swappedBytes); }
public QueueSize unacknowledgedQueueSize() { return new QueueSize(unacknowledgedCount, unacknowledgedBytes); }
public QueueSize add(final int count, final long bytes) { return new QueueSize(objectCount + count, totalSizeBytes + bytes); }
public QueueSize size() { readLock.lock(); try { final int count = queue.size(); long contentSize = 0L; for (final MockFlowFile flowFile : queue) { contentSize += flowFile.getSize(); } return new QueueSize(count, contentSize); } finally { readLock.unlock(); } }
/** * Returns a new QueueSize that is the sum of this QueueSize and the provided QueueSize * * @param other the other QueueSize to add to this QueueSize * @return a new QueueSize that is the sum of this QueueSize and the provided QueueSize */ public QueueSize add(final QueueSize other) { if (other == null) { return new QueueSize(objectCount, totalSizeBytes); } return new QueueSize(objectCount + other.getObjectCount(), totalSizeBytes + other.getByteCount()); }
@Override public QueueSize getSwapQueueSize() { return new QueueSize(queueSize.getSwappedCount(), queueSize.getSwappedBytes()); }
@Override public QueueSize getActiveQueueSize() { return new QueueSize(queueSize.getActiveCount(), queueSize.getActiveBytes()); }
@Override public QueueSize getSwapQueueSize() { return new QueueSize(queueSize.getSwappedCount(), queueSize.getSwappedBytes()); }
@Override public QueueSize getUnacknowledgedQueueSize() { return new QueueSize(queueSize.getUnacknowledgedCount(), queueSize.getUnacknowledgedCount()); }
@Override public QueueSize getActiveQueueSize() { return new QueueSize(queueSize.getActiveCount(), queueSize.getActiveBytes()); }
@Override public QueueSize getUnacknowledgedQueueSize() { return new QueueSize(queueSize.getUnacknowledgedCount(), queueSize.getUnacknowledgedCount()); }
@Override public SwapSummary getSwapSummary(final DataInputStream in, final String swapLocation, final ResourceClaimManager claimManager) throws IOException { final int swapEncodingVersion = in.readInt(); if (swapEncodingVersion > SWAP_ENCODING_VERSION) { final String errMsg = "Cannot swap FlowFiles in from " + swapLocation + " because the encoding version is " + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)"; throw new IOException(errMsg); } final int numRecords; final long contentSize; Long maxRecordId = null; try { in.readUTF(); // ignore Connection ID numRecords = in.readInt(); contentSize = in.readLong(); if (numRecords == 0) { return StandardSwapSummary.EMPTY_SUMMARY; } if (swapEncodingVersion > 7) { maxRecordId = in.readLong(); } } catch (final EOFException eof) { logger.warn("Found premature End-of-File when reading Swap File {}. EOF occurred before any FlowFiles were encountered", swapLocation); return StandardSwapSummary.EMPTY_SUMMARY; } final QueueSize queueSize = new QueueSize(numRecords, contentSize); final SwapContents swapContents = deserializeFlowFiles(in, queueSize, maxRecordId, swapEncodingVersion, claimManager, swapLocation); return swapContents.getSummary(); }
@Override public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException { final int swapEncodingVersion = in.readInt(); if (swapEncodingVersion > SWAP_ENCODING_VERSION) { throw new IOException("Cannot swap FlowFiles in from SwapFile because the encoding version is " + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)"); } final String connectionId = in.readUTF(); // Connection ID if (!connectionId.equals(queue.getIdentifier())) { throw new IllegalArgumentException("Cannot deserialize FlowFiles from Swap File at location " + swapLocation + " because those FlowFiles belong to Connection with ID " + connectionId + " and an attempt was made to swap them into a Connection with ID " + queue.getIdentifier()); } int numRecords = 0; long contentSize = 0L; Long maxRecordId = null; try { numRecords = in.readInt(); contentSize = in.readLong(); // Content Size if (swapEncodingVersion > 7) { maxRecordId = in.readLong(); // Max Record ID } } catch (final EOFException eof) { final QueueSize queueSize = new QueueSize(numRecords, contentSize); final SwapSummary summary = new StandardSwapSummary(queueSize, maxRecordId, Collections.emptyList()); final SwapContents partialContents = new StandardSwapContents(summary, Collections.emptyList()); throw new IncompleteSwapFileException(swapLocation, partialContents); } final QueueSize queueSize = new QueueSize(numRecords, contentSize); return deserializeFlowFiles(in, queueSize, maxRecordId, swapEncodingVersion, claimManager, swapLocation); }
@SuppressWarnings("unchecked") public static SwapSummary getSwapSummary(final Record record, final ResourceClaimManager claimManager) { final int flowFileCount = (Integer) record.getFieldValue(SwapSchema.FLOWFILE_COUNT); final long flowFileSize = (Long) record.getFieldValue(SwapSchema.FLOWFILE_SIZE); final QueueSize queueSize = new QueueSize(flowFileCount, flowFileSize); final long maxFlowFileId = (Long) record.getFieldValue(SwapSchema.MAX_RECORD_ID); final Map<Record, Integer> resourceClaimRecords = (Map<Record, Integer>) record.getFieldValue(SwapSchema.RESOURCE_CLAIMS); final List<ResourceClaim> resourceClaims = new ArrayList<>(); for (final Map.Entry<Record, Integer> entry : resourceClaimRecords.entrySet()) { final Record resourceClaimRecord = entry.getKey(); final ResourceClaim claim = ResourceClaimFieldMap.getResourceClaim(resourceClaimRecord, claimManager); for (int i = 0; i < entry.getValue(); i++) { resourceClaims.add(claim); } } return new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims); } }
protected QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException { // Create a Provenance Event and a FlowFile Repository record for each FlowFile final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size()); final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size()); for (final FlowFileRecord flowFile : flowFiles) { provenanceEvents.add(createDropProvenanceEvent(flowFile, requestor)); flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile)); } long dropContentSize = 0L; for (final FlowFileRecord flowFile : flowFiles) { dropContentSize += flowFile.getSize(); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim == null) { continue; } final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); if (resourceClaim == null) { continue; } resourceClaimManager.decrementClaimantCount(resourceClaim); } provRepository.registerEvents(provenanceEvents); flowFileRepository.updateRepository(flowFileRepoRecords); return new QueueSize(flowFiles.size(), dropContentSize); }
do { final FlowFileQueueSize currentSize = getFlowFileQueueSize(); swapSize = new QueueSize(currentSize.getSwappedCount() - swapQueueCount, currentSize.getSwappedBytes() - swapQueueBytes);
@Override public QueueSize getQueueSize() { verifyTaskActive(); int flowFileCount = 0; long byteCount = 0L; for (final Connection conn : context.getPollableConnections()) { final QueueSize queueSize = conn.getFlowFileQueue().size(); flowFileCount += queueSize.getObjectCount(); byteCount += queueSize.getByteCount(); } return new QueueSize(flowFileCount, byteCount); }
public QueueSize getTotalFlowFileCount(final ProcessGroup group) { int count = 0; long contentSize = 0L; for (final Connection connection : group.getConnections()) { final QueueSize size = connection.getFlowFileQueue().size(); count += size.getObjectCount(); contentSize += size.getByteCount(); } for (final ProcessGroup childGroup : group.getProcessGroups()) { final QueueSize size = getTotalFlowFileCount(childGroup); count += size.getObjectCount(); contentSize += size.getByteCount(); } return new QueueSize(count, contentSize); }