@Override public void acknowledge(final FlowFileRecord flowFile) { priorityQueue.acknowledge(flowFile); }
private void doPoll(final List<FlowFileRecord> records, int maxResults, final Set<FlowFileRecord> expiredRecords, final long expirationMillis) { migrateSwapToActive(); final long bytesDrained = drainQueue(activeQueue, records, maxResults, expiredRecords, expirationMillis); long expiredBytes = 0L; for (final FlowFileRecord record : expiredRecords) { expiredBytes += record.getSize(); } incrementActiveQueueSize(-(expiredRecords.size() + records.size()), -bytesDrained); incrementUnacknowledgedQueueSize(records.size(), bytesDrained - expiredBytes); }
@Override protected void dropFlowFiles(final DropFlowFileRequest dropRequest, final String requestor) { queue.dropFlowFiles(dropRequest, requestor); }
protected boolean isExpired(final FlowFile flowFile, final long expirationMillis) { return isLaterThan(getExpirationDate(flowFile, expirationMillis)); }
migrateSwapToActive(); final PriorityQueue<FlowFileRecord> tempQueue = new PriorityQueue<>(activeQueue.size() + swapQueue.size(), Collections.reverseOrder(new QueuePrioritizer(getPriorities()))); tempQueue.addAll(activeQueue); tempQueue.addAll(swapQueue); final int objectCount = getFlowFileCount(); logger.error("FlowFile Queue with identifier {} has {} FlowFiles queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting " + "the Java heap space but failed to write information to disk due to {}", getQueueIdentifier(), objectCount, ioe.toString()); logger.error("", ioe); if (eventReporter != null) { eventReporter.reportEvent(Severity.ERROR, "Failed to Overflow to Disk", "Flowfile Queue with identifier " + getQueueIdentifier() + " has " + objectCount + " queued up. Attempted to spill FlowFile information over to disk in order to avoid exhausting the Java heap space but failed to write information to disk. " + "See logs for more information."); final FlowFileQueueSize originalSize = getFlowFileQueueSize(); originalSize.getSwapFileCount() + numSwapFiles, originalSize.getUnacknowledgedCount(), originalSize.getUnacknowledgedBytes()); updated = updateSize(originalSize, newSize); logIfNegative(originalSize, newSize, "swap");
try { dropRequest.setState(DropFlowFileState.DROPPING_FLOWFILES); logger.debug("For DropFlowFileRequest {}, original size is {}", requestIdentifier, size()); logger.debug("For DropFlowFileRequest {}, Dropped {} from active queue", requestIdentifier, droppedSize); } catch (final IOException ioe) { logger.error("Failed to drop the FlowFiles from queue {} due to {}", getQueueIdentifier(), ioe.toString()); logger.error("", ioe); incrementActiveQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount()); dropRequest.setCurrentSize(size()); dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize)); final QueueSize swapSize = getFlowFileQueueSize().swapQueueSize(); logger.debug("For DropFlowFileRequest {}, Swap Queue has {} elements, Swapped Record Count = {}, Swapped Content Size = {}", requestIdentifier, swapQueue.size(), swapSize.getObjectCount(), swapSize.getByteCount()); droppedSize = dropAction.drop(swapQueue, requestor); } catch (final IOException ioe) { logger.error("Failed to drop the FlowFiles from queue {} due to {}", getQueueIdentifier(), ioe.toString()); logger.error("", ioe); dropRequest.setCurrentSize(size()); dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize)); swapMode = false; incrementSwapQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount(), 0); logger.debug("For DropFlowFileRequest {}, dropped {} from Swap Queue", requestIdentifier, droppedSize); swapLocation, getQueueIdentifier(), ioe.toString());
swapIn(); return; final FlowFileQueueSize size = getFlowFileQueueSize(); if (size.getSwappedCount() == 0 && swapQueue.isEmpty()) { return; incrementActiveQueueSize(recordsMigrated, bytesMigrated); incrementSwapQueueSize(-recordsMigrated, -bytesMigrated, 0);
private void incrementSwapQueueSize(final int count, final long bytes, final int fileCount) { boolean updated = false; while (!updated) { final FlowFileQueueSize original = getFlowFileQueueSize(); final FlowFileQueueSize newSize = new FlowFileQueueSize(original.getActiveCount(), original.getActiveBytes(), original.getSwappedCount() + count, original.getSwappedBytes() + bytes, original.getSwapFileCount() + fileCount, original.getUnacknowledgedCount(), original.getUnacknowledgedBytes()); updated = updateSize(original, newSize); if (updated) { logIfNegative(original, newSize, "swap"); } } }
@Override public boolean isEmpty() { return queue.getFlowFileQueueSize().isEmpty(); }
public FlowFileRecord poll(final Set<FlowFileRecord> expiredRecords, final long expirationMillis) { FlowFileRecord flowFile; // First check if we have any records Pre-Fetched. writeLock.lock(); try { flowFile = doPoll(expiredRecords, expirationMillis); if (flowFile != null) { logger.debug("{} poll() returning {}", this, flowFile); incrementUnacknowledgedQueueSize(1, flowFile.getSize()); } return flowFile; } finally { writeLock.unlock("poll(Set)"); } }
boolean updated; do { final FlowFileQueueSize currentSize = getFlowFileQueueSize(); swapSize = new QueueSize(currentSize.getSwappedCount() - swapQueueCount, currentSize.getSwappedBytes() - swapQueueBytes); updated = updateSize(currentSize, updatedSize); } while (!updated);
public SwappablePriorityQueueLocalPartition(final FlowFileSwapManager swapManager, final int swapThreshold, final EventReporter eventReporter, final FlowFileQueue flowFileQueue, final DropFlowFileAction dropAction) { this.priorityQueue = new SwappablePriorityQueue(swapManager, swapThreshold, eventReporter, flowFileQueue, dropAction, SWAP_PARTITION_NAME); this.flowFileQueue = flowFileQueue; this.description = "SwappablePriorityQueueLocalPartition[queueId=" + flowFileQueue.getIdentifier() + "]"; }
@Override public FlowFileRecord getFlowFile(final String flowFileUuid) throws IOException { return queue.getFlowFile(flowFileUuid); }
@Override protected List<FlowFileRecord> getListableFlowFiles() { return queue.getActiveFlowFiles(); }
public List<FlowFileRecord> poll(int maxResults, final Set<FlowFileRecord> expiredRecords, final long expirationMillis) { final List<FlowFileRecord> records = new ArrayList<>(Math.min(1, maxResults)); // First check if we have any records Pre-Fetched. writeLock.lock(); try { doPoll(records, maxResults, expiredRecords, expirationMillis); } finally { writeLock.unlock("poll(int, Set)"); } if (!records.isEmpty()) { logger.debug("{} poll() returning {}", this, records); } return records; }
public boolean isEmpty() { return getFlowFileQueueSize().isEmpty(); }
public StandardFlowFileQueue(final String identifier, final ConnectionEventListener eventListener, final FlowFileRepository flowFileRepo, final ProvenanceEventRepository provRepo, final ResourceClaimManager resourceClaimManager, final ProcessScheduler scheduler, final FlowFileSwapManager swapManager, final EventReporter eventReporter, final int swapThreshold, final long defaultBackPressureObjectThreshold, final String defaultBackPressureDataSizeThreshold) { super(identifier, scheduler, flowFileRepo, provRepo, resourceClaimManager); this.swapManager = swapManager; this.queue = new SwappablePriorityQueue(swapManager, swapThreshold, eventReporter, this, this::drop, null); this.eventListener = eventListener; writeLock = new TimedLock(this.lock.writeLock(), getIdentifier() + " Write Lock", 100); setBackPressureDataSizeThreshold(defaultBackPressureDataSizeThreshold); setBackPressureObjectThreshold(defaultBackPressureObjectThreshold); }
@Override public FlowFileRecord getFlowFile(final String flowFileUuid) throws IOException { return priorityQueue.getFlowFile(flowFileUuid); }
@Override public List<FlowFileRecord> getListableFlowFiles() { return priorityQueue.getActiveFlowFiles(); }
public QueueSize size() { return getFlowFileQueueSize().toQueueSize(); }