private long drainQueue(final Queue<FlowFileRecord> sourceQueue, final List<FlowFileRecord> destination, int maxResults, final Set<FlowFileRecord> expiredRecords, final long expirationMillis) { long drainedSize = 0L; FlowFileRecord pulled; while (destination.size() < maxResults && (pulled = sourceQueue.poll()) != null) { if (isExpired(pulled, expirationMillis)) { expiredRecords.add(pulled); if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) { break; } } else { if (pulled.isPenalized()) { sourceQueue.add(pulled); break; } destination.add(pulled); } drainedSize += pulled.getSize(); } return drainedSize; }
public LocalQueuePartitionDiagnostics getQueueDiagnostics() { readLock.lock(); try { final boolean anyPenalized = !activeQueue.isEmpty() && activeQueue.peek().isPenalized(); final boolean allPenalized = anyPenalized && activeQueue.stream().anyMatch(FlowFileRecord::isPenalized); return new StandardLocalQueuePartitionDiagnostics(getFlowFileQueueSize(), anyPenalized, allPenalized); } finally { readLock.unlock("getQueueDiagnostics"); } }
private FlowFileRecord doPoll(final Set<FlowFileRecord> expiredRecords, final long expirationMillis) { FlowFileRecord flowFile; boolean isExpired; migrateSwapToActive(); long expiredBytes = 0L; do { flowFile = this.activeQueue.poll(); isExpired = isExpired(flowFile, expirationMillis); if (isExpired) { expiredRecords.add(flowFile); expiredBytes += flowFile.getSize(); flowFile = null; if (expiredRecords.size() >= MAX_EXPIRED_RECORDS_PER_ITERATION) { break; } } else if (flowFile != null && flowFile.isPenalized()) { this.activeQueue.add(flowFile); flowFile = null; break; } if (flowFile != null) { incrementActiveQueueSize(-1, -flowFile.getSize()); } } while (isExpired); if (!expiredRecords.isEmpty()) { incrementActiveQueueSize(-expiredRecords.size(), -expiredBytes); } return flowFile; }
continue; } else if (flowFile.isPenalized()) { this.activeQueue.add(flowFile); break; // just stop searching because the rest are all penalized.
@Override public int compare(final FlowFileRecord f1, final FlowFileRecord f2) { int returnVal = 0; final boolean f1Penalized = f1.isPenalized(); final boolean f2Penalized = f2.isPenalized();
/** * Creates a FlowFileDTO from the specified FlowFileRecord. * * @param record record * @return dto */ public FlowFileDTO createFlowFileDTO(final FlowFileRecord record) { final Date now = new Date(); final FlowFileDTO dto = new FlowFileDTO(); dto.setUuid(record.getAttribute(CoreAttributes.UUID.key())); dto.setFilename(record.getAttribute(CoreAttributes.FILENAME.key())); dto.setPenalized(record.isPenalized()); dto.setSize(record.getSize()); dto.setAttributes(record.getAttributes()); final long queuedDuration = now.getTime() - record.getLastQueueDate(); dto.setQueuedDuration(queuedDuration); final long age = now.getTime() - record.getLineageStartDate(); dto.setLineageDuration(age); final ContentClaim contentClaim = record.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); dto.setContentClaimSection(resourceClaim.getSection()); dto.setContentClaimContainer(resourceClaim.getContainer()); dto.setContentClaimIdentifier(resourceClaim.getId()); dto.setContentClaimOffset(contentClaim.getOffset() + record.getContentClaimOffset()); dto.setContentClaimFileSizeBytes(record.getSize()); dto.setContentClaimFileSize(FormatUtils.formatDataSize(record.getSize())); } return dto; }