@Override public void run() { dropFlowFiles(dropRequest, requestor); } }, "Drop FlowFiles for Connection " + getIdentifier());
@Override public long getBackPressureObjectThreshold() { return getMaxQueueSize().getMaxCount(); }
@Override public void run() { int position = 0; final List<FlowFileSummary> summaries = new ArrayList<>(); // Create an ArrayList that contains all of the contents of the active queue. // We do this so that we don't have to hold the lock any longer than absolutely necessary. // We cannot simply pull the first 'maxResults' records from the queue, however, because the // Iterator provided by PriorityQueue does not return records in order. So we would have to either // use a writeLock and 'pop' the first 'maxResults' records off the queue or use a read lock and // do a shallow copy of the queue. The shallow copy is generally quicker because it doesn't have to do // the sorting to put the records back. So even though this has an expensive of Java Heap to create the // extra collection, we are making this trade-off to avoid locking the queue any longer than required. final List<FlowFileRecord> allFlowFiles = getListableFlowFiles(); final QueuePrioritizer prioritizer = new QueuePrioritizer(getPriorities()); listRequest.setState(ListFlowFileState.CALCULATING_LIST); // sort the FlowFileRecords so that we have the list in the same order as on the queue. allFlowFiles.sort(prioritizer); for (final FlowFileRecord flowFile : allFlowFiles) { summaries.add(summarize(flowFile, ++position)); if (summaries.size() >= maxResults) { break; } } logger.debug("{} Finished listing FlowFiles for active queue with a total of {} results out of {} FlowFiles", this, summaries.size(), allFlowFiles.size()); listRequest.setFlowFileSummaries(summaries); listRequest.setState(ListFlowFileState.COMPLETE); } }, "List FlowFiles for Connection " + getIdentifier());
final ListFlowFileRequest listRequest = new ListFlowFileRequest(requestIdentifier, maxResults, size()); listRequest.setState(ListFlowFileState.COMPLETE); }, "List FlowFiles for Connection " + getIdentifier()); t.setDaemon(true); t.start();
protected QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException { // Create a Provenance Event and a FlowFile Repository record for each FlowFile final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size()); final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size()); for (final FlowFileRecord flowFile : flowFiles) { provenanceEvents.add(createDropProvenanceEvent(flowFile, requestor)); flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile)); } long dropContentSize = 0L; for (final FlowFileRecord flowFile : flowFiles) { dropContentSize += flowFile.getSize(); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim == null) { continue; } final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); if (resourceClaim == null) { continue; } resourceClaimManager.decrementClaimantCount(resourceClaim); } provRepository.registerEvents(provenanceEvents); flowFileRepository.updateRepository(flowFileRepoRecords); return new QueueSize(flowFiles.size(), dropContentSize); }
@Override public synchronized void setLoadBalanceStrategy(final LoadBalanceStrategy strategy, final String partitioningAttribute) { final LoadBalanceStrategy currentStrategy = getLoadBalanceStrategy(); final String currentPartitioningAttribute = getPartitioningAttribute(); super.setLoadBalanceStrategy(strategy, partitioningAttribute); if (strategy == currentStrategy && Objects.equals(partitioningAttribute, currentPartitioningAttribute)) { // Nothing changed. return; } if (clusterCoordinator == null) { // Not clustered so nothing to worry about. return; } if (!offloaded) { // We are already load balancing but are changing how we are load balancing. final FlowFilePartitioner partitioner; partitioner = getPartitionerForLoadBalancingStrategy(strategy, partitioningAttribute); setFlowFilePartitioner(partitioner); } }
private ProvenanceEventRecord createDropProvenanceEvent(final FlowFileRecord flowFile, final String requestor) { final ProvenanceEventBuilder builder = provRepository.eventBuilder(); builder.fromFlowFile(flowFile); builder.setEventType(ProvenanceEventType.DROP); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(getIdentifier()); builder.setComponentType("Connection"); builder.setAttributes(flowFile.getAttributes(), Collections.emptyMap()); builder.setDetails("FlowFile Queue emptied by " + requestor); builder.setSourceQueueIdentifier(getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset(), flowFile.getSize()); } return builder.build(); }
final QueueSize originalSize = size(); dropRequest.setCurrentSize(originalSize); dropRequest.setOriginalSize(originalSize); dropFlowFiles(dropRequest, requestor); }, "Drop FlowFiles for Connection " + getIdentifier()); t.setDaemon(true); t.start();
@Override public String getBackPressureDataSizeThreshold() { return getMaxQueueSize().getMaxSize(); }
@Override public void setBackPressureDataSizeThreshold(final String maxDataSize) { final long maxBytes = DataUnit.parseDataSize(maxDataSize, DataUnit.B).longValue(); boolean updated = false; while (!updated) { MaxQueueSize maxSize = getMaxQueueSize(); final MaxQueueSize updatedSize = new MaxQueueSize(maxDataSize, maxBytes, maxSize.getMaxCount()); updated = maxQueueSize.compareAndSet(maxSize, updatedSize); } }
@Override public void setBackPressureObjectThreshold(final long threshold) { boolean updated = false; while (!updated) { MaxQueueSize maxSize = getMaxQueueSize(); final MaxQueueSize updatedSize = new MaxQueueSize(maxSize.getMaxSize(), maxSize.getMaxBytes(), threshold); updated = maxQueueSize.compareAndSet(maxSize, updatedSize); } }
protected boolean isFull(final QueueSize queueSize) { final MaxQueueSize maxSize = getMaxQueueSize(); // Check if max size is set if (maxSize.getMaxBytes() <= 0 && maxSize.getMaxCount() <= 0) { return false; } if (maxSize.getMaxCount() > 0 && queueSize.getObjectCount() >= maxSize.getMaxCount()) { return true; } if (maxSize.getMaxBytes() > 0 && queueSize.getByteCount() >= maxSize.getMaxBytes()) { return true; } return false; }