@Override public boolean isActivelyLoadBalancing() { final QueueSize size = size(); if (size.getObjectCount() == 0) { return false; } final int localObjectCount = localPartition.size().getObjectCount(); return (size.getObjectCount() > localObjectCount); }
@Override public boolean isEmpty() { return size().getObjectCount() == 0; }
public boolean isEmpty() { return size().getObjectCount() == 0; } }
@Override public boolean equals(final Object obj) { if (obj == this) { return true; } if (obj == null) { return false; } if (!(obj instanceof QueueSize)) { return false; } final QueueSize other = (QueueSize) obj; return getObjectCount() == other.getObjectCount() && getByteCount() == other.getByteCount(); } }
private int getWorkCount(final Connectable connectable) { int sum = 0; for (final Connection connection : connectable.getIncomingConnections()) { sum += connection.getFlowFileQueue().size().getObjectCount(); } return sum; }
/** * Returns a new QueueSize that is the sum of this QueueSize and the provided QueueSize * * @param other the other QueueSize to add to this QueueSize * @return a new QueueSize that is the sum of this QueueSize and the provided QueueSize */ public QueueSize add(final QueueSize other) { if (other == null) { return new QueueSize(objectCount, totalSizeBytes); } return new QueueSize(objectCount + other.getObjectCount(), totalSizeBytes + other.getByteCount()); }
private QueueSizeDTO createQueueSizeDTO(final QueueSize queueSize) { final QueueSizeDTO dto = new QueueSizeDTO(); dto.setByteCount(queueSize.getByteCount()); dto.setObjectCount(queueSize.getObjectCount()); return dto; }
private RemoteQueuePartitionDTO createRemoteQueuePartitionDto(final RemoteQueuePartitionDiagnostics queueDiagnostics) { final RemoteQueuePartitionDTO dto = new RemoteQueuePartitionDTO(); dto.setNodeIdentifier(queueDiagnostics.getNodeIdentifier()); final QueueSize activeSize = queueDiagnostics.getActiveQueueSize(); dto.setActiveQueueByteCount(activeSize.getByteCount()); dto.setActiveQueueFlowFileCount(activeSize.getObjectCount()); final QueueSize inFlightSize = queueDiagnostics.getUnacknowledgedQueueSize(); dto.setInFlightByteCount(inFlightSize.getByteCount()); dto.setInFlightFlowFileCount(inFlightSize.getObjectCount()); final QueueSize swapSize = queueDiagnostics.getSwapQueueSize(); dto.setSwapByteCount(swapSize.getByteCount()); dto.setSwapFlowFileCount(swapSize.getObjectCount()); dto.setSwapFiles(queueDiagnostics.getSwapFileCount()); dto.setTotalByteCount(activeSize.getByteCount() + inFlightSize.getByteCount() + swapSize.getByteCount()); dto.setTotalFlowFileCount(activeSize.getObjectCount() + inFlightSize.getObjectCount() + swapSize.getObjectCount()); return dto; }
public QueueSize getTotalFlowFileCount(final ProcessGroup group) { int count = 0; long contentSize = 0L; for (final Connection connection : group.getConnections()) { final QueueSize size = connection.getFlowFileQueue().size(); count += size.getObjectCount(); contentSize += size.getByteCount(); } for (final ProcessGroup childGroup : group.getProcessGroups()) { final QueueSize size = getTotalFlowFileCount(childGroup); count += size.getObjectCount(); contentSize += size.getByteCount(); } return new QueueSize(count, contentSize); }
@Override public Object getFieldValue(final String fieldName) { switch (fieldName) { case SwapSchema.MAX_RECORD_ID: return swapSummary.getMaxFlowFileId(); case SwapSchema.FLOWFILE_COUNT: return swapSummary.getQueueSize().getObjectCount(); case SwapSchema.FLOWFILE_SIZE: return swapSummary.getQueueSize().getByteCount(); case SwapSchema.QUEUE_IDENTIFIER: return queueIdentifier; case SwapSchema.RESOURCE_CLAIMS: return claimCounts; } return null; }
private LocalQueuePartitionDTO createLocalQueuePartitionDto(final LocalQueuePartitionDiagnostics queueDiagnostics) { final LocalQueuePartitionDTO dto = new LocalQueuePartitionDTO(); final QueueSize activeSize = queueDiagnostics.getActiveQueueSize(); dto.setActiveQueueByteCount(activeSize.getByteCount()); dto.setActiveQueueFlowFileCount(activeSize.getObjectCount()); final QueueSize inFlightSize = queueDiagnostics.getUnacknowledgedQueueSize(); dto.setInFlightByteCount(inFlightSize.getByteCount()); dto.setInFlightFlowFileCount(inFlightSize.getObjectCount()); final QueueSize swapSize = queueDiagnostics.getSwapQueueSize(); dto.setSwapByteCount(swapSize.getByteCount()); dto.setSwapFlowFileCount(swapSize.getObjectCount()); dto.setSwapFiles(queueDiagnostics.getSwapFileCount()); dto.setTotalByteCount(activeSize.getByteCount() + inFlightSize.getByteCount() + swapSize.getByteCount()); dto.setTotalFlowFileCount(activeSize.getObjectCount() + inFlightSize.getObjectCount() + swapSize.getObjectCount()); dto.setAllActiveQueueFlowFilesPenalized(queueDiagnostics.isAllActiveFlowFilesPenalized()); dto.setAnyActiveQueueFlowFilesPenalized(queueDiagnostics.isAnyActiveFlowFilePenalized()); return dto; }
public void inheritQueueContents(final FlowFileQueueContents queueContents) { writeLock.lock(); try { putAll(queueContents.getActiveFlowFiles()); swapLocations.addAll(queueContents.getSwapLocations()); incrementSwapQueueSize(queueContents.getSwapSize().getObjectCount(), queueContents.getSwapSize().getByteCount(), queueContents.getSwapLocations().size()); } finally { writeLock.unlock("inheritQueueContents"); } }
@Override public QueueSize getQueueSize() { verifyTaskActive(); int flowFileCount = 0; long byteCount = 0L; for (final Connection conn : context.getPollableConnections()) { final QueueSize queueSize = conn.getFlowFileQueue().size(); flowFileCount += queueSize.getObjectCount(); byteCount += queueSize.getByteCount(); } return new QueueSize(flowFileCount, byteCount); }
protected boolean isFull(final QueueSize queueSize) { final MaxQueueSize maxSize = getMaxQueueSize(); // Check if max size is set if (maxSize.getMaxBytes() <= 0 && maxSize.getMaxCount() <= 0) { return false; } if (maxSize.getMaxCount() > 0 && queueSize.getObjectCount() >= maxSize.getMaxCount()) { return true; } if (maxSize.getMaxBytes() > 0 && queueSize.getByteCount() >= maxSize.getMaxBytes()) { return true; } return false; }
@Test public void testFlowFileRollbackYield() { runner.setProperty(DebugFlow.FF_ROLLBACK_YIELD_ITERATIONS, "1"); runner.assertValid(); for (int n = 0; n < 6; n++) { runner.enqueue(contents.get(n).getBytes(), attribs.get(n)); } runner.run(7); runner.assertTransferCount(DebugFlow.REL_SUCCESS, 0); runner.assertTransferCount(DebugFlow.REL_FAILURE, 0); runner.assertQueueNotEmpty(); assertEquals(6, runner.getQueueSize().getObjectCount()); }
@Test public void testFlowFileRollbackPenalty() { runner.setProperty(DebugFlow.FF_ROLLBACK_PENALTY_ITERATIONS, "1"); runner.assertValid(); for (int n = 0; n < 6; n++) { runner.enqueue(contents.get(n).getBytes(), attribs.get(n)); } runner.run(7); runner.assertTransferCount(DebugFlow.REL_SUCCESS, 0); runner.assertTransferCount(DebugFlow.REL_FAILURE, 0); runner.assertQueueNotEmpty(); assertEquals(6, runner.getQueueSize().getObjectCount()); }
@Test public void testMaxRecords() { runner.setProperty(MergeRecord.MIN_RECORDS, "5"); runner.setProperty(MergeRecord.MAX_RECORDS, "10"); for (int i = 0; i < 34; i++) { runner.enqueue("Name, Age\nJohn, 35"); } runner.run(); runner.assertTransferCount(MergeRecord.REL_MERGED, 3); runner.assertTransferCount(MergeRecord.REL_ORIGINAL, 30); assertEquals(4, runner.getQueueSize().getObjectCount()); runner.getFlowFilesForRelationship(MergeRecord.REL_MERGED).stream().forEach(ff -> ff.assertAttributeEquals("record.count", "10")); }
@Test public void testFlowFileRollback() throws IOException { runner.setProperty(DebugFlow.FF_ROLLBACK_ITERATIONS, "1"); runner.assertValid(); for (int n = 0; n < 6; n++) { runner.enqueue(contents.get(n).getBytes(), attribs.get(n)); } runner.run(7); runner.assertTransferCount(DebugFlow.REL_SUCCESS, 0); runner.assertTransferCount(DebugFlow.REL_FAILURE, 0); runner.assertQueueNotEmpty(); assertEquals(6, runner.getQueueSize().getObjectCount()); MockFlowFile ff1 = (MockFlowFile) session.get(); assertNotNull(ff1); assertEquals(namesToContent.get(ff1.getAttribute(CoreAttributes.FILENAME.key())), new String(ff1.toByteArray())); session.rollback(); }
@Test public void testMaxSize() { runner.setProperty(MergeRecord.MIN_RECORDS, "5"); runner.setProperty(MergeRecord.MAX_SIZE, "100 B"); for (int i = 0; i < 36; i++) { runner.enqueue("Name, Age\nJohnny, 5"); } runner.run(); runner.assertTransferCount(MergeRecord.REL_MERGED, 3); runner.assertTransferCount(MergeRecord.REL_ORIGINAL, 33); assertEquals(3, runner.getQueueSize().getObjectCount()); }
@Test public void testLeavesSmallBinUnmerged() { final TestRunner runner = TestRunners.newTestRunner(new MergeContent()); runner.setProperty(MergeContent.MIN_ENTRIES, "5"); runner.setProperty(MergeContent.MAX_ENTRIES, "5"); runner.setProperty(MergeContent.MAX_BIN_COUNT, "3"); for (int i = 0; i < 17; i++) { runner.enqueue(String.valueOf(i) + "\n"); } runner.run(5); runner.assertTransferCount(MergeContent.REL_MERGED, 3); runner.assertTransferCount(MergeContent.REL_ORIGINAL, 15); assertEquals(2, runner.getQueueSize().getObjectCount()); }