@Override public String toString() { return "SwappablePriorityQueue[queueId=" + flowFileQueue.getIdentifier() + "]"; } }
private String getQueueIdentifier() { return flowFileQueue.getIdentifier(); }
public SwappablePriorityQueueLocalPartition(final FlowFileSwapManager swapManager, final int swapThreshold, final EventReporter eventReporter, final FlowFileQueue flowFileQueue, final DropFlowFileAction dropAction) { this.priorityQueue = new SwappablePriorityQueue(swapManager, swapThreshold, eventReporter, flowFileQueue, dropAction, SWAP_PARTITION_NAME); this.flowFileQueue = flowFileQueue; this.description = "SwappablePriorityQueueLocalPartition[queueId=" + flowFileQueue.getIdentifier() + "]"; }
@Override public Set<String> getSwappedPartitionNames(final FlowFileQueue queue) { final File[] swapFiles = storageDirectory.listFiles(new FilenameFilter() { @Override public boolean accept(final File dir, final String name) { return SWAP_FILE_PATTERN.matcher(name).matches(); } }); if (swapFiles == null) { return Collections.emptySet(); } final String queueId = queue.getIdentifier(); return Stream.of(swapFiles) .filter(swapFile -> queueId.equals(getOwnerQueueIdentifier(swapFile))) .map(this::getOwnerPartition) .filter(Objects::nonNull) .collect(Collectors.toSet()); }
public SwappablePriorityQueue(final FlowFileSwapManager swapManager, final int swapThreshold, final EventReporter eventReporter, final FlowFileQueue flowFileQueue, final DropFlowFileAction dropAction, final String swapPartitionName) { this.swapManager = swapManager; this.swapThreshold = swapThreshold; this.activeQueue = new PriorityQueue<>(20, new QueuePrioritizer(Collections.emptyList())); this.swapQueue = new ArrayList<>(); this.eventReporter = eventReporter; this.flowFileQueue = flowFileQueue; this.dropAction = dropAction; this.swapPartitionName = swapPartitionName; final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); readLock = new TimedLock(lock.readLock(), flowFileQueue.getIdentifier() + " Read Lock", 100); writeLock = new TimedLock(lock.writeLock(), flowFileQueue.getIdentifier() + " Write Lock", 100); }
final String swapFilePrefix = System.currentTimeMillis() + "-" + flowFileQueue.getIdentifier() + "-" + UUID.randomUUID().toString(); final String swapFileBaseName = partitionName == null ? swapFilePrefix : swapFilePrefix + "." + partitionName; final String swapFileName = swapFileBaseName + ".swap";
details.append(", "); if (repoRecord.getOriginalQueue() != null && repoRecord.getOriginalQueue().getIdentifier() != null) { details.append("queue=") .append(repoRecord.getOriginalQueue().getIdentifier()) .append("/");
@Override public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException { final int swapEncodingVersion = in.readInt(); if (swapEncodingVersion > SWAP_ENCODING_VERSION) { throw new IOException("Cannot swap FlowFiles in from SwapFile because the encoding version is " + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)"); } final String connectionId = in.readUTF(); // Connection ID if (!connectionId.equals(queue.getIdentifier())) { throw new IllegalArgumentException("Cannot deserialize FlowFiles from Swap File at location " + swapLocation + " because those FlowFiles belong to Connection with ID " + connectionId + " and an attempt was made to swap them into a Connection with ID " + queue.getIdentifier()); } int numRecords = 0; long contentSize = 0L; Long maxRecordId = null; try { numRecords = in.readInt(); contentSize = in.readLong(); // Content Size if (swapEncodingVersion > 7) { maxRecordId = in.readLong(); // Max Record ID } } catch (final EOFException eof) { final QueueSize queueSize = new QueueSize(numRecords, contentSize); final SwapSummary summary = new StandardSwapSummary(queueSize, maxRecordId, Collections.emptyList()); final SwapContents partialContents = new StandardSwapContents(summary, Collections.emptyList()); throw new IncompleteSwapFileException(swapLocation, partialContents); } final QueueSize queueSize = new QueueSize(numRecords, contentSize); return deserializeFlowFiles(in, queueSize, maxRecordId, swapEncodingVersion, claimManager, swapLocation); }
if (!ownerQueueId.equals(flowFileQueue.getIdentifier())) { continue; if (connectionId.equals(flowFileQueue.getIdentifier())) { swapLocations.add(swapFile.getAbsolutePath());
try { out.writeInt(SWAP_ENCODING_VERSION); out.writeUTF(queue.getIdentifier()); out.writeInt(toSwap.size()); out.writeLong(contentSize);
final Record summaryRecord = new SwapSummaryFieldMap(swapSummary, queue.getIdentifier(), SwapSchema.SWAP_SUMMARY_SCHEMA_V1);
final String originalQueueId = repoRecord.getOriginalQueue().getIdentifier(); final Connection destinationConnection = connectionsForRelationship.iterator().next(); final String destinationQueueId = destinationConnection.getFlowFileQueue().getIdentifier(); return originalQueueId.equals(destinationQueueId);
final String connectionId = inputQueue.getIdentifier(); incrementConnectionOutputCounts(connectionId, -1, -repoRecord.getOriginal().getSize()); newOwner.incrementConnectionOutputCounts(connectionId, 1, repoRecord.getOriginal().getSize());
final Map<String, FlowFileQueue> queueMap = new HashMap<>(); for (final FlowFileQueue queue : queueProvider.getAllQueues()) { queueMap.put(queue.getIdentifier(), queue);
out.write(ACTION_SWAPPED_OUT); out.writeLong(getRecordIdentifier(record)); out.writeUTF(associatedQueue.getIdentifier()); out.writeUTF(getLocation(record)); return; writeString("", out); } else { writeString(associatedQueue.getIdentifier(), out);
recordBuilder.setSourceQueueIdentifier(originalQueue.getIdentifier());
case RepositoryRecordSchema.QUEUE_IDENTIFIER: final FlowFileQueue queue = record.getDestination() == null ? record.getOriginalQueue() : record.getDestination(); return queue == null ? null : queue.getIdentifier(); default: return null;
recordBuilder.setSourceQueueIdentifier(originalQueue.getIdentifier());
out.write(ACTION_SWAPPED_OUT); out.writeLong(getRecordIdentifier(record)); out.writeUTF(associatedQueue.getIdentifier()); out.writeUTF(getLocation(record)); return; writeString("", out); } else { writeString(associatedQueue.getIdentifier(), out);
case RepositoryRecordSchema.QUEUE_IDENTIFIER: final FlowFileQueue queue = record.getDestination() == null ? record.getOriginalQueue() : record.getDestination(); return queue == null ? null : queue.getIdentifier(); default: return null;