public void expireFlowFiles() { final Set<FlowFileRecord> expired = new HashSet<>(); final FlowFileFilter filter = new FlowFileFilter() { @Override public FlowFileFilterResult filter(final FlowFile flowFile) { return FlowFileFilterResult.REJECT_AND_CONTINUE; } }; for (final Connection conn : context.getConnectable().getIncomingConnections()) { do { expired.clear(); conn.getFlowFileQueue().poll(filter, expired); removeExpired(expired, conn); } while (!expired.isEmpty()); } }
public Set<Relationship> getAvailableRelationships() { final Set<Relationship> set = new HashSet<>(); for (final Relationship relationship : getConnectable().getRelationships()) { final Collection<Connection> connections = getConnections(relationship); if (connections.isEmpty()) { set.add(relationship); } else { boolean available = true; for (final Connection connection : connections) { if (connection.getFlowFileQueue().isFull()) { available = false; } } if (available) { set.add(relationship); } } } return set; } }
public boolean isAnyRelationshipAvailable() { for (final Relationship relationship : getConnectable().getRelationships()) { final Collection<Connection> connections = getConnections(relationship); boolean available = true; for (final Connection connection : connections) { if (connection.getFlowFileQueue().isFull()) { available = false; break; } } if (available) { return true; } } return false; }
boolean selfRelationship = false; final int numDestinations = context.getConnections(relationship).size(); if (numDestinations == 0 && context.getConnectable().isAutoTerminated(relationship)) {
this.taskTermination = taskTermination; final Connectable connectable = context.getConnectable(); final String componentType;
newClaim = context.getContentRepository().create(context.getConnectable().isLossTolerant()); claimLog.debug("Creating ContentClaim {} for 'importFrom' for {}", newClaim, destination); } catch (final IOException e) {
final FlowFileRecord flowFile = record.getCurrent(); final long flowFileLife = System.currentTimeMillis() - flowFile.getEntryDate(); final Connectable connectable = context.getConnectable(); final Object terminator = connectable instanceof ProcessorNode ? ((ProcessorNode) connectable).getProcessor() : connectable; LOG.info("{} terminated by {}; life of FlowFile = {} ms", new Object[]{flowFile, terminator, flowFileLife});
if (originalQueue != null) { if (penalize) { final long expirationEpochMillis = System.currentTimeMillis() + context.getConnectable().getPenalizationPeriod(TimeUnit.MILLISECONDS); final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getOriginal()).penaltyExpirationTime(expirationEpochMillis).build(); originalQueue.put(newFile); final Connectable connectable = context.getConnectable(); final StandardFlowFileEvent flowFileEvent = new StandardFlowFileEvent(); flowFileEvent.setBytesRead(bytesRead);
@Override public FlowFile penalize(FlowFile flowFile) { verifyTaskActive(); flowFile = validateRecordState(flowFile); final StandardRepositoryRecord record = getRecord(flowFile); final long expirationEpochMillis = System.currentTimeMillis() + context.getConnectable().getPenalizationPeriod(TimeUnit.MILLISECONDS); final FlowFileRecord newFile = new StandardFlowFileRecord.Builder().fromFlowFile(record.getCurrent()).penaltyExpirationTime(expirationEpochMillis).build(); record.setWorking(newFile); return newFile; }
@Override public void transfer(FlowFile flowFile, final Relationship relationship) { verifyTaskActive(); flowFile = validateRecordState(flowFile); final int numDestinations = context.getConnections(relationship).size(); final int multiplier = Math.max(1, numDestinations); boolean autoTerminated = false; boolean selfRelationship = false; if (numDestinations == 0 && context.getConnectable().isAutoTerminated(relationship)) { // auto terminated. autoTerminated = true; } else if (numDestinations == 0 && relationship == Relationship.SELF) { selfRelationship = true; } else if (numDestinations == 0) { // the relationship specified is not known in this session/context throw new IllegalArgumentException("Relationship '" + relationship.getName() + "' is not known"); } final StandardRepositoryRecord record = getRecord(flowFile); record.setTransferRelationship(relationship); updateLastQueuedDate(record); if (autoTerminated) { removedCount += multiplier; removedBytes += flowFile.getSize(); } else if (!selfRelationship) { flowFilesOut += multiplier; contentSizeOut += flowFile.getSize() * multiplier; } }
private void registerForkEvent(final FlowFile parent, final FlowFile child) { ProvenanceEventBuilder eventBuilder = forkEventBuilders.get(parent); if (eventBuilder == null) { eventBuilder = context.getProvenanceRepository().eventBuilder(); eventBuilder.setEventType(ProvenanceEventType.FORK); eventBuilder.setFlowFileEntryDate(parent.getEntryDate()); eventBuilder.setLineageStartDate(parent.getLineageStartDate()); eventBuilder.setFlowFileUUID(parent.getAttribute(CoreAttributes.UUID.key())); eventBuilder.setComponentId(context.getConnectable().getIdentifier()); final Connectable connectable = context.getConnectable(); final String processorType = connectable.getComponentType(); eventBuilder.setComponentType(processorType); eventBuilder.addParentFlowFile(parent); updateEventContentClaims(eventBuilder, parent, getRecord(parent)); forkEventBuilders.put(parent, eventBuilder); } eventBuilder.addChildFlowFile(child); }
final List<RepositoryRecord> expiredRecords = new ArrayList<>(flowFiles.size()); final Connectable connectable = context.getConnectable(); final String processorType = connectable.getComponentType(); final StandardProvenanceReporter expiredReporter = new StandardProvenanceReporter(this, connectable.getIdentifier(),
if (destinations.isEmpty() && !context.getConnectable().isAutoTerminated(relationship)) { if (relationship != Relationship.SELF) { rollback(); throw new FlowFileHandlingException(relationship + " does not have any destinations for " + context.getConnectable());
final ContentClaim newClaim; try { newClaim = contentRepo.create(context.getConnectable().isLossTolerant()); claimLog.debug("Creating ContentClaim {} for 'merge' for {}", newClaim, destinationRecord.getCurrent()); } catch (final IOException e) {
final Connectable connectable = context.getConnectable(); final StandardFlowFileEvent flowFileEvent = new StandardFlowFileEvent(); flowFileEvent.setBytesRead(checkpoint.bytesRead);
try { try { newClaim = context.getContentRepository().create(context.getConnectable().isLossTolerant()); claimLog.debug("Creating ContentClaim {} for 'importFrom' for {}", newClaim, destination);
newClaim = context.getContentRepository().create(context.getConnectable().isLossTolerant()); claimLog.debug("Creating ContentClaim {} for 'append' for {}", newClaim, source);