private void mapToConnectableId(final Collection<? extends Connectable> connectables, final Map<String, List<Connectable>> destination) { for (final Connectable connectable : connectables) { final Optional<String> versionedIdOption = connectable.getVersionedComponentId(); // Determine the Versioned ID by using the ID that is assigned, if one is. Otherwise, // we will calculate the Versioned ID. This allows us to map connectables that currently are not under // version control. We have to do this so that if we are changing flow versions and have a component that is running and it does not exist // in the Versioned Flow, we still need to be able to create an AffectedComponentDTO for it. final String versionedId; if (versionedIdOption.isPresent()) { versionedId = versionedIdOption.get(); } else { versionedId = UUID.nameUUIDFromBytes(connectable.getIdentifier().getBytes(StandardCharsets.UTF_8)).toString(); } final List<Connectable> byVersionedId = destination.computeIfAbsent(versionedId, key -> new ArrayList<>()); byVersionedId.add(connectable); } }
@Override public String toString() { return "Connection[ID=" + getIdentifier() + ", Source ID=" + getSource().getIdentifier() + ", Dest ID=" + getDestination().getIdentifier() + "]"; }
private boolean isInputPort(final Connectable connectable) { if (connectable.getConnectableType() != ConnectableType.INPUT_PORT) { return false; } return findInputPort(connectable.getIdentifier()) != null; }
private boolean isOutputPort(final Connectable connectable) { if (connectable.getConnectableType() != ConnectableType.OUTPUT_PORT) { return false; } return findOutputPort(connectable.getIdentifier()) != null; }
@Override public void verifyCanStart(Connectable connectable) { readLock.lock(); try { if (connectable.getScheduledState() == ScheduledState.STOPPED) { if (scheduler.getActiveThreadCount(connectable) > 0) { throw new IllegalStateException("Cannot start component with id" + connectable.getIdentifier() + " because it is currently stopping"); } connectable.verifyCanStart(); } } finally { readLock.unlock(); } }
@Override public void verifyCanDelete() { if (!flowFileQueue.isEmpty()) { throw new IllegalStateException("Queue not empty for " + this.getIdentifier()); } if (source.isRunning()) { if (!ConnectableType.FUNNEL.equals(source.getConnectableType())) { throw new IllegalStateException("Source of Connection (" + source.getIdentifier() + ") is running"); } } final Connectable dest = destination.get(); if (dest.isRunning()) { if (!ConnectableType.FUNNEL.equals(dest.getConnectableType())) { throw new IllegalStateException("Destination of Connection (" + dest.getIdentifier() + ") is running"); } } }
void adjustCounter(final String name, final long delta) { final String localContext = connectable.getName() + " (" + connectable.getIdentifier() + ")"; final String globalContext = "All " + connectable.getComponentType() + "'s"; counterRepo.adjustCounter(localContext, name, delta); counterRepo.adjustCounter(globalContext, name, delta); }
private synchronized void startConnectable(final Connectable connectable) { if (connectable.getScheduledState() == ScheduledState.DISABLED) { throw new IllegalStateException(connectable.getIdentifier() + " is disabled, so it cannot be started"); } final LifecycleState lifecycleState = getLifecycleState(requireNonNull(connectable), true); if (lifecycleState.isScheduled()) { return; } final int activeThreads = lifecycleState.getActiveThreadCount(); if (activeThreads > 0) { throw new IllegalStateException("Port cannot be scheduled to run until its last " + activeThreads + " threads finish"); } lifecycleState.clearTerminationFlag(); getSchedulingAgent(connectable).schedule(connectable, lifecycleState); lifecycleState.setScheduled(true); }
/** * Returns the status history for the specified connection. * * @param connectionId connection id * @return status history */ public StatusHistoryDTO getConnectionStatusHistory(final String connectionId) { final ProcessGroup root = getRootGroup(); final Connection connection = root.findConnection(connectionId); // ensure the connection was found if (connection == null) { throw new ResourceNotFoundException(String.format("Unable to locate connection with id '%s'.", connectionId)); } final StatusHistoryDTO statusHistory = flowController.getConnectionStatusHistory(connectionId); // if not authorized if (!connection.isAuthorized(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser())) { statusHistory.getComponentDetails().put(ComponentStatusRepository.COMPONENT_DETAIL_NAME, connectionId); statusHistory.getComponentDetails().put(ComponentStatusRepository.COMPONENT_DETAIL_SOURCE_NAME, connection.getSource().getIdentifier()); statusHistory.getComponentDetails().put(ComponentStatusRepository.COMPONENT_DETAIL_DESTINATION_NAME, connection.getDestination().getIdentifier()); } return statusHistory; }
private Authorizable getAuthorizable(final Connectable connectable) { switch (connectable.getConnectableType()) { case REMOTE_INPUT_PORT: case REMOTE_OUTPUT_PORT: final String rpgId = ((RemoteGroupPort) connectable).getRemoteProcessGroup().getIdentifier(); return authorizableLookup.getRemoteProcessGroup(rpgId); default: return authorizableLookup.getLocalConnectable(connectable.getIdentifier()); } }
private synchronized void stopConnectable(final Connectable connectable) { final LifecycleState state = getLifecycleState(requireNonNull(connectable), false); if (!state.isScheduled()) { return; } state.setScheduled(false); getSchedulingAgent(connectable).unschedule(connectable, state); if (!state.isScheduled() && state.getActiveThreadCount() == 0 && state.mustCallOnStoppedMethods()) { final ConnectableProcessContext processContext = new ConnectableProcessContext(connectable, encryptor, getStateManager(connectable.getIdentifier())); try (final NarCloseable x = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), connectable.getClass(), connectable.getIdentifier())) { ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, connectable, processContext); } } }
try (final AutoCloseable ncl = NarCloseable.withComponentNarLoader(extensionManager, worker.getClass(), worker.getIdentifier())) { worker.onTrigger(processContext, sessionFactory); } catch (final ProcessException pe) { try (final NarCloseable x = NarCloseable.withComponentNarLoader(extensionManager, worker.getClass(), worker.getIdentifier())) { ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, worker, processContext);
this.provenanceReporter = new StandardProvenanceReporter(this, connectable.getIdentifier(), componentType, context.getProvenanceRepository(), this); this.sessionId = idGenerator.getAndIncrement();
public ConnectableTask(final SchedulingAgent schedulingAgent, final Connectable connectable, final FlowController flowController, final RepositoryContextFactory contextFactory, final LifecycleState scheduleState, final StringEncryptor encryptor) { this.schedulingAgent = schedulingAgent; this.connectable = connectable; this.scheduleState = scheduleState; this.numRelationships = connectable.getRelationships().size(); this.flowController = flowController; final StateManager stateManager = new TaskTerminationAwareStateManager(flowController.getStateManagerProvider().getStateManager(connectable.getIdentifier()), scheduleState::isTerminated); if (connectable instanceof ProcessorNode) { processContext = new StandardProcessContext((ProcessorNode) connectable, flowController.getControllerServiceProvider(), encryptor, stateManager, scheduleState::isTerminated); } else { processContext = new ConnectableProcessContext(connectable, encryptor, stateManager); } repositoryContext = contextFactory.newProcessContext(connectable, new AtomicLong(0L)); }
/** * Creates action details for connect/disconnect actions. * * @param connection connection * @param source source * @param relationships relationships * @param destination destinations * @return details */ public ConnectDetails createConnectDetails(final Connection connection, final Connectable source, final Collection<Relationship> relationships, final Connectable destination) { final Component sourceType = determineConnectableType(source); final Component destiantionType = determineConnectableType(destination); // format the relationship names Collection<String> relationshipNames = new HashSet<>(connection.getRelationships().size()); for (final Relationship relationship : relationships) { relationshipNames.add(relationship.getName()); } final String formattedRelationships = relationshipNames.isEmpty() ? StringUtils.EMPTY : StringUtils.join(relationshipNames, ", "); // create the connect details final FlowChangeConnectDetails connectDetails = new FlowChangeConnectDetails(); connectDetails.setSourceId(source.getIdentifier()); connectDetails.setSourceName(source.getName()); connectDetails.setSourceType(sourceType); connectDetails.setRelationship(formattedRelationships); connectDetails.setDestinationId(destination.getIdentifier()); connectDetails.setDestinationName(destination.getName()); connectDetails.setDestinationType(destiantionType); return connectDetails; }
public ConnectableComponent mapConnectable(final Connectable connectable) { final ConnectableComponent component = new InstantiatedConnectableComponent(connectable.getIdentifier(), connectable.getProcessGroupIdentifier()); final String versionedId = getIdOrThrow(connectable.getVersionedComponentId(), connectable.getIdentifier(), () -> new IllegalArgumentException("Unable to map Connectable Component with identifier " + connectable.getIdentifier() + " to any version-controlled component")); component.setId(versionedId); component.setComments(connectable.getComments()); final String groupId; if (connectable instanceof RemoteGroupPort) { final RemoteGroupPort port = (RemoteGroupPort) connectable; final RemoteProcessGroup rpg = port.getRemoteProcessGroup(); final Optional<String> rpgVersionedId = rpg.getVersionedComponentId(); groupId = getIdOrThrow(rpgVersionedId, rpg.getIdentifier(), () -> new IllegalArgumentException("Unable to find the Versioned Component ID for Remote Process Group that " + connectable + " belongs to")); } else { groupId = getIdOrThrow(connectable.getProcessGroup().getVersionedComponentId(), connectable.getProcessGroupIdentifier(), () -> new IllegalArgumentException("Unable to find the Versioned Component ID for the Process Group that " + connectable + " belongs to")); } component.setGroupId(groupId); component.setName(connectable.getName()); component.setType(ConnectableComponentType.valueOf(connectable.getConnectableType().name())); return component; }
public static Bulletin createBulletin(final Connectable connectable, final String category, final String severity, final String message) { final ComponentType type; switch (connectable.getConnectableType()) { case REMOTE_INPUT_PORT: case REMOTE_OUTPUT_PORT: type = ComponentType.REMOTE_PROCESS_GROUP; break; case INPUT_PORT: type = ComponentType.INPUT_PORT; break; case OUTPUT_PORT: type = ComponentType.OUTPUT_PORT; break; case PROCESSOR: default: type = ComponentType.PROCESSOR; break; } final ProcessGroup group = connectable.getProcessGroup(); final String groupId = group == null ? null : group.getIdentifier(); final String groupName = group == null ? null : group.getName(); return BulletinFactory.createBulletin(groupId, groupName, connectable.getIdentifier(), type, connectable.getName(), category, severity, message); }
private AffectedComponentEntity createAffectedComponentEntity(final Connectable connectable) { final AffectedComponentEntity entity = new AffectedComponentEntity(); entity.setRevision(dtoFactory.createRevisionDTO(revisionManager.getRevision(connectable.getIdentifier()))); entity.setId(connectable.getIdentifier()); final Authorizable authorizable = getAuthorizable(connectable); final PermissionsDTO permissionsDto = dtoFactory.createPermissionsDto(authorizable); entity.setPermissions(permissionsDto); final AffectedComponentDTO dto = new AffectedComponentDTO(); dto.setId(connectable.getIdentifier()); dto.setReferenceType(connectable.getConnectableType().name()); dto.setState(connectable.getScheduledState().name()); final String groupId = connectable instanceof RemoteGroupPort ? ((RemoteGroupPort) connectable).getRemoteProcessGroup().getIdentifier() : connectable.getProcessGroupIdentifier(); dto.setProcessGroupId(groupId); entity.setComponent(dto); return entity; }
private Connection addConnection(final ProcessGroup destinationGroup, final VersionedConnection proposed, final String componentIdSeed) { final Connectable source = getConnectable(destinationGroup, proposed.getSource()); if (source == null) { throw new IllegalArgumentException("Connection has a source with identifier " + proposed.getIdentifier() + " but no component could be found in the Process Group with a corresponding identifier"); } final Connectable destination = getConnectable(destinationGroup, proposed.getDestination()); if (destination == null) { throw new IllegalArgumentException("Connection has a destination with identifier " + proposed.getDestination().getId() + " but no component could be found in the Process Group with a corresponding identifier"); } final Connection connection = flowController.createConnection(generateUuid(proposed.getIdentifier(), destination.getIdentifier(), componentIdSeed), proposed.getName(), source, destination, proposed.getSelectedRelationships()); connection.setVersionedComponentId(proposed.getIdentifier()); destinationGroup.addConnection(connection); updateConnection(connection, proposed); flowManager.onConnectionAdded(connection); return connection; }
private void registerForkEvent(final FlowFile parent, final FlowFile child) { ProvenanceEventBuilder eventBuilder = forkEventBuilders.get(parent); if (eventBuilder == null) { eventBuilder = context.getProvenanceRepository().eventBuilder(); eventBuilder.setEventType(ProvenanceEventType.FORK); eventBuilder.setFlowFileEntryDate(parent.getEntryDate()); eventBuilder.setLineageStartDate(parent.getLineageStartDate()); eventBuilder.setFlowFileUUID(parent.getAttribute(CoreAttributes.UUID.key())); eventBuilder.setComponentId(context.getConnectable().getIdentifier()); final Connectable connectable = context.getConnectable(); final String processorType = connectable.getComponentType(); eventBuilder.setComponentType(processorType); eventBuilder.addParentFlowFile(parent); updateEventContentClaims(eventBuilder, parent, getRecord(parent)); forkEventBuilders.put(parent, eventBuilder); } eventBuilder.addChildFlowFile(child); }