private boolean shouldReportOnlyOnPrimary(boolean isClusterScope, final ProcessContext context) { if (REPORT_NODE_PRIMARY.equals(context.getProperty(REPORTING_NODE).getValue())) { if (isClusterScope) { return true; } } return false; }
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { final String listingStrategy = context.getProperty(LISTING_STRATEGY).getValue(); if (BY_TIMESTAMPS.equals(listingStrategy)) { listByTrackingTimestamps(context, session); } else if (BY_ENTITIES.equals(listingStrategy)) { listByTrackingEntities(context, session); } else { throw new ProcessException("Unknown listing strategy: " + listingStrategy); } }
@Override public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) { if (descriptor.equals(ROUTE_STRATEGY)) { configuredRouteStrategy = newValue; } else { final Set<String> newDynamicPropertyNames = new HashSet<>(dynamicPropertyNames); if (newValue == null) { newDynamicPropertyNames.remove(descriptor.getName()); } else if (oldValue == null) { // new property newDynamicPropertyNames.add(descriptor.getName()); } this.dynamicPropertyNames = Collections.unmodifiableSet(newDynamicPropertyNames); } // formulate the new set of Relationships final Set<String> allDynamicProps = this.dynamicPropertyNames; final Set<Relationship> newRelationships = new HashSet<>(); final String routeStrategy = configuredRouteStrategy; if (ROUTE_PROPERTY_NAME.equals(routeStrategy)) { for (final String propName : allDynamicProps) { newRelationships.add(new Relationship.Builder().name(propName).build()); } } else { newRelationships.add(REL_MATCH); } newRelationships.add(REL_NO_MATCH); this.relationships.set(newRelationships); }
private SslContextFactory createSslFactory(final ConfigurationContext context) { final SSLContextService sslService = context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class); final String clientAuthValue = context.getProperty(CLIENT_AUTH).getValue(); final boolean need; final boolean want; if (CLIENT_NEED.equals(clientAuthValue)) { need = true; want = false; } else if (CLIENT_WANT.equals(clientAuthValue)) { need = false; want = true; } else { need = false; want = false; } final SslContextFactory sslFactory = (sslService == null) ? null : createSslFactory(sslService, need, want); return sslFactory; }
private void initLineageStrategy(ConfigurationContext context) throws IOException { nifiAtlasHook = new NiFiAtlasHook(); final String strategy = context.getProperty(NIFI_LINEAGE_STRATEGY).getValue(); if (LINEAGE_STRATEGY_SIMPLE_PATH.equals(strategy)) { lineageStrategy = new SimpleFlowPathLineage(); } else if (LINEAGE_STRATEGY_COMPLETE_PATH.equals(strategy)) { lineageStrategy = new CompleteFlowPathLineage(); } lineageStrategy.setLineageContext(nifiAtlasHook); initProvenanceConsumer(context); }
/** * In order to add custom validation at sub-classes, implement {@link #customValidate(ValidationContext, Collection)} method. */ @Override protected final Collection<ValidationResult> customValidate(ValidationContext context) { final Collection<ValidationResult> results = new ArrayList<>(); final String listingStrategy = context.getProperty(LISTING_STRATEGY).getValue(); if (BY_ENTITIES.equals(listingStrategy)) { ListedEntityTracker.validateProperties(context, results, getStateScope(context)); } customValidate(context, results); return results; }
private byte[] getDelimiterContent(final ProcessContext context, final List<FlowFile> wrappers, final PropertyDescriptor descriptor) throws IOException { final String delimiterStrategyValue = context.getProperty(DELIMITER_STRATEGY).getValue(); if (DELIMITER_STRATEGY_FILENAME.equals(delimiterStrategyValue)) { return getDelimiterFileContent(context, wrappers, descriptor); } else { return getDelimiterTextContent(context, wrappers, descriptor); } }
@Override protected void setUpBinManager(final BinManager binManager, final ProcessContext context) { if (MERGE_STRATEGY_DEFRAGMENT.equals(context.getProperty(MERGE_STRATEGY).getValue())) { binManager.setFileCountAttribute(FRAGMENT_COUNT_ATTRIBUTE); } else { binManager.setFileCountAttribute(null); } }
protected String getGroupId(final ProcessContext context, final FlowFile flowFile, final RecordSchema schema, final ProcessSession session) { final String mergeStrategy = context.getProperty(MERGE_STRATEGY).getValue(); if (MERGE_STRATEGY_DEFRAGMENT.equals(mergeStrategy)) { return flowFile.getAttribute(FRAGMENT_ID_ATTRIBUTE); } final Optional<String> optionalText = schema.getSchemaText(); final String schemaText = optionalText.isPresent() ? optionalText.get() : AvroTypeUtil.extractAvroSchema(schema).toString(); final String groupId; final String correlationshipAttributeName = context.getProperty(CORRELATION_ATTRIBUTE_NAME).getValue(); if (correlationshipAttributeName != null) { final String correlationAttr = flowFile.getAttribute(correlationshipAttributeName); groupId = correlationAttr == null ? schemaText : schemaText + correlationAttr; } else { groupId = schemaText; } return groupId; }
private void setKafkaConfig(Map<Object, Object> mapToPopulate, PropertyContext context) { final String kafkaBootStrapServers = context.getProperty(KAFKA_BOOTSTRAP_SERVERS).evaluateAttributeExpressions().getValue(); mapToPopulate.put(ATLAS_PROPERTY_KAFKA_BOOTSTRAP_SERVERS, kafkaBootStrapServers); mapToPopulate.put(ATLAS_PROPERTY_KAFKA_CLIENT_ID, String.format("%s.%s", getName(), getIdentifier())); final String kafkaSecurityProtocol = context.getProperty(KAFKA_SECURITY_PROTOCOL).getValue(); mapToPopulate.put(ATLAS_KAFKA_PREFIX + "security.protocol", kafkaSecurityProtocol); // Translate SSLContext Service configuration into Kafka properties final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); if (sslContextService != null && sslContextService.isKeyStoreConfigured()) { mapToPopulate.put(ATLAS_KAFKA_PREFIX + SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, sslContextService.getKeyStoreFile()); mapToPopulate.put(ATLAS_KAFKA_PREFIX + SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, sslContextService.getKeyStorePassword()); final String keyPass = sslContextService.getKeyPassword() == null ? sslContextService.getKeyStorePassword() : sslContextService.getKeyPassword(); mapToPopulate.put(ATLAS_KAFKA_PREFIX + SslConfigs.SSL_KEY_PASSWORD_CONFIG, keyPass); mapToPopulate.put(ATLAS_KAFKA_PREFIX + SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, sslContextService.getKeyStoreType()); } if (sslContextService != null && sslContextService.isTrustStoreConfigured()) { mapToPopulate.put(ATLAS_KAFKA_PREFIX + SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, sslContextService.getTrustStoreFile()); mapToPopulate.put(ATLAS_KAFKA_PREFIX + SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, sslContextService.getTrustStorePassword()); mapToPopulate.put(ATLAS_KAFKA_PREFIX + SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG, sslContextService.getTrustStoreType()); } if (SEC_SASL_PLAINTEXT.equals(kafkaSecurityProtocol) || SEC_SASL_SSL.equals(kafkaSecurityProtocol)) { setKafkaJaasConfig(mapToPopulate, context); } }
private boolean isClusterScope(final ProcessContext context, boolean logInvalidConfig) { if (SCOPE_CLUSTER.equals(context.getProperty(MONITORING_SCOPE).getValue())) { if (getNodeTypeProvider().isClustered()) { return true; } if (logInvalidConfig) { getLogger().warn("NiFi is running as a Standalone mode, but 'cluster' scope is set." + " Fallback to 'node' scope. Fix configuration to stop this message."); } } return false; }
@Override protected Collection<ValidationResult> additionalCustomValidation(ValidationContext context) { final Collection<ValidationResult> results = new ArrayList<>(); final String delimiterStrategy = context.getProperty(DELIMITER_STRATEGY).getValue(); if(DELIMITER_STRATEGY_FILENAME.equals(delimiterStrategy)) { final String headerValue = context.getProperty(HEADER).getValue(); if (headerValue != null) { results.add(StandardValidators.FILE_EXISTS_VALIDATOR.validate(HEADER.getName(), headerValue, context)); } final String footerValue = context.getProperty(FOOTER).getValue(); if (footerValue != null) { results.add(StandardValidators.FILE_EXISTS_VALIDATOR.validate(FOOTER.getName(), footerValue, context)); } final String demarcatorValue = context.getProperty(DEMARCATOR).getValue(); if (demarcatorValue != null) { results.add(StandardValidators.FILE_EXISTS_VALIDATOR.validate(DEMARCATOR.getName(), demarcatorValue, context)); } } return results; }
@Override protected String getGroupId(final ProcessContext context, final FlowFile flowFile, final ProcessSession session) { final String correlationAttributeName = context.getProperty(CORRELATION_ATTRIBUTE_NAME) .evaluateAttributeExpressions(flowFile).getValue(); String groupId = correlationAttributeName == null ? null : flowFile.getAttribute(correlationAttributeName); // when MERGE_STRATEGY is Defragment and correlationAttributeName is null then bin by fragment.identifier if (groupId == null && MERGE_STRATEGY_DEFRAGMENT.equals(context.getProperty(MERGE_STRATEGY).getValue())) { groupId = flowFile.getAttribute(FRAGMENT_ID_ATTRIBUTE); } return groupId; }
@Override protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { final List<ValidationResult> results = new ArrayList<>(); if (validationContext.getProperty(BASIC_AUTH).asBoolean()) { final String loginServiceValue = validationContext.getProperty(LOGIN_SERVICE).getValue(); if (LOGIN_SERVICE_HASH.equals(loginServiceValue)) { if (!validationContext.getProperty(USERS_PROPERTIES_FILE).isSet()) { results.add(new ValidationResult.Builder().subject(USERS_PROPERTIES_FILE.getDisplayName()) .explanation("it is required by HashLoginService").valid(false).build()); } } } return results; }
private void validateKafkaProperties(ValidationContext context, Collection<ValidationResult> results, boolean isSSLContextServiceSet, ValidationResult.Builder invalidSSLService) { final String kafkaSecurityProtocol = context.getProperty(KAFKA_SECURITY_PROTOCOL).getValue(); if ((SEC_SSL.equals(kafkaSecurityProtocol) || SEC_SASL_SSL.equals(kafkaSecurityProtocol)) && !isSSLContextServiceSet) { results.add(invalidSSLService.explanation("required by SSL Kafka connection").build()); if (SEC_SASL_PLAINTEXT.equals(kafkaSecurityProtocol) || SEC_SASL_SSL.equals(kafkaSecurityProtocol)) { if (!context.getProperty(KAFKA_KERBEROS_SERVICE_NAME).isSet()) { results.add(new ValidationResult.Builder()
final AtomicReference<Exception> connectionError = new AtomicReference<>(null); final boolean isSolrCloud = SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue()); final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions(flowFile).getValue(); final Long commitWithin = context.getProperty(COMMIT_WITHIN).evaluateAttributeExpressions(flowFile).asLong();
if (MERGE_STRATEGY_DEFRAGMENT.equals(mergeStrategy)) { block = true; } else if (context.getProperty(CORRELATION_ATTRIBUTE_NAME).isSet()) {
final ProcessSession binSession = bin.getSession(); if (MERGE_STRATEGY_DEFRAGMENT.equals(context.getProperty(MERGE_STRATEGY).getValue())) { final String error = getDefragmentValidationError(bin.getContents());
final List<ValidationResult> problems = new ArrayList<>(); if (SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue())) { final String collection = context.getProperty(COLLECTION).getValue(); if (collection == null || collection.trim().isEmpty()) { if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) { final String solrLocation = context.getProperty(SOLR_LOCATION).evaluateAttributeExpressions().getValue(); if (solrLocation != null) {
if (LOGIN_SERVICE_HASH.equals(loginServiceValue)) { final String usersFilePath = context.getProperty(USERS_PROPERTIES_FILE).evaluateAttributeExpressions().getValue(); loginService = new HashLoginService("HashLoginService", usersFilePath);