@CapabilityDescription("Fetches files from an FTP Server and creates FlowFiles from them") @WritesAttributes({ @WritesAttribute(attribute = "filename", description = "The filename is set to the name of the file on the remote server"), @WritesAttribute(attribute = "path", description = "The path is set to the path of the file's directory on the remote server. " + "For example, if the <Remote Path> property is set to /tmp, files picked up from /tmp will have the path attribute set " + "to /tmp. If the <Search Recursively> property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path " + "attribute will be set to /tmp/abc/1/2/3"), @WritesAttribute(attribute = "file.lastModifiedTime", description = "The date and time that the source file was last modified"), @WritesAttribute(attribute = "file.lastAccessTime", description = "The date and time that the file was last accessed. May not work on " + "all file systems"), @WritesAttribute(attribute = "file.owner", description = "The numeric owner id of the source file"), @WritesAttribute(attribute = "file.group", description = "The numeric group id of the source file"), @WritesAttribute(attribute = "file.permissions", description = "The read/write/execute permissions of the source file"), @WritesAttribute(attribute = "absolute.path", description = "The full/absolute path from where a file was picked up. The current 'path' " + "attribute is still populated, but may be a relative path")}) @SeeAlso(PutFTP.class)
@CapabilityDescription("Fetches files from an SFTP Server and creates FlowFiles from them") @WritesAttributes({ @WritesAttribute(attribute = "filename", description = "The filename is set to the name of the file on the remote server"), @WritesAttribute(attribute = "path", description = "The path is set to the path of the file's directory on the remote server. " + "For example, if the <Remote Path> property is set to /tmp, files picked up from /tmp will have the path attribute set " + "to /tmp. If the <Search Recursively> property is set to true and a file is picked up from /tmp/abc/1/2/3, then the path " + "attribute will be set to /tmp/abc/1/2/3"), @WritesAttribute(attribute = "file.lastModifiedTime", description = "The date and time that the source file was last modified"), @WritesAttribute(attribute = "file.owner", description = "The numeric owner id of the source file"), @WritesAttribute(attribute = "file.group", description = "The numeric group id of the source file"), @WritesAttribute(attribute = "file.permissions", description = "The read/write/execute permissions of the source file"), @WritesAttribute(attribute = "absolute.path", description = "The full/absolute path from where a file was picked up. The current 'path' " + "attribute is still populated, but may be a relative path")}) @SeeAlso(PutSFTP.class)
" as WebSocket client configured with this processor receives messages from remote WebSocket server.") @WritesAttributes({ @WritesAttribute(attribute = ATTR_WS_CS_ID, description = "WebSocket Controller Service id."), @WritesAttribute(attribute = ATTR_WS_SESSION_ID, description = "Established WebSocket session id."), @WritesAttribute(attribute = ATTR_WS_ENDPOINT_ID, description = "WebSocket endpoint id."), @WritesAttribute(attribute = ATTR_WS_LOCAL_ADDRESS, description = "WebSocket client address."), @WritesAttribute(attribute = ATTR_WS_REMOTE_ADDRESS, description = "WebSocket server address."), @WritesAttribute(attribute = ATTR_WS_MESSAGE_TYPE, description = "TEXT or BINARY."), }) public class ConnectWebSocket extends AbstractWebSocketGatewayProcessor {
+ "will take the form X.geo.<fieldName>") @WritesAttributes({ @WritesAttribute(attribute = "X.geo.lookup.micros", description = "The number of microseconds that the geo lookup took"), @WritesAttribute(attribute = "X.geo.city", description = "The city identified for the IP address"), @WritesAttribute(attribute = "X.geo.accuracy", description = "The accuracy radius if provided by the database (in Kilometers)"), @WritesAttribute(attribute = "X.geo.latitude", description = "The latitude identified for this IP address"), @WritesAttribute(attribute = "X.geo.longitude", description = "The longitude identified for this IP address"), @WritesAttribute(attribute = "X.geo.subdivision.N", description = "Each subdivision that is identified for this IP address is added with a one-up number appended to the attribute name, starting with 0"), @WritesAttribute(attribute = "X.geo.subdivision.isocode.N", description = "The ISO code for the subdivision that is identified by X.geo.subdivision.N"), @WritesAttribute(attribute = "X.geo.country", description = "The country identified for this IP address"), @WritesAttribute(attribute = "X.geo.country.isocode", description = "The ISO Code for the country identified"), @WritesAttribute(attribute = "X.geo.postalcode", description = "The postal code for the country identified"),}) public class GeoEnrichIP extends AbstractEnrichIP {
@SeeAlso({FetchSFTP.class, GetSFTP.class, PutSFTP.class}) @WritesAttributes({ @WritesAttribute(attribute = "sftp.remote.host", description = "The hostname of the SFTP Server"), @WritesAttribute(attribute = "sftp.remote.port", description = "The port that was connected to on the SFTP Server"), @WritesAttribute(attribute = "sftp.listing.user", description = "The username of the user that performed the SFTP Listing"), @WritesAttribute(attribute = ListFile.FILE_OWNER_ATTRIBUTE, description = "The numeric owner id of the source file"), @WritesAttribute(attribute = ListFile.FILE_GROUP_ATTRIBUTE, description = "The numeric group id of the source file"), @WritesAttribute(attribute = ListFile.FILE_PERMISSIONS_ATTRIBUTE, description = "The read/write/execute permissions of the source file"), @WritesAttribute(attribute = ListFile.FILE_SIZE_ATTRIBUTE, description = "The number of bytes in the source file"), @WritesAttribute(attribute = ListFile.FILE_LAST_MODIFY_TIME_ATTRIBUTE, description = "The timestamp of when the file in the filesystem was" + "last modified as 'yyyy-MM-dd'T'HH:mm:ssZ'"), @WritesAttribute(attribute = "filename", description = "The name of the file on the SFTP Server"), @WritesAttribute(attribute = "path", description = "The fully qualified name of the directory on the SFTP Server from which the file was pulled"), }) @Stateful(scopes = {Scope.CLUSTER}, description = "After performing a listing of files, the timestamp of the newest file is stored. "
@SeeAlso({GetSFTP.class, PutSFTP.class, GetFTP.class, PutFTP.class}) @WritesAttributes({ @WritesAttribute(attribute = "ftp.remote.host", description = "The hostname or IP address from which the file was pulled"), @WritesAttribute(attribute = "ftp.remote.port", description = "The port that was used to communicate with the remote FTP server"), @WritesAttribute(attribute = "ftp.remote.filename", description = "The name of the remote file that was pulled"), @WritesAttribute(attribute = "filename", description = "The filename is updated to point to the filename fo the remote file"), @WritesAttribute(attribute = "path", description = "If the Remote File contains a directory name, that directory name will be added to the FlowFile using the 'path' attribute") }) public class FetchFTP extends FetchFileTransfer {
@CapabilityDescription("Puts content into an Azure Storage Blob") @InputRequirement(Requirement.INPUT_REQUIRED) @WritesAttributes({ @WritesAttribute(attribute = "azure.container", description = "The name of the Azure container"), @WritesAttribute(attribute = "azure.blobname", description = "The name of the Azure blob"), @WritesAttribute(attribute = "azure.primaryUri", description = "Primary location for blob content"), @WritesAttribute(attribute = "azure.etag", description = "Etag for the Azure blob"), @WritesAttribute(attribute = "azure.length", description = "Length of the blob"), @WritesAttribute(attribute = "azure.timestamp", description = "The timestamp in Azure for the blob")}) public class PutAzureBlobStorage extends AbstractAzureBlobProcessor {
@SeeAlso({FetchFTP.class, GetFTP.class, PutFTP.class}) @WritesAttributes({ @WritesAttribute(attribute = "ftp.remote.host", description = "The hostname of the FTP Server"), @WritesAttribute(attribute = "ftp.remote.port", description = "The port that was connected to on the FTP Server"), @WritesAttribute(attribute = "ftp.listing.user", description = "The username of the user that performed the FTP Listing"), @WritesAttribute(attribute = ListFile.FILE_OWNER_ATTRIBUTE, description = "The numeric owner id of the source file"), @WritesAttribute(attribute = ListFile.FILE_GROUP_ATTRIBUTE, description = "The numeric group id of the source file"), @WritesAttribute(attribute = ListFile.FILE_PERMISSIONS_ATTRIBUTE, description = "The read/write/execute permissions of the source file"), @WritesAttribute(attribute = ListFile.FILE_SIZE_ATTRIBUTE, description = "The number of bytes in the source file"), @WritesAttribute(attribute = ListFile.FILE_LAST_MODIFY_TIME_ATTRIBUTE, description = "The timestamp of when the file in the filesystem was" + "last modified as 'yyyy-MM-dd'T'HH:mm:ssZ'"), @WritesAttribute(attribute = "filename", description = "The name of the file on the SFTP Server"), @WritesAttribute(attribute = "path", description = "The fully qualified name of the directory on the SFTP Server from which the file was pulled"), }) @Stateful(scopes = {Scope.CLUSTER}, description = "After performing a listing of files, the timestamp of the newest file is stored. "
" as the WebSocket server configured with this processor receives client requests") @WritesAttributes({ @WritesAttribute(attribute = ATTR_WS_CS_ID, description = "WebSocket Controller Service id."), @WritesAttribute(attribute = ATTR_WS_SESSION_ID, description = "Established WebSocket session id."), @WritesAttribute(attribute = ATTR_WS_ENDPOINT_ID, description = "WebSocket endpoint id."), @WritesAttribute(attribute = ATTR_WS_LOCAL_ADDRESS, description = "WebSocket server address."), @WritesAttribute(attribute = ATTR_WS_REMOTE_ADDRESS, description = "WebSocket client address."), @WritesAttribute(attribute = ATTR_WS_MESSAGE_TYPE, description = "TEXT or BINARY."), }) public class ListenWebSocket extends AbstractWebSocketGatewayProcessor {
@SeeAlso({GetSFTP.class, PutSFTP.class, GetFTP.class, PutFTP.class}) @WritesAttributes({ @WritesAttribute(attribute = "sftp.remote.host", description = "The hostname or IP address from which the file was pulled"), @WritesAttribute(attribute = "sftp.remote.port", description = "The port that was used to communicate with the remote SFTP server"), @WritesAttribute(attribute = "sftp.remote.filename", description = "The name of the remote file that was pulled"), @WritesAttribute(attribute = "filename", description = "The filename is updated to point to the filename fo the remote file"), @WritesAttribute(attribute = "path", description = "If the Remote File contains a directory name, that directory name will be added to the FlowFile using the 'path' attribute") }) public class FetchSFTP extends FetchFileTransfer {
" enrichment will take the form X.isp.<fieldName>") @WritesAttributes({ @WritesAttribute(attribute = "X.isp.lookup.micros", description = "The number of microseconds that the geo lookup took"), @WritesAttribute(attribute = "X.isp.asn", description = "The Autonomous System Number (ASN) identified for the IP address"), @WritesAttribute(attribute = "X.isp.asn.organization", description = "The Organization Associated with the ASN identified"), @WritesAttribute(attribute = "X.isp.name", description = "The name of the ISP associated with the IP address provided"), @WritesAttribute(attribute = "X.isp.organization", description = "The Organization associated with the IP address provided"),}) public class ISPEnrichIP extends AbstractEnrichIP {
+ "per the State Management documentation") @WritesAttributes({ @WritesAttribute(attribute = "tablename", description="Name of the table being queried"), @WritesAttribute(attribute = "querydbtable.row.count", description="The number of rows selected by the query"), @WritesAttribute(attribute="fragment.identifier", description="If 'Max Rows Per Flow File' is set then all FlowFiles from the same query result set " + "will have the same value for the fragment.identifier attribute. This can then be used to correlate the results."), @WritesAttribute(attribute = "fragment.count", description = "If 'Max Rows Per Flow File' is set then this is the total number of " + "FlowFiles produced by a single ResultSet. This can be used in conjunction with the " + "fragment.identifier attribute in order to know how many FlowFiles belonged to the same incoming ResultSet. If Output Batch Size is set, then this " + "attribute will not be populated."), @WritesAttribute(attribute="fragment.index", description="If 'Max Rows Per Flow File' is set then the position of this FlowFile in the list of " + "outgoing FlowFiles that were all derived from the same result set FlowFile. This can be " + "used in conjunction with the fragment.identifier attribute to know which FlowFiles originated from the same query result set and in what order " + "FlowFiles were produced"), @WritesAttribute(attribute = "maxvalue.*", description = "Each attribute contains the observed maximum value of a specified 'Maximum-value Column'. The " + "suffix of the attribute is the name of the column. If Output Batch Size is set, then this attribute will not be populated.")}) @DynamicProperty(name = "initial.maxvalue.<max_value_column>", value = "Initial maximum value for the specified column",
@Tags({ "elastic", "elasticsearch", "delete", "query"}) @WritesAttributes({ @WritesAttribute(attribute = "elasticsearch.delete.took", description = "The amount of time that it took to complete the delete operation in ms."), @WritesAttribute(attribute = "elasticsearch.delete.error", description = "The error message provided by ElasticSearch if there is an error running the delete.") }) public class DeleteByQueryElasticsearch extends AbstractProcessor implements ElasticSearchRestProcessor {
@Tags({"convert", "record", "generic", "schema", "json", "csv", "avro", "log", "logs", "freeform", "text"}) @WritesAttributes({ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Record Writer"), @WritesAttribute(attribute = "record.count", description = "The number of records in the FlowFile") }) @CapabilityDescription("Converts records from one data format to another using configured Record Reader and Record Write Controller Services. "
@ReadsAttribute(attribute = "filename", description = "The name of the file to write comes from the value of this attribute.") @WritesAttributes({ @WritesAttribute(attribute = "filename", description = "The name of the file is stored in this attribute."), @WritesAttribute(attribute = "absolute.hdfs.path", description = "The absolute path to the file is stored in this attribute."), @WritesAttribute(attribute = "record.count", description = "The number of records written to the Parquet file") }) @Restricted(restrictions = {
"the Receive Buffer Size must be greater than 100kb.") @WritesAttributes({ @WritesAttribute(attribute="tcp.sender", description="The sending host of the messages."), @WritesAttribute(attribute="tcp.port", description="The sending port the messages were received.") }) public class ListenTCP extends AbstractListenEventBatchingProcessor<StandardEvent> {
@SupportsBatching @InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) @Tags({"parquet", "hadoop", "HDFS", "get", "ingest", "fetch", "source", "record"}) @CapabilityDescription("Reads from a given Parquet file and writes records to the content of the flow file using " + "the selected record writer. The original Parquet file will remain unchanged, and the content of the flow file " + "will be replaced with records of the selected type. This processor can be used with ListHDFS or ListFile to obtain " + "a listing of files to fetch.") @WritesAttributes({ @WritesAttribute(attribute="fetch.failure.reason", description="When a FlowFile is routed to 'failure', this attribute is added " + "indicating why the file could not be fetched from the given filesystem."), @WritesAttribute(attribute = "record.count", description = "The number of records in the resulting flow file") }) @SeeAlso({PutParquet.class}) @Restricted(restrictions = { @Restriction( requiredPermission = RequiredPermission.READ_FILESYSTEM, explanation = "Provides operator the ability to retrieve any file that NiFi has access to in HDFS or the local filesystem.") }) public class FetchParquet extends AbstractFetchHDFSRecord { @Override public HDFSRecordReader createHDFSRecordReader(final ProcessContext context, final FlowFile flowFile, final Configuration conf, final Path path) throws IOException { final ParquetReader.Builder<GenericRecord> readerBuilder = AvroParquetReader.<GenericRecord>builder(path).withConf(conf); return new AvroParquetHDFSRecordReader(readerBuilder.build()); } }
@InputRequirement(Requirement.INPUT_REQUIRED) @WritesAttributes({ @WritesAttribute(attribute = "azure.length", description = "The length of the blob fetched") }) public class FetchAzureBlobStorage extends AbstractAzureBlobProcessor {
+ "a value of zero, and all copies receive incremented integer values.") @WritesAttributes({ @WritesAttribute(attribute = "copy.index", description = "A zero-based incrementing integer value based on which copy the FlowFile is.") }) public class DuplicateFlowFile extends AbstractProcessor {
" can be turned off by setting the log level for logger 'org.apache.ignite' to WARN in the logback.xml configuration file.") @WritesAttributes({ @WritesAttribute(attribute = GetIgniteCache.IGNITE_GET_FAILED_REASON_ATTRIBUTE_KEY, description = "The reason for getting entry from cache"), }) @SeeAlso({PutIgniteCache.class})