/** * @return an unmodifiable map of the flow file attributes */ @Override public Map<String, String> getAttributes() { return flowFile.getAttributes(); }
private boolean allMatch(final FlowFile flowFile, final Pattern attributePattern, final Set<String> dictionary) { for (final Map.Entry<String, String> entry : flowFile.getAttributes().entrySet()) { if (attributePattern == null || attributePattern.matcher(entry.getKey()).matches()) { if (!dictionary.contains(entry.getValue())) { return false; } } } return true; }
private boolean anyMatch(final FlowFile flowFile, final Pattern attributePattern, final Set<String> dictionary) { for (final Map.Entry<String, String> entry : flowFile.getAttributes().entrySet()) { if (attributePattern == null || attributePattern.matcher(entry.getKey()).matches()) { if (dictionary.contains(entry.getValue())) { return true; } } } return false; } }
/** * Create a RecordReader instance to read records from specified InputStream. * This method calls {@link #createRecordReader(Map, InputStream, ComponentLog)} with Attributes of the specified FlowFile. * @param flowFile Attributes of this FlowFile are used to resolve Record Schema via Expression Language dynamically. This can be null. * @param in InputStream containing Records. This can be null or empty stream. * @param logger A logger bind to a component * @return Created RecordReader instance */ default RecordReader createRecordReader(FlowFile flowFile, InputStream in, ComponentLog logger) throws MalformedRecordException, IOException, SchemaNotFoundException { return createRecordReader(flowFile == null ? Collections.emptyMap() : flowFile.getAttributes(), in, logger); }
private static String formatAttributes(final FlowFile flowFile, final String messagePrepend) { StringBuilder message = new StringBuilder(messagePrepend); message.append(BODY_SEPARATOR); message.append("\nStandard FlowFile Metadata:"); message.append(String.format("\n\t%1$s = '%2$s'", "id", flowFile.getAttribute(CoreAttributes.UUID.key()))); message.append(String.format("\n\t%1$s = '%2$s'", "entryDate", new Date(flowFile.getEntryDate()))); message.append(String.format("\n\t%1$s = '%2$s'", "fileSize", flowFile.getSize())); message.append("\nFlowFile Attributes:"); for (Entry<String, String> attribute : flowFile.getAttributes().entrySet()) { message.append(String.format("\n\t%1$s = '%2$s'", attribute.getKey(), attribute.getValue())); } message.append("\n"); return message.toString(); }
private void addHeaders(final FlowFile flowFile, final Map<String, String> additionalAttributes, final ProducerRecord<?, ?> record) { if (attributeNameRegex == null) { return; } final Headers headers = record.headers(); for (final Map.Entry<String, String> entry : flowFile.getAttributes().entrySet()) { if (attributeNameRegex.matcher(entry.getKey()).matches()) { headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet)); } } for (final Map.Entry<String, String> entry : additionalAttributes.entrySet()) { if (attributeNameRegex.matcher(entry.getKey()).matches()) { headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet)); } } }
private void addHeaders(final FlowFile flowFile, final Map<String, String> additionalAttributes, final ProducerRecord<?, ?> record) { if (attributeNameRegex == null) { return; } final Headers headers = record.headers(); for (final Map.Entry<String, String> entry : flowFile.getAttributes().entrySet()) { if (attributeNameRegex.matcher(entry.getKey()).matches()) { headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet)); } } for (final Map.Entry<String, String> entry : additionalAttributes.entrySet()) { if (attributeNameRegex.matcher(entry.getKey()).matches()) { headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet)); } } }
private void addHeaders(final FlowFile flowFile, final Map<String, String> additionalAttributes, final ProducerRecord<?, ?> record) { if (attributeNameRegex == null) { return; } final Headers headers = record.headers(); for (final Map.Entry<String, String> entry : flowFile.getAttributes().entrySet()) { if (attributeNameRegex.matcher(entry.getKey()).matches()) { headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet)); } } for (final Map.Entry<String, String> entry : additionalAttributes.entrySet()) { if (attributeNameRegex.matcher(entry.getKey()).matches()) { headers.add(entry.getKey(), entry.getValue().getBytes(headerCharacterSet)); } } }
@Override public MockFlowFile removeAllAttributes(FlowFile flowFile, final Pattern keyPattern) { flowFile = validateState(flowFile); if (flowFile == null) { throw new IllegalArgumentException("flowFile cannot be null"); } if (keyPattern == null) { return (MockFlowFile) flowFile; } final Set<String> attrsToRemove = new HashSet<>(); for (final String key : flowFile.getAttributes().keySet()) { if (keyPattern.matcher(key).matches()) { attrsToRemove.add(key); } } return removeAllAttributes(flowFile, attrsToRemove); }
@Override public Map<String, String> getHeaders() { synchronized (headers) { if (!headersLoaded) { headers.putAll(flowFile.getAttributes()); headers.put(ENTRY_DATE_HEADER, Long.toString(flowFile.getEntryDate())); headers.put(ID_HEADER, Long.toString(flowFile.getId())); headers.put(LAST_QUEUE_DATE_HEADER, Long.toString(flowFile.getLastQueueDate())); headers.put(LINEAGE_START_DATE_HEADER, Long.toString(flowFile.getLineageStartDate())); headers.put(SIZE_HEADER, Long.toString(flowFile.getSize())); headersLoaded = true; } } return headers; }
private List<Tag> getObjectTags(ProcessContext context, FlowFile flowFile) { final String prefix = context.getProperty(OBJECT_TAGS_PREFIX).evaluateAttributeExpressions(flowFile).getValue(); final List<Tag> objectTags = new ArrayList<>(); final Map<String, String> attributesMap = flowFile.getAttributes(); attributesMap.entrySet().stream().sequential() .filter(attribute -> attribute.getKey().startsWith(prefix)) .forEach(attribute -> { String tagKey = attribute.getKey(); String tagValue = attribute.getValue(); if (context.getProperty(REMOVE_TAG_PREFIX).asBoolean()) { tagKey = tagKey.replace(prefix, ""); } objectTags.add(new Tag(tagKey, tagValue)); }); return objectTags; }
public MockFlowFile(final long id, final FlowFile toCopy) { this.creationTime = System.nanoTime(); this.id = id; entryDate = System.currentTimeMillis(); final Map<String, String> attributesToCopy = toCopy.getAttributes(); String filename = attributesToCopy.get(CoreAttributes.FILENAME.key()); if (filename == null) { filename = String.valueOf(System.nanoTime()) + ".mockFlowFile"; } attributes.put(CoreAttributes.FILENAME.key(), filename); String path = attributesToCopy.get(CoreAttributes.PATH.key()); if (path == null) { path = "target"; } attributes.put(CoreAttributes.PATH.key(), path); String uuid = attributesToCopy.get(CoreAttributes.UUID.key()); if (uuid == null) { uuid = UUID.randomUUID().toString(); } attributes.put(CoreAttributes.UUID.key(), uuid); attributes.putAll(toCopy.getAttributes()); final byte[] dataToCopy = ((MockFlowFile) toCopy).data; this.data = new byte[dataToCopy.length]; System.arraycopy(dataToCopy, 0, this.data, 0, dataToCopy.length); this.penalized = toCopy.isPenalized(); }
@Override public void process(final InputStream in) throws IOException { final DataPacket dataPacket = new StandardDataPacket(toSend.getAttributes(), in, toSend.getSize()); codec.encode(dataPacket, checkedOutputStream); } });
/** * Inherits the attributes from the given source flow file into another flow * file. The UUID of the source becomes the parent UUID of this flow file. * If a parent uuid had previously been established it will be replaced by * the uuid of the given source * * @param source the FlowFile from which to copy attributes * @param destination the FlowFile to which to copy attributes */ private FlowFile inheritAttributes(final FlowFile source, final FlowFile destination) { if (source == null || destination == null || source == destination) { return destination; // don't need to inherit from ourselves } final FlowFile updated = putAllAttributes(destination, source.getAttributes()); getProvenanceReporter().fork(source, Collections.singletonList(updated)); return updated; }
private void forwardFlowFile(final ProcessContext context, final ProcessSession session, HttpRequestContainer container, final long start, final HttpServletRequest request, FlowFile flowFile) { final long receiveMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); String subjectDn = flowFile.getAttribute(HTTPUtils.HTTP_SSL_CERT); session.getProvenanceReporter().receive(flowFile, HTTPUtils.getURI(flowFile.getAttributes()), "Received from " + request.getRemoteAddr() + (subjectDn == null ? "" : " with DN=" + subjectDn), receiveMillis); session.transfer(flowFile, REL_SUCCESS); getLogger().info("Transferring {} to 'success'; received from {}", new Object[]{flowFile, request.getRemoteAddr()}); }
@Override public void process(final InputStream in) throws IOException { final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize()); transaction.send(dataPacket); } });
@Override public ProvenanceEventBuilder fromFlowFile(final FlowFile flowFile) { setFlowFileEntryDate(flowFile.getEntryDate()); setLineageStartDate(flowFile.getLineageStartDate()); setAttributes(Collections.emptyMap(), flowFile.getAttributes()); uuid = flowFile.getAttribute(CoreAttributes.UUID.key()); this.contentSize = flowFile.getSize(); return this; }
@Override protected SqlWriter configureSqlWriter(ProcessSession session, ProcessContext context, FlowFile fileToProcess) { final Integer maxRowsPerFlowFile = context.getProperty(MAX_ROWS_PER_FLOW_FILE).evaluateAttributeExpressions().asInteger(); final boolean convertNamesForAvro = context.getProperty(NORMALIZE_NAMES).asBoolean(); final Boolean useAvroLogicalTypes = context.getProperty(USE_AVRO_LOGICAL_TYPES).asBoolean(); final JdbcCommon.AvroConversionOptions options = JdbcCommon.AvroConversionOptions.builder() .convertNames(convertNamesForAvro) .useLogicalTypes(useAvroLogicalTypes) .build(); final RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class); return new RecordSqlWriter(recordSetWriterFactory, options, maxRowsPerFlowFile, fileToProcess == null ? Collections.emptyMap() : fileToProcess.getAttributes()); } }
/** * */ private void sendToSpring(FlowFile flowFileToProcess, ProcessContext context, ProcessSession processSession) { byte[] payload = this.extractMessage(flowFileToProcess, processSession); boolean sent = false; try { sent = this.exchanger.send(payload, flowFileToProcess.getAttributes(), this.sendTimeout); if (sent) { processSession.getProvenanceReporter().send(flowFileToProcess, this.applicationContextConfigFileName); processSession.remove(flowFileToProcess); } else { processSession.transfer(processSession.penalize(flowFileToProcess), REL_FAILURE); this.getLogger().error("Timed out while sending FlowFile to Spring Application Context " + this.applicationContextConfigFileName); context.yield(); } } catch (Exception e) { processSession.transfer(flowFileToProcess, REL_FAILURE); this.getLogger().error("Failed while sending FlowFile to Spring Application Context " + this.applicationContextConfigFileName + "; " + e.getMessage(), e); context.yield(); } }
/** * Test MapMessage to FlowFile conversion * * @throws java.lang.Exception ex */ @Test public void testMap2FlowFileMapMessage() throws Exception { TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class); MapMessage mapMessage = createMapMessage(); ProcessContext context = runner.getProcessContext(); ProcessSession session = runner.getProcessSessionFactory().createSession(); ProcessorInitializationContext pic = new MockProcessorInitializationContext(runner.getProcessor(), (MockProcessContext) runner.getProcessContext()); JmsProcessingSummary summary = JmsConsumer.map2FlowFile(context, session, mapMessage, true, pic.getLogger()); assertEquals("MapMessage should not create FlowFile content", 0, summary.getBytesReceived()); Map<String, String> attributes = summary.getLastFlowFile().getAttributes(); assertEquals("", "Arnold", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "name")); assertEquals("", "97", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "age")); assertEquals("", "89686.564", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "xyz")); assertEquals("", "true", attributes.get(JmsConsumer.MAP_MESSAGE_PREFIX + "good")); }