@Override public ProvenanceEventType getEventType() { return record.getEventType(); }
@Override public ProvenanceEventType getEventType() { return record.getEventType(); }
@Override public ProvenanceEventType getEventType() { return event.getEventType(); }
/** * Checks if the given event is a spurious FORK, meaning that the FORK has a * single child and that child was removed in this session. This happens * when a Processor calls #create(FlowFile) and then removes the created * FlowFile. * * @param event event * @return true if spurious fork */ private boolean isSpuriousForkEvent(final ProvenanceEventRecord event, final Set<String> removedFlowFiles) { if (event.getEventType() == ProvenanceEventType.FORK) { final List<String> childUuids = event.getChildUuids(); if (childUuids != null && childUuids.size() == 1 && removedFlowFiles.contains(childUuids.get(0))) { return true; } } return false; }
public boolean isComplete() { // If the FlowFile is DROPed right after create child FlowFile, then the path is not worth for reporting. final boolean isDroppedImmediately = events.size() == 2 && events.get(0).getEventType().equals(ProvenanceEventType.DROP) && shouldCreateSeparatePath(events.get(1).getEventType()); return !isDroppedImmediately && hasInput() && hasOutput(); }
private void removeForkEvents(final FlowFile flowFile) { for (final ProvenanceEventBuilder builder : forkEventBuilders.values()) { final ProvenanceEventRecord event = builder.build(); if (event.getEventType() == ProvenanceEventType.FORK) { builder.removeChildFlowFile(flowFile); } } }
private static String stringify(final ProvenanceEventRecord event, final int index, final long byteOffset) { final StringBuilder sb = new StringBuilder(); sb.append("Event Index in File = ").append(index).append(", Byte Offset = ").append(byteOffset); sb.append("\n\t").append("Event ID = ").append(event.getEventId()); sb.append("\n\t").append("Event Type = ").append(event.getEventType()); sb.append("\n\t").append("Event Time = ").append(new Date(event.getEventTime())); sb.append("\n\t").append("Event UUID = ").append(event.getFlowFileUuid()); sb.append("\n\t").append("Component ID = ").append(event.getComponentId()); sb.append("\n\t").append("Event ID = ").append(event.getComponentType()); sb.append("\n\t").append("Transit URI = ").append(event.getTransitUri()); sb.append("\n\t").append("Parent IDs = ").append(event.getParentUuids()); sb.append("\n\t").append("Child IDs = ").append(event.getChildUuids()); sb.append("\n\t").append("Previous Attributes = ").append(event.getPreviousAttributes()); sb.append("\n\t").append("Updated Attributes = ").append(event.getUpdatedAttributes()); return sb.toString(); } }
protected DataSetRefs executeAnalyzer(AnalysisContext analysisContext, ProvenanceEventRecord event) { final NiFiProvenanceEventAnalyzer analyzer = NiFiProvenanceEventAnalyzerFactory.getAnalyzer(event.getComponentType(), event.getTransitUri(), event.getEventType()); if (analyzer == null) { return null; } if (logger.isDebugEnabled()) { logger.debug("Analyzer {} is found for event: {}", analyzer, event); } return analyzer.analyze(analysisContext, event); }
@Override public String toString() { return "Event[ID=" + record.getEventId() + ", Type=" + record.getEventType() + ", UUID=" + record.getFlowFileUuid() + ", Component=" + record.getComponentId() + "]"; } }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final String transitUri = event.getTransitUri(); final Matcher uriMatcher = URI_PATTERN.matcher(transitUri); if (!uriMatcher.matches()) { logger.warn("Unexpected transit URI: {}", new Object[]{transitUri}); return null; } final Referenceable ref = new Referenceable(TYPE); final String[] hostNames = splitHostNames(uriMatcher.group(1)); final String clusterName = context.getClusterResolver().fromHostNames(hostNames); final String tableName = uriMatcher.group(2); ref.set(ATTR_NAME, tableName); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(clusterName, tableName)); // TODO: 'uri' is a mandatory attribute, but what should we set? ref.set(ATTR_URI, transitUri); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { if (!ProvenanceEventType.SEND.equals(event.getEventType()) && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) { return null; } final boolean isRemoteInputPort = event.getComponentType().equals("Remote Input Port"); final String type = isRemoteInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT; final S2SPort s2SPort = analyzeS2SPort(event, context.getClusterResolver()); // Find connections that connects to/from the remote port. final String componentId = event.getComponentId(); final List<ConnectionStatus> connections = isRemoteInputPort ? context.findConnectionTo(componentId) : context.findConnectionFrom(componentId); if (connections == null || connections.isEmpty()) { logger.warn("Connection was not found: {}", new Object[]{event}); return null; } // The name of remote port can be retrieved from any connection, use the first one. final ConnectionStatus connection = connections.get(0); final Referenceable ref = new Referenceable(type); ref.set(ATTR_NAME, isRemoteInputPort ? connection.getDestinationName() : connection.getSourceName()); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2SPort.clusterName, s2SPort.targetPortId)); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
@Test public void testSelectQueryInFlowFile() throws InitializationException, ClassNotFoundException, SQLException, IOException { invokeOnTriggerRecords(null, QUERY_WITHOUT_EL, true, null, false); assertEquals(ProvenanceEventType.FORK, runner.getProvenanceEvents().get(0).getEventType()); assertEquals(ProvenanceEventType.FETCH, runner.getProvenanceEvents().get(1).getEventType()); }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final Referenceable ref = new Referenceable(TYPE); final URI uri = parseUri(event.getTransitUri()); final String clusterName; try { // use hostname in uri if available for remote path. final String uriHost = uri.getHost(); final String hostname = StringUtils.isEmpty(uriHost) ? InetAddress.getLocalHost().getHostName() : uriHost; clusterName = context.getClusterResolver().fromHostNames(hostname); } catch (UnknownHostException e) { logger.warn("Failed to get localhost name due to " + e, e); return null; } final String path = uri.getPath(); ref.set(ATTR_NAME, path); ref.set(ATTR_PATH, path); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(clusterName, path)); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final Referenceable ref = new Referenceable(TYPE); final String transitUri = event.getTransitUri(); if (transitUri == null) { return null; } final Matcher uriMatcher = URI_PATTERN.matcher(transitUri); if (!uriMatcher.matches()) { logger.warn("Unexpected transit URI: {}", new Object[]{transitUri}); return null; } final String[] hostNames = splitHostNames(uriMatcher.group(1)); final String clusterName = context.getClusterResolver().fromHostNames(hostNames); final String topicName = uriMatcher.group(2); ref.set(ATTR_NAME, topicName); ref.set(ATTR_TOPIC, topicName); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(clusterName, topicName)); ref.set(ATTR_URI, transitUri); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final Referenceable ref = new Referenceable(TYPE); final URI uri = parseUri(event.getTransitUri()); final String clusterName = context.getClusterResolver().fromHostNames(uri.getHost()); final String path = uri.getPath(); ref.set(ATTR_NAME, path); ref.set(ATTR_PATH, path); ref.set(ATTR_CLUSTER_NAME, clusterName); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(clusterName, path)); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
private void verifyProvenance(int expectedNumEvents) { List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents(); Assert.assertEquals(expectedNumEvents, provEvents.size()); for (ProvenanceEventRecord event : provEvents) { Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType()); Assert.assertTrue(event.getTransitUri().startsWith("udp://")); } }
@Test public void testNoIncomingConnection() throws ClassNotFoundException, SQLException, InitializationException, IOException { runner.setIncomingConnection(false); invokeOnTriggerRecords(null, QUERY_WITHOUT_EL, false, null, true); assertEquals(ProvenanceEventType.RECEIVE, runner.getProvenanceEvents().get(0).getEventType()); }
protected Referenceable createDataSetRef(AnalysisContext context, ProvenanceEventRecord event) { final Referenceable ref = new Referenceable(TYPE); ref.set(ATTR_NAME, event.getComponentType()); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(context.getNiFiClusterName(), event.getComponentId())); ref.set(ATTR_DESCRIPTION, event.getEventType() + " was performed by " + event.getComponentType()); return ref; }
@Test public void testBatching() throws IOException, InterruptedException { runner.setProperty(ListenRELP.MAX_BATCH_SIZE, "5"); final List<RELPFrame> frames = new ArrayList<>(); frames.add(OPEN_FRAME); frames.add(SYSLOG_FRAME); frames.add(SYSLOG_FRAME); frames.add(SYSLOG_FRAME); frames.add(CLOSE_FRAME); // one syslog frame should be transferred since we are batching, but three responses should be sent run(frames, 1, 3, null); final List<ProvenanceEventRecord> events = runner.getProvenanceEvents(); Assert.assertNotNull(events); Assert.assertEquals(1, events.size()); final ProvenanceEventRecord event = events.get(0); Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType()); Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("relp")); final List<MockFlowFile> mockFlowFiles = runner.getFlowFilesForRelationship(ListenRELP.REL_SUCCESS); Assert.assertEquals(1, mockFlowFiles.size()); final MockFlowFile mockFlowFile = mockFlowFiles.get(0); Assert.assertEquals(SYSLOG_FRAME.getCommand(), mockFlowFile.getAttribute(ListenRELP.RELPAttributes.COMMAND.key())); Assert.assertTrue(!StringUtils.isBlank(mockFlowFile.getAttribute(ListenRELP.RELPAttributes.PORT.key()))); Assert.assertTrue(!StringUtils.isBlank(mockFlowFile.getAttribute(ListenRELP.RELPAttributes.SENDER.key()))); }
@Test public void testValidMessageStaticPropertiesUdp() { final String pri = "34"; final String version = "1"; final String stamp = "2003-10-11T22:14:15.003Z"; final String host = "mymachine.example.com"; final String body = "su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8"; final String expectedMessage = "<" + pri + ">" + version + " " + stamp + " " + host + " " + body; runner.setProperty(PutSyslog.MSG_PRIORITY, pri); runner.setProperty(PutSyslog.MSG_VERSION, version); runner.setProperty(PutSyslog.MSG_TIMESTAMP, stamp); runner.setProperty(PutSyslog.MSG_HOSTNAME, host); runner.setProperty(PutSyslog.MSG_BODY, body); runner.enqueue("incoming data".getBytes(Charset.forName("UTF-8"))); runner.run(); runner.assertAllFlowFilesTransferred(PutSyslog.REL_SUCCESS, 1); Assert.assertEquals(1, sender.messages.size()); Assert.assertEquals(expectedMessage, sender.messages.get(0)); final List<ProvenanceEventRecord> events = runner.getProvenanceEvents(); Assert.assertNotNull(events); Assert.assertEquals(1, events.size()); final ProvenanceEventRecord event = events.get(0); Assert.assertEquals(ProvenanceEventType.SEND, event.getEventType()); Assert.assertEquals("UDP://localhost:12345", event.getTransitUri()); }