@Override public ProcessorResult<List<LookupKV<EnrichmentKey, EnrichmentValue>>> getResult() { ProcessorResult.Builder<List<LookupKV<EnrichmentKey,EnrichmentValue>>> builder = new ProcessorResult.Builder(); return builder.withResult(messages).build(); } });
@Override public ProcessorResult<Set<JSONObject>> getResult() { return new ProcessorResult<>(outputMessages, null); } }
@SuppressWarnings("unchecked") public ProcessorResult<List<byte[]>> getResults() { return new ProcessorResult.Builder<List<byte[]>>().withProcessErrors(errors) .withResult(output) .build(); } }
@BeforeClass public static void setupBeforeClass() throws Exception { UnitTestHelper.setJavaLoggingLevel(Level.SEVERE); LOG.info("Starting up YARN cluster"); zkServerComponent = new ZKServerComponent(); yarnComponent = new YarnComponent().withApplicationMasterClass(ApplicationMaster.class).withTestName(MaasIntegrationTest.class.getSimpleName()); runner = new ComponentRunner.Builder() .withComponent("yarn", yarnComponent) .withComponent("zk", zkServerComponent) .withMillisecondsBetweenAttempts(15000) .withNumRetries(10) .build(); runner.start(); String zookeeperUrl = zkServerComponent.getConnectionString(); RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy); client.start(); }
@Before public void setup() throws Exception { // a component that uploads the global configuration Map<String, Object> globals = new HashMap<>(); ConfigUploadComponent configUploader = new ConfigUploadComponent() .withGlobals(globals); // create zookeeper component properties = new Properties(); zkServer = getZKServerComponent(properties); // can only get the zookeeperUrl AFTER it has started zkServer.withPostStartCallback((zk) -> { zookeeperURL = zk.getConnectionString(); configUploader.withZookeeperURL(zookeeperURL); }); // start the integration test components runner = new ComponentRunner.Builder() .withComponent("zk", zkServer) .withComponent("config", configUploader) .build(); runner.start(); context = mock(InterpreterContext.class); }
public void testCiscoPixEnvelopingCisco302020(ParserDriver syslogDriver, ParserDriver driver) throws Exception { byte[] envelopedData = null; String inputRecord = "Mar 29 2004 09:54:18: %PIX-6-302005: Built UDP connection for faddr 198.207.223.240/53337 gaddr 10.0.0.187/53 laddr 192.168.0.2/53"; ProcessorResult<List<byte[]>> syslogResult = syslogDriver.run(ImmutableList.of(inputRecord.getBytes())); Assert.assertFalse(syslogResult.failed()); List<byte[]> syslogResultList = syslogResult.getResult(); envelopedData = syslogResultList.get(0); ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(envelopedData)); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> result = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("UDP", result.get("protocol")); Assert.assertTrue((long) result.get("timestamp") > 1000); } }
@Override public ReadinessState process(ComponentRunner runner) { KafkaComponent kafkaComponent = runner.getComponent("kafka", KafkaComponent.class); try { docs = readDocsFromDisk(hdfsDir); } catch (IOException e) { throw new IllegalStateException("Unable to retrieve indexed documents.", e); } if (docs.size() < inputMessages.size()) { errors = kafkaComponent.readMessages(ERROR_TOPIC); if(errors.size() > 0 && errors.size() + docs.size() == inputMessages.size()){ return ReadinessState.READY; } return ReadinessState.NOT_READY; } else { return ReadinessState.READY; } }
@After public void tearDown() throws Exception { runner.stop(); }
@Override public void start() throws UnableToStartException { try { upload(); } catch (Exception e) { throw new UnableToStartException(e.getMessage(), e); } }
@Override public ReadinessState process(ComponentRunner runner) { KafkaComponent kc = runner.getComponent("kafka", KafkaComponent.class); outputMessages.addAll(readMessagesFromKafka(kc, outputKafkaTopic)); return calcReadiness(inputMessages.size(), outputMessages.size()); }
@After public void tearDown() { runner.reset(); }
@AfterClass public static void stop() { indexComponent.stop(); }
protected static InMemoryComponent startIndex() throws Exception { InMemoryComponent es = new ElasticSearchComponent.Builder() .withHttpPort(9211) .withIndexDir(new File(indexDir)) .withAccessConfig(accessConfig) .build(); es.start(); return es; }
@Override public ProcessorResult<List<Map<String, Object>>> getResult() { ProcessorResult.Builder<List<Map<String,Object>>> builder = new ProcessorResult.Builder(); return builder.withResult(docs).withProcessErrors(errors).build(); } };
public void testEnvelopedData_withMetadataPrefix(ParserDriver driver) throws IOException { Map<String, Object> inputRecord = new HashMap<String, Object>() {{ put(Constants.Fields.ORIGINAL.getName(), "real_original_string"); put("data", "field1_val,100"); put("metadata_field", "metadata_val"); }}; ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(JSONUtils.INSTANCE.toJSONPretty(inputRecord))); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("field1_val", outputRecord.get("field1")); Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName())); Assert.assertEquals(inputRecord.get("metadata_field"), outputRecord.get("metadata_field")); } /**
@Override public ReadinessState process(ComponentRunner runner) { ElasticSearchComponent elasticSearchComponent = runner.getComponent("search", ElasticSearchComponent.class); KafkaComponent kafkaComponent = runner.getComponent("kafka", KafkaComponent.class); if (elasticSearchComponent.hasIndex(index)) { try { docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc"); } catch (IOException e) { throw new IllegalStateException("Unable to retrieve indexed documents.", e); } if (docs.size() < inputMessages.size() ) { errors = kafkaComponent.readMessages(ERROR_TOPIC); if(errors.size() > 0 && errors.size() + docs.size() == inputMessages.size()){ return ReadinessState.READY; } return ReadinessState.NOT_READY; } else { return ReadinessState.READY; } } else { return ReadinessState.NOT_READY; } }
@AfterClass public static void tearDownAfterClass(){ if(client != null){ client.close(); } runner.stop(); }
public static void waitForCluster(Client client, ClusterHealthStatus statusThreshold, String timeout) throws UnableToStartException { try { ClusterHealthResponse healthResponse = (ClusterHealthResponse) client .execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus(statusThreshold).timeout(timeout)) .actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { throw new UnableToStartException("cluster state is " + healthResponse.getStatus().name() + " and not " + statusThreshold.name() + ", from here on, everything will fail!"); } } catch (ElasticsearchTimeoutException e) { throw new UnableToStartException( "timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); } }
@Override public ProcessorResult<List<Map<String, Object>>> getResult() { ProcessorResult.Builder<List<Map<String,Object>>> builder = new ProcessorResult.Builder(); return builder.withResult(docs).withProcessErrors(errors).build(); } };
public void testEnvelopedData_noMergeMetadata(ParserDriver driver) throws IOException { Map<String, Object> inputRecord = new HashMap<String, Object>() {{ put(Constants.Fields.ORIGINAL.getName(), "real_original_string"); put("data", "field1_val,100"); put("metadata_field", "metadata_val"); }}; ProcessorResult<List<byte[]>> results = driver.run(ImmutableList.of(JSONUtils.INSTANCE.toJSONPretty(inputRecord))); Assert.assertFalse(results.failed()); List<byte[]> resultList = results.getResult(); Assert.assertEquals(1, resultList.size()); Map<String, Object> outputRecord = JSONUtils.INSTANCE.load(new String(resultList.get(0)), JSONUtils.MAP_SUPPLIER); Assert.assertEquals("field1_val", outputRecord.get("field1")); Assert.assertEquals(inputRecord.get(Constants.Fields.ORIGINAL.getName()), outputRecord.get(Constants.Fields.ORIGINAL.getName())); Assert.assertFalse(outputRecord.containsKey(MetadataUtil.METADATA_PREFIX + ".metadata_field")); }