@Override public SensorEnrichmentConfig readConfig(String sensor) throws Exception { SensorEnrichmentConfig sensorEnrichmentConfig = new SensorEnrichmentConfig(); try { sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(sensor, client)); }catch (KeeperException.NoNodeException e) { } return sensorEnrichmentConfig; }
@Override public EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config) { return config.getThreatIntel(); }
@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; SensorEnrichmentConfig that = (SensorEnrichmentConfig) o; if (getEnrichment() != null ? !getEnrichment().equals(that.getEnrichment()) : that.getEnrichment() != null) return false; if (getThreatIntel() != null ? !getThreatIntel().equals(that.getThreatIntel()) : that.getThreatIntel() != null) return false; return getConfiguration() != null ? getConfiguration().equals(that.getConfiguration()) : that.getConfiguration() == null; }
public static EnrichmentConfig getConfig(SensorEnrichmentConfig sensorConfig, Type type) { EnrichmentConfig enrichmentConfig = null; switch(type) { case ENRICHMENT: enrichmentConfig = sensorConfig.getEnrichment(); break; case THREAT_INTEL: case THREATINTEL: enrichmentConfig = sensorConfig.getThreatIntel(); } return enrichmentConfig; }
SensorEnrichmentConfig testSensorConfig = new SensorEnrichmentConfig(); Map<String, Object> enrichmentFieldMap = new HashMap<>(); enrichmentFieldMap.put("enrichmentTest", new ArrayList<String>() {{ add("enrichmentField"); }}); testSensorConfig.getEnrichment().setFieldMap(enrichmentFieldMap); Map<String, Object> threatIntelFieldMap = new HashMap<>(); threatIntelFieldMap.put("threatIntelTest", new ArrayList<String>() {{ add("threatIntelField"); }}); testSensorConfig.getThreatIntel().setFieldMap(threatIntelFieldMap); sampleConfigurations.updateSensorEnrichmentConfig(sensorType, testSensorConfig); ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl);
@Override public Object apply(List<Object> args, Context context) throws ParseException { ThreatTriageProcessor processor; SensorEnrichmentConfig config = new SensorEnrichmentConfig(); // the user can provide an initial config if(args.size() > 0) { String json = Util.getArg(0, String.class, args); if (json != null) { config = (SensorEnrichmentConfig) ENRICHMENT.deserialize(json); } else { throw new IllegalArgumentException(format("Invalid configuration: unable to deserialize '%s'", json)); } } processor = new ThreatTriageProcessor(config, new ClasspathFunctionResolver(), context); return processor; }
@Override public EnrichmentConfig getUnderlyingConfig(SensorEnrichmentConfig config) { return config.getEnrichment(); }
if(config == null) { LOG.debug("Unable to find SensorEnrichmentConfig for sourceType: {}", sourceType); config = new SensorEnrichmentConfig(); config.getConfiguration().putIfAbsent(STELLAR_CONTEXT_CONF, stellarContext); String guid = getGUID(input, message);
@Override public Map<String, Object> getSensorConfig(String sensorName) { return config.orElse(new EnrichmentConfigurations()).getSensorEnrichmentConfig(sensorName) .getConfiguration(); }
@Override public JSONObject enrich(CacheKey value) { Context stellarContext = (Context) value.getConfig().getConfiguration().get(STELLAR_CONTEXT_CONF); ConfigHandler handler = getHandler.apply(value.getConfig()); Map<String, Object> globalConfig = value.getConfig().getConfiguration(); Map<String, Object> sensorConfig = value.getConfig().getEnrichment().getConfig(); if(handler == null) { _LOG.trace("Stellar ConfigHandler is null."); return new JSONObject(); } Long slowLogThreshold = null; if(_PERF_LOG.isDebugEnabled()) { slowLogThreshold = ConversionUtils.convert(globalConfig.getOrDefault(STELLAR_SLOW_LOG, STELLAR_SLOW_LOG_DEFAULT), Long.class); } //Ensure that you clone the message, because process will modify the message. If the message object is modified //then cache misses will happen because the cache will be modified. Map<String, Object> message = new HashMap<>(value.getValue(Map.class)); VariableResolver resolver = new MapVariableResolver(message, sensorConfig, globalConfig); StellarProcessor processor = new StellarProcessor(); JSONObject enriched = process(message , handler , value.getField() , slowLogThreshold , processor , resolver , stellarContext ); _LOG.trace("Stellar Enrichment Success: {}", enriched); return enriched; }
Assert.assertNotNull(finalEnrichmentConfig.get("bro")); Assert.assertNotSame(finalEnrichmentConfig.get("bro"), broSc); Assert.assertEquals( finalEnrichmentConfig.get("bro").toJSON() , ((List<String>)finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap().get(Constants.SIMPLE_HBASE_THREAT_INTEL)).size() , 2 ); Assert.assertEquals(1, finalEnrichmentConfig.get("bro").getThreatIntel().getTriageConfig().getRiskLevelRules().size()); Assert.assertTrue( finalEnrichmentConfig.get("bro").toJSON() , ((List<String>)finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap() .get(Constants.SIMPLE_HBASE_THREAT_INTEL)) .contains("ip_src_addr") ); Assert.assertTrue( finalEnrichmentConfig.get("bro").toJSON() , ((List<String>)finalEnrichmentConfig.get("bro").getThreatIntel().getFieldMap() .get(Constants.SIMPLE_HBASE_THREAT_INTEL)) .contains("ip_dst_addr") ); Assert.assertEquals( finalEnrichmentConfig.get("bro").toJSON() , finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().keySet().size() , 2 ); Assert.assertEquals( finalEnrichmentConfig.get("bro").toJSON() , ((List<String>)(finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_src_addr"))).size() , 2 ); Assert.assertTrue( finalEnrichmentConfig.get("bro").toJSON() , ((List<String>)(finalEnrichmentConfig.get("bro").getThreatIntel().getFieldToTypeMap().get("ip_src_addr"))).contains("playful") ); Assert.assertTrue( finalEnrichmentConfig.get("bro").toJSON()
Assert.assertNotNull(outputScs.get("bro")); Assert.assertNotSame(outputScs.get("bro"), broSc); Assert.assertEquals( outputScs.get("bro").toJSON() , ((List<String>)outputScs.get("bro").getEnrichment().getFieldMap().get(Constants.SIMPLE_HBASE_ENRICHMENT)).size() , 2 ); Assert.assertTrue( outputScs.get("bro").toJSON() , ((List<String>)outputScs.get("bro").getEnrichment().getFieldMap() .get(Constants.SIMPLE_HBASE_ENRICHMENT)) .contains("ip_src_addr") ); Assert.assertTrue( outputScs.get("bro").toJSON() , ((List<String>)outputScs.get("bro").getEnrichment().getFieldMap() .get(Constants.SIMPLE_HBASE_ENRICHMENT)) .contains("ip_dst_addr") ); Assert.assertEquals( outputScs.get("bro").toJSON() , outputScs.get("bro").getEnrichment().getFieldToTypeMap().keySet().size() , 2 ); Assert.assertEquals( outputScs.get("bro").toJSON() , ((List<String>)(outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_src_addr"))).size() , 1 ); Assert.assertEquals( outputScs.get("bro").toJSON() , ((List<String>)(outputScs.get("bro").getEnrichment().getFieldToTypeMap().get("ip_src_addr"))).get(0) , "playful" ); Assert.assertEquals( outputScs.get("bro").toJSON()
fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.SAMPLE_CONFIG_PATH).get(sensorType)); sensorEnrichmentConfig.getConfiguration().put(GenericEnrichmentBolt.STELLAR_CONTEXT_CONF, genericEnrichmentBolt.getStellarContext()); CacheKey cacheKey1 = new CacheKey("field1", "value1", sensorEnrichmentConfig); CacheKey cacheKey2 = new CacheKey("field2", "value2", sensorEnrichmentConfig);
@Test public void testSerDe() throws IOException { for(File enrichmentConfig : new File(new File(TestConstants.ENRICHMENTS_CONFIGS_PATH), "enrichments").listFiles()) { SensorEnrichmentConfig config = null; try (BufferedReader br = new BufferedReader(new FileReader(enrichmentConfig))) { String parserStr = IOUtils.toString(br); config = SensorEnrichmentConfig.fromBytes(parserStr.getBytes()); } SensorEnrichmentConfig config2 = SensorEnrichmentConfig.fromBytes(config.toJSON().getBytes()); Assert.assertEquals(config2, config); } } }
@Test public void test() throws IOException { EqualsVerifier.forClass(SensorEnrichmentConfig.class).suppress(Warning.NONFINAL_FIELDS).usingGetClass().verify(); Map<String, byte[]> testSensorConfigMap = ConfigurationsUtils.readSensorEnrichmentConfigsFromFile(TestConstants.ENRICHMENTS_CONFIGS_PATH); byte[] sensorConfigBytes = testSensorConfigMap.get("yaf"); SensorEnrichmentConfig sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(sensorConfigBytes); Assert.assertNotNull(sensorEnrichmentConfig); Assert.assertTrue(sensorEnrichmentConfig.toString() != null && sensorEnrichmentConfig.toString().length() > 0); }
/** * Reads the Enrichment configuration from Zookeeper. * * @param sensorType The type of sensor. * @param client The Zookeeper client. * @return The Enrichment configuration for the given sensor type, if one exists. Otherwise, null. * @throws Exception */ public static SensorEnrichmentConfig readSensorEnrichmentConfigFromZookeeper(String sensorType, CuratorFramework client) throws Exception { SensorEnrichmentConfig config = null; Optional<byte[]> bytes = readFromZookeeperSafely(ENRICHMENT.getZookeeperRoot() + "/" + sensorType, client); if (bytes.isPresent()) { config = SensorEnrichmentConfig.fromBytes(bytes.get()); } return config; }
private static SensorEnrichmentConfig findConfigBySensorType(SourceConfigHandler scHandler, Map<String, SensorEnrichmentConfig> sourceConfigsChanged, String key) throws Exception { SensorEnrichmentConfig config = sourceConfigsChanged.get(key); if(config == null) { config = scHandler.readConfig(key); if(LOG.isDebugEnabled()) { LOG.debug(config.toJSON()); } } return config; }
List<String> fieldList = null; if(kv.getValue().type == Type.THREAT_INTEL) { fieldMap = config.getThreatIntel().getFieldMap(); if(fieldMap!= null) { fieldList = (List<String>)fieldMap.get(Constants.SIMPLE_HBASE_THREAT_INTEL); fieldMap.put(Constants.SIMPLE_HBASE_THREAT_INTEL, fieldList); fieldToTypeMap = config.getThreatIntel().getFieldToTypeMap(); if(fieldToTypeMap == null) { fieldToTypeMap = new HashMap<>(); config.getThreatIntel().setFieldToTypeMap(fieldToTypeMap); fieldMap = config.getEnrichment().getFieldMap(); if(fieldMap!= null) { fieldList = (List<String>)fieldMap.get(Constants.SIMPLE_HBASE_ENRICHMENT); fieldMap.put(Constants.SIMPLE_HBASE_ENRICHMENT, fieldList); fieldToTypeMap = config.getEnrichment().getFieldToTypeMap(); if(fieldToTypeMap == null) { fieldToTypeMap = new HashMap<>(); config.getEnrichment().setFieldToTypeMap(fieldToTypeMap);
/** * Retrieve the Enrichment configuration. * * @param args The function arguments. * @return The Enrichment configuration as a JSON string. * @throws Exception */ private String getEnrichmentConfig(List<Object> args) throws Exception { // retrieve the enrichment config for the given sensor String sensor = getArg(1, String.class, args); SensorEnrichmentConfig sensorConfig = readSensorEnrichmentConfigFromZookeeper(sensor, zkClient); // provide empty/default config if one is not present? if(sensorConfig == null && emptyIfNotPresent(args)) { sensorConfig = new SensorEnrichmentConfig(); } return toJSON(sensorConfig); }
public Map<String, Object> getFieldMap(String sourceType) { if(sourceType != null) { SensorEnrichmentConfig config = getConfigurations().getSensorEnrichmentConfig(sourceType); if (config != null && config.getEnrichment() != null) { return config.getEnrichment().getFieldMap(); } else { LOG.debug("Unable to retrieve a sensor enrichment config of {}", sourceType); } } else { LOG.error("Trying to retrieve a field map with source type of null"); } return null; } }