public LookupKV<KEY_T, VALUE_T> fromResult(Result result, String columnFamily, KEY_T key, VALUE_T value) throws IOException { if(result == null || result.getRow() == null) { return null; } key.fromBytes(result.getRow()); byte[] cf = Bytes.toBytes(columnFamily); NavigableMap<byte[], byte[]> cols = result.getFamilyMap(cf); value.fromColumns(cols.entrySet()); return new LookupKV<>(key, value); } @Override
public List<Put> toPut(String line , Extractor extractor , String cf , HbaseConverter converter ) throws IOException { List<Put> ret = new ArrayList<>(); Iterable<LookupKV> kvs = extractor.extract(line); for(LookupKV kv : kvs) { Put put = converter.toPut(cf, kv.getKey(), kv.getValue()); ret.add(put); } return ret; }
@Override public Iterable<LookupKV> extract(String line) throws IOException { if(ignore(line)) { return Collections.emptyList(); } String[] tokens = parser.parseLine(line); LookupKey key = converter.toKey(getType(tokens), tokens[indicatorColumn]); Map<String, Object> values = new HashMap<>(); for(Map.Entry<String, Integer> kv : columnMap.entrySet()) { values.put(kv.getKey(), tokens[kv.getValue()]); } return Arrays.asList(new LookupKV(key, converter.toValue(values))); }
@Override public void map(Object key, Text value, Context context) throws IOException, InterruptedException { for(LookupKV results : extractor.extract(value.toString())) { if (results != null) { Put put = converter.toPut(columnFamily, results.getKey(), results.getValue()); write(new ImmutableBytesWritable(results.getKey().toBytes()), put, context); } } }
public LookupKV<KEY_T, VALUE_T> fromPut(Put put, String columnFamily, KEY_T key, VALUE_T value) throws IOException { key.fromBytes(put.getRow()); byte[] cf = Bytes.toBytes(columnFamily); value.fromColumns(Iterables.transform(put.getFamilyCellMap().get(cf), CELL_TO_ENTRY)); return new LookupKV<>(key, value); }
Map<String, Object> ret = lkv.getValue().getMetadata(); Map<String, Object> ind = new LinkedHashMap<>(); String indicator = lkv.getKey().getIndicator(); throw new UnsupportedOperationException("Indicator transform must return String type"); lkv.getKey().setIndicator((String) updatedIndicator); boolean update = filter(indicatorFilter, resolver) && filter(valueFilter, resolver); if(update && !stateUpdate.isEmpty()) {
for(String token : StixExtractor.split(value)) { final String indicatorType = typeStr + ":" + category; LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token) , new EnrichmentValue( new HashMap<String, Object>() {{
if (kv != null && kv.getValue() != null && kv.getValue().getMetadata() != null) { for (Map.Entry<String, Object> values : kv.getValue().getMetadata().entrySet()) { enriched.put(kv.getKey().type + "." + values.getKey(), values.getValue()); LOG.trace("Enriched type {} => {}", kv.getKey().type, enriched);
@Override public Iterable<LookupKV> extract(final Hostname type, Map<String, Object> config) throws IOException { StringObjectPropertyType value = type.getHostnameValue(); String typeStr = getType(); if(config != null) { Object o = config.get(TYPE_CONFIG); if(o != null) { typeStr = o.toString(); } } List<LookupKV> ret = new ArrayList<>(); for(String token : StixExtractor.split(value)) { final String indicatorType = typeStr; LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token) , new EnrichmentValue(new HashMap<String, Object>() {{ put("source-type", "STIX"); put("indicator-type", indicatorType); put("source", type.toXMLString()); }} ) ); ret.add(results); } return ret; } @Override
|| allowedIndicatorTypes.contains(kv.getKey().type) kv.getValue().getMetadata().put("source_type", "taxii"); kv.getValue().getMetadata().put("taxii_url", endpoint.toString()); kv.getValue().getMetadata().put("taxii_collection", collection); Put p = converter.toPut(columnFamily, kv.getKey(), kv.getValue()); HTableInterface table = getTable(hbaseTable); table.put(p); LOG.info("Found Threat Intel: {} => ", kv.getKey(), kv.getValue());
@Override public Iterable<LookupKV> extract(final DomainName type, Map<String, Object> config) throws IOException { List<LookupKV> ret = new ArrayList<>(); String typeStr = getType(); if(config != null) { Object o = config.get(TYPE_CONFIG); if(o != null) { typeStr = o.toString(); } } final DomainNameTypeEnum domainType = type.getType(); if(domainType == null || SUPPORTED_TYPES.contains(domainType)) { StringObjectPropertyType value = type.getValue(); for (String token : StixExtractor.split(value)) { final String indicatorType = typeStr + ":" + DomainNameTypeEnum.FQDN; LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token) , new EnrichmentValue( new HashMap<String, Object>() {{ put("source-type", "STIX"); put("indicator-type", indicatorType); put("source", type.toXMLString()); }} ) ); ret.add(results); } } return ret; } @Override
@Test public void testBatchOneNormalPath() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertEquals("bar", values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(2, values.get(0).getValue().getMetadata().size()); }
@Override public Iterable<LookupKV> extract(URIObjectType type, Map<String, Object> config) throws IOException { List<LookupKV> ret = new ArrayList<>(); if(type != null) { AnyURIObjectPropertyType val = type.getValue(); if(val != null) { Object v = val.getValue(); if(v != null) { final String indicatorType = getType(); LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, v.toString()) , new EnrichmentValue( new HashMap<String, Object>() {{ put("source-type", "STIX"); put("uri", v.toString()); put("indicator-type", indicatorType); put("source", type.toXMLString()); }} ) ); ret.add(results); } } } return ret; }
@Test public void testFilteredKeys() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), ImmutableList.of("user", "ip")); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertEquals("localhost", values.get(0).getValue().getMetadata().get("ip")); Assert.assertNull(values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(2, values.get(0).getValue().getMetadata().size()); }
@Before public void setup() throws Exception { final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(atTableName, cf); final MockHTable threatIntelTable = (MockHTable) MockHBaseTableProvider.addToCache(threatIntelTableName, cf); EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{ add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<>()))); }}); BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03); PersistentAccessTracker pat = new PersistentAccessTracker(threatIntelTableName, "0", trackerTable, cf, bat, 0L); lookup = new EnrichmentLookup(threatIntelTable, cf, pat); JSONParser jsonParser = new JSONParser(); expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString); }
@Test public void testFilteredKey() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), "user"); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertNull(values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(1, values.get(0).getValue().getMetadata().size()); }
@Before public void setup() throws Exception { final MockHTable hbaseTable = (MockHTable) MockHBaseTableProvider.addToCache(hbaseTableName, cf); EnrichmentHelper.INSTANCE.load(hbaseTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{ for(int i = 0;i < 5;++i) { add(new LookupKV<>(new EnrichmentKey(ENRICHMENT_TYPE, "indicator" + i) , new EnrichmentValue(ImmutableMap.of("key" + i, "value" + i)) ) ); } }}); context = new Context.Builder() .with( Context.Capabilities.GLOBAL_CONFIG , () -> ImmutableMap.of( SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF , MockHBaseTableProvider.class.getName() ) ) .build(); } public Object run(String rule, Map<String, Object> variables) throws Exception {
@Test public void testValueConversion() throws IOException { EnrichmentConverter converter = new EnrichmentConverter(); EnrichmentKey k1 = new EnrichmentKey("type", "indicator"); EnrichmentValue v1 = new EnrichmentValue(new HashMap<String, Object>() {{ put("k1", "v1"); put("k2", "v2"); }}); Put serialized = converter.toPut("cf", k1, v1); LookupKV<EnrichmentKey, EnrichmentValue> kv = converter.fromPut(serialized,"cf"); Assert.assertEquals(k1, kv.getKey()); Assert.assertEquals(v1, kv.getValue()); } }