public List<Put> toPut(String line , Extractor extractor , String cf , HbaseConverter converter ) throws IOException { List<Put> ret = new ArrayList<>(); Iterable<LookupKV> kvs = extractor.extract(line); for(LookupKV kv : kvs) { Put put = converter.toPut(cf, kv.getKey(), kv.getValue()); ret.add(put); } return ret; }
@Override public void map(Object key, Text value, Context context) throws IOException, InterruptedException { for(LookupKV results : extractor.extract(value.toString())) { if (results != null) { Put put = converter.toPut(columnFamily, results.getKey(), results.getValue()); write(new ImmutableBytesWritable(results.getKey().toBytes()), put, context); } } }
public void load(HTableInterface table, String cf, Iterable<LookupKV<EnrichmentKey, EnrichmentValue>> results) throws IOException { for(LookupKV<EnrichmentKey, EnrichmentValue> result : results) { Put put = converter.toPut(cf, result.getKey(), result.getValue()); table.put(put); } } }
Map<String, Object> ret = lkv.getValue().getMetadata(); Map<String, Object> ind = new LinkedHashMap<>(); String indicator = lkv.getKey().getIndicator(); throw new UnsupportedOperationException("Indicator transform must return String type"); lkv.getKey().setIndicator((String) updatedIndicator); boolean update = filter(indicatorFilter, resolver) && filter(valueFilter, resolver); if(update && !stateUpdate.isEmpty()) {
|| allowedIndicatorTypes.contains(kv.getKey().type) kv.getValue().getMetadata().put("taxii_url", endpoint.toString()); kv.getValue().getMetadata().put("taxii_collection", collection); Put p = converter.toPut(columnFamily, kv.getKey(), kv.getValue()); HTableInterface table = getTable(hbaseTable); table.put(p); LOG.info("Found Threat Intel: {} => ", kv.getKey(), kv.getValue());
@Test public void testBatchOneNormalPath() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertEquals("bar", values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(2, values.get(0).getValue().getMetadata().size()); }
@Test public void testFilteredKeys() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), ImmutableList.of("user", "ip")); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertEquals("localhost", values.get(0).getValue().getMetadata().get("ip")); Assert.assertNull(values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(2, values.get(0).getValue().getMetadata().size()); }
@Test public void testFilteredKey() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), "user"); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertNull(values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(1, values.get(0).getValue().getMetadata().size()); }
}}; for (LookupKV<EnrichmentKey, EnrichmentValue> kv : result.getResult()) { Assert.assertTrue(validIndicators.contains(kv.getKey().indicator)); Assert.assertEquals(kv.getValue().getMetadata().get("source.type"), "dummy"); Assert.assertNotNull(kv.getValue().getMetadata().get("timestamp")); Assert.assertNotNull(kv.getValue().getMetadata().get("original_string")); Map<String, String> metadata = validMetadata.get(kv.getKey().indicator); for (Map.Entry<String, String> x : metadata.entrySet()) { Assert.assertEquals(kv.getValue().getMetadata().get(x.getKey()), x.getValue());
@Test public void testValueConversion() throws IOException { EnrichmentConverter converter = new EnrichmentConverter(); EnrichmentKey k1 = new EnrichmentKey("type", "indicator"); EnrichmentValue v1 = new EnrichmentValue(new HashMap<String, Object>() {{ put("k1", "v1"); put("k2", "v2"); }}); Put serialized = converter.toPut("cf", k1, v1); LookupKV<EnrichmentKey, EnrichmentValue> kv = converter.fromPut(serialized,"cf"); Assert.assertEquals(k1, kv.getKey()); Assert.assertEquals(v1, kv.getValue()); } }