@Override public LookupKV<EnrichmentKey, EnrichmentValue> fromResult(Result result, String columnFamily) throws IOException { return fromResult(result, columnFamily, new EnrichmentKey(), new EnrichmentValue()); } }
@Override public LookupKV<EnrichmentKey, EnrichmentValue> fromPut(Put put, String columnFamily) throws IOException { return fromPut(put, columnFamily, new EnrichmentKey(), new EnrichmentValue()); }
@Override public LookupValue toValue(Map<String, Object> metadata) { return new EnrichmentValue(metadata); } })
@Test public void testValueConversion() throws IOException { EnrichmentConverter converter = new EnrichmentConverter(); EnrichmentKey k1 = new EnrichmentKey("type", "indicator"); EnrichmentValue v1 = new EnrichmentValue(new HashMap<String, Object>() {{ put("k1", "v1"); put("k2", "v2"); }}); Put serialized = converter.toPut("cf", k1, v1); LookupKV<EnrichmentKey, EnrichmentValue> kv = converter.fromPut(serialized,"cf"); Assert.assertEquals(k1, kv.getKey()); Assert.assertEquals(v1, kv.getValue()); } }
@Before public void setup() throws Exception { final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(atTableName, cf); final MockHTable threatIntelTable = (MockHTable) MockHBaseTableProvider.addToCache(threatIntelTableName, cf); EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{ add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<>()))); }}); BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03); PersistentAccessTracker pat = new PersistentAccessTracker(threatIntelTableName, "0", trackerTable, cf, bat, 0L); lookup = new EnrichmentLookup(threatIntelTable, cf, pat); JSONParser jsonParser = new JSONParser(); expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString); }
@Override public void init(Map stormConf, TopologyContext topologyContext, WriterConfiguration configuration) throws Exception { if(converter == null) { converter = new EnrichmentConverter(); } }
@Override public int hashCode() { return getMetadata() != null ? getMetadata().hashCode() : 0; }
@Test public void testKeyConversion() { EnrichmentKey k1 = new EnrichmentKey("type", "indicator1"); byte[] serialized = k1.toBytes(); EnrichmentKey k2 = new EnrichmentKey(); k2.fromBytes(serialized); Assert.assertEquals(k1, k2); }
@Override public LookupKey toKey(String type, String indicator) { return new EnrichmentKey(type, indicator); }
public void load(HTableInterface table, String cf, Iterable<LookupKV<EnrichmentKey, EnrichmentValue>> results) throws IOException { for(LookupKV<EnrichmentKey, EnrichmentValue> result : results) { Put put = converter.toPut(cf, result.getKey(), result.getValue()); table.put(put); } } }
@Override public Iterable<Map.Entry<byte[], byte[]>> toColumns() { return AbstractConverter.toEntries( VALUE_COLUMN_NAME_B, Bytes.toBytes(valueToString(metadata)) ); }
@Override public LookupKV<EnrichmentKey, EnrichmentValue> get(EnrichmentKey key, HBaseContext context, boolean logAccess) throws IOException { return converter.fromResult(context.getTable().get(converter.toGet(getColumnFamily(context), key)), getColumnFamily(context)); }
@Override public void fromColumns(Iterable<Map.Entry<byte[], byte[]>> values) { for(Map.Entry<byte[], byte[]> cell : values) { if(Bytes.equals(cell.getKey(), VALUE_COLUMN_NAME_B)) { metadata = stringToValue(Bytes.toString(cell.getValue())); } } } public Map<String, Object> stringToValue(String s){
@Override public byte[] toBytes() { byte[] indicatorBytes = new byte[0]; try { indicatorBytes = typedIndicatorToBytes(); } catch (IOException e) { throw new RuntimeException("Unable to convert type and indicator to bytes", e); } byte[] prefix = KeyUtil.INSTANCE.getPrefix(Bytes.toBytes(indicator)); return KeyUtil.INSTANCE.merge(prefix, indicatorBytes); }
@Override public Result toResult(String columnFamily, KEY_T key, VALUE_T values) throws IOException { Put put = toPut(columnFamily, key, values); return Result.create(put.getFamilyCellMap().get(Bytes.toBytes(columnFamily))); }
@Override public boolean exists(EnrichmentKey key, HBaseContext context, boolean logAccess) throws IOException { return context.getTable().exists(converter.toGet(getColumnFamily(context), key)); }
@Before public void setup() throws Exception { final MockHTable hbaseTable = (MockHTable) MockHBaseTableProvider.addToCache(hbaseTableName, cf); EnrichmentHelper.INSTANCE.load(hbaseTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{ for(int i = 0;i < 5;++i) { add(new LookupKV<>(new EnrichmentKey(ENRICHMENT_TYPE, "indicator" + i) , new EnrichmentValue(ImmutableMap.of("key" + i, "value" + i)) ) ); } }}); context = new Context.Builder() .with( Context.Capabilities.GLOBAL_CONFIG , () -> ImmutableMap.of( SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF , MockHBaseTableProvider.class.getName() ) ) .build(); } public Object run(String rule, Map<String, Object> variables) throws Exception {
@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; EnrichmentValue that = (EnrichmentValue) o; return getMetadata() != null ? getMetadata().equals(that.getMetadata()) : that.getMetadata() == null; }
private EnrichmentKey getKey(JSONObject message, KeyTransformer transformer, String enrichmentType) { if(enrichmentType != null) { return new EnrichmentKey(enrichmentType, transformer.transform(message)); } else { return null; } }
@Nullable @Override public KeyWithContext<EnrichmentKey, EnrichmentLookup.HBaseContext> apply(@Nullable String enrichmentType) { EnrichmentKey key = new EnrichmentKey(enrichmentType, indicator); EnrichmentLookup.HBaseContext context = new EnrichmentLookup.HBaseContext(table, getColumnFamily(enrichmentType, config)); return new KeyWithContext<>(key, context); } }