@Override public void cleanup() { super.cleanup(); if (client != null) { try { client.close(); } catch (final IOException ioe) { LOGGER.error("Failed to close client", ioe); } } }
@Override public Map<String, Object> getComponentConfiguration() { return TupleUtils.putTickFrequencyIntoComponentConfig(super.getComponentConfiguration(), flushIntervalInSeconds); } }
@Override public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) { EventLoggingOutputCollector collector = new EventLoggingOutputCollector(topologyContext, outputCollector, TestRunEventLogger.getEventLogger(eventLogFilePath)); processorBolt.prepare(map, topologyContext, collector); }
@Test public void testDodgyURL() throws IOException { TestOutputCollector output = new TestOutputCollector(); Map config = new HashMap(); config.put("http.agent.name", "this is only a test"); bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); Tuple tuple = mock(Tuple.class); when(tuple.getSourceComponent()).thenReturn("source"); when(tuple.getStringByField("url")).thenReturn("ahahaha"); when(tuple.getValueByField("metadata")).thenReturn(null); bolt.execute(tuple); boolean acked = output.getAckedTuples().contains(tuple); boolean failed = output.getAckedTuples().contains(tuple); // should be acked or failed Assert.assertEquals(true, acked || failed); List<List<Object>> statusTuples = output .getEmitted(Constants.StatusStreamName); // we should get one tuple on the status stream // to notify that the URL is an error Assert.assertEquals(1, statusTuples.size()); }
@Override public void execute(Tuple tuple) { processorBolt.execute(tuple); }
@Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { processorBolt.declareOutputFields(outputFieldsDeclarer); }
protected void prepareParserBolt(String configFile, Map parserConfig) { parserConfig.put("parsefilters.config.file", configFile); bolt.prepare(parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); }
protected void parse(String url, byte[] content, Metadata metadata) throws IOException { Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); when(tuple.getStringByField("url")).thenReturn(url); when(tuple.getValueByField("metadata")).thenReturn(metadata); bolt.execute(tuple); }
heartBeatTimer.cancel(); super.cleanup(); LOG.info("Hive Bolt stopped");
@Override public Map<String, Object> getComponentConfiguration() { return TupleUtils.putTickFrequencyIntoComponentConfig(super.getComponentConfiguration(), tickTupleInterval); }
@Test public void testFeedParsingDetextBytes() throws IOException { Map parserConfig = new HashMap(); parserConfig.put("feed.sniffContent", true); parserConfig.put("parsefilters.config.file", "test.parsefilters.json"); bolt.prepare(parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); Metadata metadata = new Metadata(); parse("http://www.guardian.com/feed.xml", "guardian.rss", metadata); checkOutput(); }
protected void parse(String url, String filename, Metadata metadata) throws IOException { byte[] content = readContent(filename); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); when(tuple.getStringByField("url")).thenReturn(url); when(tuple.getValueByField("metadata")).thenReturn(metadata); bolt.execute(tuple); }
public void close() { if(COMMAND_BOLT != null) COMMAND_BOLT.cleanup(); COMMAND_BOLT = null; TOPOLOGY = null; }
@Override public Map<String, Object> getComponentConfiguration() { Map<String, Object> conf = super.getComponentConfiguration(); if (conf == null) { conf = new Config(); } if (options.getTickTupleInterval() > 0) { conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, options.getTickTupleInterval()); } return conf; }
@Test /** * Checks that content in script is not included in the text representation **/ public void testNoScriptInText() throws IOException { bolt.prepare(new HashMap(), TestUtil.getMockedTopologyContext(), new OutputCollector(output)); parse("http://www.digitalpebble.com", "digitalpebble.com.html"); List<Object> parsedTuple = output.getEmitted().remove(0); // check in the metadata that the values match String text = (String) parsedTuple.get(3); Assert.assertFalse( "Text should not contain the content of script tags", text.contains("urchinTracker")); }
@Override public void cleanup() { processorBolt.cleanup(); }
@Override public Map<String, Object> getComponentConfiguration() { return processorBolt.getComponentConfiguration(); }
@Test public void testExecuteWithOutlinksLimitDisabled() throws IOException { stormConf.put("parser.emitOutlinks.max.per.page", -1); bolt.prepare(stormConf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); parse("http://www.digitalpebble.com", "digitalpebble.com.html"); List<List<Object>> statusTuples = output .getEmitted(Constants.StatusStreamName); // outlinks NOT being limited by property, since is disabled with -1 Assert.assertEquals(10, statusTuples.size()); }
@Override public void cleanup() { // TODO Auto-generated method stub super.cleanup(); this.signalConnection.close(); }