private void emitErrorsAsResult(String id, Metadata metadata, BulletError... errors) { emitErrorsAsResult(id, metadata, Arrays.asList(errors)); }
private void emitOrBufferFinished(String id, Querier querier) { /* * Three cases: * 1) If we shouldn't buffer, then emit it and return. If it was being delayed and somehow finished, it is * cleaned up and removed. There should be no query that needs delaying AND buffering. * 2) If the query became closed after it finished (wherever it is), we emit it. We should still honor isClosed. * 3) If it should buffer and it isn't closed, postFinishBuffer it till it becomes closed or ticks emit it. */ if (!querier.shouldBuffer()) { log.debug("Emitting query since it shouldn't be buffered {}", id); emitFinished(id, querier); } else if (querier.isClosed()) { log.debug("Emitting query since it finished but this is the last window for it {}", id); emitFinished(id, querier); } else if (queries.containsKey(id)) { log.debug("Starting to buffer while waiting for more final results for query {}", id); queries.remove(id); postFinishBuffer.put(id, querier); } }
private void emitFinished(String id, Querier querier) { log.info("Query is done {}...", id); emitResult(id, withSignal(bufferedMetadata.get(id), Metadata.Signal.COMPLETE), querier.finish()); emitMetaSignal(id, Metadata.Signal.COMPLETE); removeQuery(id); }
private void emitErrorsAsResult(String id, Metadata metadata, List<BulletError> errors) { updateCount(improperQueriesCount, 1L); emitResult(id, withSignal(metadata, Metadata.Signal.FAIL), Clip.of(Meta.of(errors))); }
private void onError(Tuple tuple) { String id = tuple.getString(TopologyConstants.ID_POSITION); Querier querier = getQuery(id); if (querier == null) { log.debug("Received error for {} without the query existing", id); // TODO Might later create this query if it is received late but whose error was ignored here. This is a leak. return; } RateLimitError error = (RateLimitError) tuple.getValue(TopologyConstants.ERROR_POSITION); emitRateLimitError(id, querier, error); }
private void onQuery(Tuple tuple) { String id = tuple.getString(TopologyConstants.ID_POSITION); String query = tuple.getString(TopologyConstants.QUERY_POSITION); Metadata metadata = (Metadata) tuple.getValue(TopologyConstants.QUERY_METADATA_POSITION); // bufferedMetadata has an entry for each query that exists in the JoinBolt; therefore, we check bufferedMetadata // for existing queries (as opposed to individually checking the queries, preStartBuffer, and postFinishBuffer maps) if (bufferedMetadata.containsKey(id)) { updateCount(duplicatedQueriesCount, 1L); log.error("Duplicate for request {} with query {}", id, query); return; } Querier querier; try { querier = createQuerier(Querier.Mode.ALL, id, query, config); Optional<List<BulletError>> optionalErrors = querier.initialize(); if (!optionalErrors.isPresent()) { setupQuery(id, query, metadata, querier); return; } emitErrorsAsResult(id, metadata, optionalErrors.get()); } catch (RuntimeException re) { // Includes JSONParseException emitErrorsAsResult(id, metadata, ParsingError.makeError(re, query)); } log.error("Failed to initialize query for request {} with query {}", id, query); }
@Test public void testImproperQueryCountingMetrics() { config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true); setup(new JoinBolt(config)); Assert.assertEquals(context.getLongMetric(TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(0)); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); Assert.assertEquals(context.getLongMetric(TopologyConstants.IMPROPER_QUERIES_METRIC), Long.valueOf(0)); Tuple badQuery = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{", EMPTY); bolt.execute(badQuery); bolt.execute(badQuery); Assert.assertEquals(context.getLongMetric(TopologyConstants.CREATED_QUERIES_METRIC), Long.valueOf(0)); Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0)); Assert.assertEquals(context.getLongMetric(TopologyConstants.IMPROPER_QUERIES_METRIC), Long.valueOf(2)); }
private void onData(Tuple tuple) { String id = tuple.getString(TopologyConstants.ID_POSITION); Querier querier = getQuery(id); if (querier == null) { log.debug("Received data for query {} before query. Ignoring...", id); return; } byte[] data = (byte[]) tuple.getValue(TopologyConstants.DATA_POSITION); querier.combine(data); if (querier.isDone()) { emitOrBufferFinished(id, querier); } else if (querier.isExceedingRateLimit()) { emitRateLimitError(id, querier, querier.getRateLimitError()); } else if (querier.isClosed()) { emitWindow(id, querier); } }
@Override protected Querier createQuerier(Querier.Mode mode, String id, String query, BulletConfig config) { Querier spied = spy(super.createQuerier(mode, id, query, config)); List<Boolean> doneAnswers = IntStream.range(0, doneAfter).mapToObj(i -> false) .collect(ArrayList::new, ArrayList::add, ArrayList::addAll); doneAnswers.add(true); doAnswer(returnsElementsOf(doneAnswers)).when(spied).isDone(); doReturn(shouldBuffer).when(spied).shouldBuffer(); return spied; } }
@BeforeMethod public void setup() { config = configWithRawMaxAndNoMeta(); setup(new JoinBolt(config)); }
private void emitWindow(String id, Querier querier) { // No matter where it is - emit and reset. log.debug("Emitting window for {} and resetting...", id); emitResult(id, bufferedMetadata.get(id), querier.getResult()); querier.reset(); // We should not receive window for queries in the pre-start buffer because those are only time-based windowed // queries that the config ensures have a minimum emit time greater than the pre-start delay. }
private void emitRateLimitError(Map.Entry<String, Querier> query) { Querier querier = query.getValue(); emitRateLimitError(query.getKey(), querier, querier.getRateLimitError()); }
private void emitWindow(Map.Entry<String, Querier> query) { emitWindow(query.getKey(), query.getValue()); }
private void emitFinished(Map.Entry<String, Querier> query) { emitFinished(query.getKey(), query.getValue()); }
@Test public void testOutputFields() { CustomOutputFieldsDeclarer declarer = new CustomOutputFieldsDeclarer(); bolt.declareOutputFields(declarer); Fields expectedResultFields = new Fields(TopologyConstants.ID_FIELD, TopologyConstants.RESULT_FIELD, TopologyConstants.METADATA_FIELD); Fields expectedFeedbackFields = new Fields(TopologyConstants.ID_FIELD, TopologyConstants.METADATA_FIELD); Assert.assertTrue(declarer.areFieldsPresent(TopologyConstants.RESULT_STREAM, false, expectedResultFields)); Assert.assertTrue(declarer.areFieldsPresent(TopologyConstants.FEEDBACK_STREAM, false, expectedFeedbackFields)); }
private void onQuery(Tuple tuple) { String id = tuple.getString(TopologyConstants.ID_POSITION); String query = tuple.getString(TopologyConstants.QUERY_POSITION); Metadata metadata = (Metadata) tuple.getValue(TopologyConstants.QUERY_METADATA_POSITION); // bufferedMetadata has an entry for each query that exists in the JoinBolt; therefore, we check bufferedMetadata // for existing queries (as opposed to individually checking the queries, preStartBuffer, and postFinishBuffer maps) if (bufferedMetadata.containsKey(id)) { updateCount(duplicatedQueriesCount, 1L); log.error("Duplicate for request {} with query {}", id, query); return; } Querier querier; try { querier = createQuerier(Querier.Mode.ALL, id, query, config); Optional<List<BulletError>> optionalErrors = querier.initialize(); if (!optionalErrors.isPresent()) { setupQuery(id, query, metadata, querier); return; } emitErrorsAsResult(id, metadata, optionalErrors.get()); } catch (RuntimeException re) { // Includes JSONParseException emitErrorsAsResult(id, metadata, ParsingError.makeError(re, query)); } log.error("Failed to initialize query for request {} with query {}", id, query); }
private void onData(Tuple tuple) { String id = tuple.getString(TopologyConstants.ID_POSITION); Querier querier = getQuery(id); if (querier == null) { log.debug("Received data for query {} before query. Ignoring...", id); return; } byte[] data = (byte[]) tuple.getValue(TopologyConstants.DATA_POSITION); querier.combine(data); if (querier.isDone()) { emitOrBufferFinished(id, querier); } else if (querier.isExceedingRateLimit()) { emitRateLimitError(id, querier, querier.getRateLimitError()); } else if (querier.isClosed()) { emitWindow(id, querier); } }
private void emitErrorsAsResult(String id, Metadata metadata, List<BulletError> errors) { updateCount(improperQueriesCount, 1L); emitResult(id, withSignal(metadata, Metadata.Signal.FAIL), Clip.of(Meta.of(errors))); }
@Override protected Querier createQuerier(Querier.Mode mode, String id, String query, BulletConfig config) { Querier spied = spy(super.createQuerier(mode, id, query, config)); List<Boolean> answers = IntStream.range(0, limitedAfter).mapToObj(i -> false) .collect(ArrayList::new, ArrayList::add, ArrayList::addAll); answers.add(true); doAnswer(returnsElementsOf(answers)).when(spied).isExceedingRateLimit(); doReturn(error).when(spied).getRateLimitError(); return spied; } }