private void startDelayed(Map.Entry<String, Querier> query) { String id = query.getKey(); Querier querier = query.getValue(); preStartBuffer.remove(id); // Make the query start again to mark the correct start for the query. querier.restart(); queries.put(id, querier); // Now it's active updateCount(activeQueriesCount, 1L); log.info("Started delayed query {}", id); }
private void setupQuery(String id, String query, Metadata metadata, Querier querier) { updateCount(createdQueriesCount, 1L); bufferedMetadata.put(id, metadata); // If the query should be post-finish buffered, it should not be pre-start delayed. if (querier.shouldBuffer()) { queries.put(id, querier); updateCount(activeQueriesCount, 1L); log.info("Received and started query {}", querier.toString()); } else { preStartBuffer.put(id, querier); log.info("Received but delaying starting query {}", id); } }
private void startDelayed(Map.Entry<String, Querier> query) { String id = query.getKey(); Querier querier = query.getValue(); preStartBuffer.remove(id); // Make the query start again to mark the correct start for the query. querier.restart(); queries.put(id, querier); // Now it's active updateCount(activeQueriesCount, 1L); log.info("Started delayed query {}", id); }
private void setupQuery(String id, String query, Metadata metadata, Querier querier) { updateCount(createdQueriesCount, 1L); bufferedMetadata.put(id, metadata); // If the query should be post-finish buffered, it should not be pre-start delayed. if (querier.shouldBuffer()) { queries.put(id, querier); updateCount(activeQueriesCount, 1L); log.info("Received and started query {}", querier.toString()); } else { preStartBuffer.put(id, querier); log.info("Received but delaying starting query {}", id); } }
@Override protected void removeQuery(String id) { // Only update count if query was in queries or postFinishBuffer. if (queries.containsKey(id) || postFinishBuffer.containsKey(id)) { updateCount(activeQueriesCount, -1L); } queries.remove(id); postFinishBuffer.remove(id); bufferedMetadata.remove(id); // It should not be in the preStartBuffer under normal operations but could be if it was killed. preStartBuffer.remove(id); }
@Override protected void removeQuery(String id) { // Only update count if query was in queries or postFinishBuffer. if (queries.containsKey(id) || postFinishBuffer.containsKey(id)) { updateCount(activeQueriesCount, -1L); } queries.remove(id); postFinishBuffer.remove(id); bufferedMetadata.remove(id); // It should not be in the preStartBuffer under normal operations but could be if it was killed. preStartBuffer.remove(id); }
private void onTick() { // Force emit all the done queries queries that are being rotated out. Map<String, Querier> forceDone = postFinishBuffer.rotate(); forceDone.entrySet().forEach(this::emitFinished); // The active queries count is not updated for these since these queries are not in any map, so do it here updateCount(activeQueriesCount, -forceDone.size()); // Start all the delayed queries and add them to queries Map<String, Querier> delayed = preStartBuffer.rotate(); delayed.entrySet().forEach(this::startDelayed); // Categorize all the active queries and do the buffering or emit as necessary handleCategorizedQueries(new QueryCategorizer().categorize(queries)); }
private void onTick() { // Force emit all the done queries queries that are being rotated out. Map<String, Querier> forceDone = postFinishBuffer.rotate(); forceDone.entrySet().forEach(this::emitFinished); // The active queries count is not updated for these since these queries are not in any map, so do it here updateCount(activeQueriesCount, -forceDone.size()); // Start all the delayed queries and add them to queries Map<String, Querier> delayed = preStartBuffer.rotate(); delayed.entrySet().forEach(this::startDelayed); // Categorize all the active queries and do the buffering or emit as necessary handleCategorizedQueries(new QueryCategorizer().categorize(queries)); }
private void emitErrorsAsResult(String id, Metadata metadata, List<BulletError> errors) { updateCount(improperQueriesCount, 1L); emitResult(id, withSignal(metadata, Metadata.Signal.FAIL), Clip.of(Meta.of(errors))); }
private void emitErrorsAsResult(String id, Metadata metadata, List<BulletError> errors) { updateCount(improperQueriesCount, 1L); emitResult(id, withSignal(metadata, Metadata.Signal.FAIL), Clip.of(Meta.of(errors))); }
private void onQuery(Tuple tuple) { String id = tuple.getString(TopologyConstants.ID_POSITION); String query = tuple.getString(TopologyConstants.QUERY_POSITION); Metadata metadata = (Metadata) tuple.getValue(TopologyConstants.QUERY_METADATA_POSITION); // bufferedMetadata has an entry for each query that exists in the JoinBolt; therefore, we check bufferedMetadata // for existing queries (as opposed to individually checking the queries, preStartBuffer, and postFinishBuffer maps) if (bufferedMetadata.containsKey(id)) { updateCount(duplicatedQueriesCount, 1L); log.error("Duplicate for request {} with query {}", id, query); return; } Querier querier; try { querier = createQuerier(Querier.Mode.ALL, id, query, config); Optional<List<BulletError>> optionalErrors = querier.initialize(); if (!optionalErrors.isPresent()) { setupQuery(id, query, metadata, querier); return; } emitErrorsAsResult(id, metadata, optionalErrors.get()); } catch (RuntimeException re) { // Includes JSONParseException emitErrorsAsResult(id, metadata, ParsingError.makeError(re, query)); } log.error("Failed to initialize query for request {} with query {}", id, query); }
private void onQuery(Tuple tuple) { String id = tuple.getString(TopologyConstants.ID_POSITION); String query = tuple.getString(TopologyConstants.QUERY_POSITION); Metadata metadata = (Metadata) tuple.getValue(TopologyConstants.QUERY_METADATA_POSITION); // bufferedMetadata has an entry for each query that exists in the JoinBolt; therefore, we check bufferedMetadata // for existing queries (as opposed to individually checking the queries, preStartBuffer, and postFinishBuffer maps) if (bufferedMetadata.containsKey(id)) { updateCount(duplicatedQueriesCount, 1L); log.error("Duplicate for request {} with query {}", id, query); return; } Querier querier; try { querier = createQuerier(Querier.Mode.ALL, id, query, config); Optional<List<BulletError>> optionalErrors = querier.initialize(); if (!optionalErrors.isPresent()) { setupQuery(id, query, metadata, querier); return; } emitErrorsAsResult(id, metadata, optionalErrors.get()); } catch (RuntimeException re) { // Includes JSONParseException emitErrorsAsResult(id, metadata, ParsingError.makeError(re, query)); } log.error("Failed to initialize query for request {} with query {}", id, query); }
private void emitRateLimitError(String id, Querier querier, RateLimitError error) { Metadata metadata = bufferedMetadata.get(id); Meta meta = error.makeMeta(); Clip clip = querier.finish(); clip.getMeta().merge(meta); emitResult(id, withSignal(metadata, Metadata.Signal.FAIL), clip); emitMetaSignal(id, Metadata.Signal.KILL); updateCount(rateExceededQueries, 1L); removeQuery(id); }
private void emitRateLimitError(String id, Querier querier, RateLimitError error) { Metadata metadata = bufferedMetadata.get(id); Meta meta = error.makeMeta(); Clip clip = querier.finish(); clip.getMeta().merge(meta); emitResult(id, withSignal(metadata, Metadata.Signal.FAIL), clip); emitMetaSignal(id, Metadata.Signal.KILL); updateCount(rateExceededQueries, 1L); removeQuery(id); }