@Override public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<MESSAGE_T> messages) throws Exception { BulkWriterResponse response = new BulkWriterResponse(); if(messages.size() > 1) { response.addAllErrors(new IllegalStateException("WriterToBulkWriter expects a batch of exactly 1"), tuples); return response; } try { messageWriter.write(sensorType, configurations, Iterables.getFirst(tuples, null), Iterables.getFirst(messages, null)); } catch(Exception e) { response.addAllErrors(e, tuples); return response; } response.addAllSuccesses(tuples); return response; }
@Override public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) { BulkWriterResponse writerResponse = new BulkWriterResponse(); List<Map.Entry<Tuple, Future>> results = new ArrayList<>(); int i = 0; jsonMessage = message.toJSONString(); } catch (Throwable t) { writerResponse.addError(t, tuple); continue; writerResponse.addAllErrors(e, tuples); return writerResponse; try { kv.getValue().get(); writerResponse.addSuccess(kv.getKey()); } catch (Exception e) { writerResponse.addError(e, kv.getKey());
} else if (response.hasErrors()) { throw new IllegalStateException("Unhandled bulk errors in response: " + response.getErrors()); tuplesToAck.removeAll(response.getSuccesses()); response.getErrors().values().forEach(tuplesToAck::removeAll); commit(tuplesToAck);
@Override public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception { if(sleepFunction != null) { sleepFunction.apply(null); } BulkWriterResponse response = new BulkWriterResponse(); response.addAllSuccesses(tuples); return response; }
BulkWriterResponse writerResponse = new BulkWriterResponse(); writerResponse.addAllSuccesses(tuples); writerResponse.addSuccess(goodTuple); writerResponse.addError(new IllegalStateException(), errorTuple); when(batchWriter.write(any(), any(), any(), any())).thenReturn(writerResponse); bolt.execute(errorTuple);
BulkWriterResponse writerResponse = new BulkWriterResponse(); writerResponse.addAllSuccesses(tuples); writerResponse.addSuccess(goodTuple); when(batchWriter.write(any(), any(), any(), any())).thenReturn(writerResponse);
BulkWriterResponse response = new BulkWriterResponse(); for(WriteSuccess<TupleBasedDocument> success: results.getSuccesses()) { response.addSuccess(success.getDocument().getTuple()); response.addError(failure.getCause(), failure.getDocument().getTuple());
@Test public void shouldWriteManySuccessfully() { // create a few tuples and the messages associated with the tuples List<Tuple> tuples = createTuples(3); List<JSONObject> messages = createMessages(3); // create a document writer which will successfully write all BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addSuccess(createDocument(messages.get(0), tuples.get(0))); results.addSuccess(createDocument(messages.get(1), tuples.get(1))); results.addSuccess(createDocument(messages.get(2), tuples.get(2))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should only contain successes assertFalse(response.hasErrors()); assertTrue(response.getSuccesses().contains(tuples.get(0))); assertTrue(response.getSuccesses().contains(tuples.get(1))); assertTrue(response.getSuccesses().contains(tuples.get(2))); }
@Test public void shouldHandleWriteFailure() { // create a tuple and a message associated with that tuple List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); Exception cause = new Exception(); // create a document writer which will fail all writes BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // the writer response should only contain failures assertEquals(0, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); Collection<Tuple> errors = response.getErrors().get(cause); assertTrue(errors.contains(tuples.get(0))); }
public void error(String sensorType, BulkWriterResponse errors, MessageGetStrategy messageGetStrategy) { Map<Throwable, Collection<Tuple>> errorMap = errors.getErrors(); for(Map.Entry<Throwable, Collection<Tuple>> entry : errorMap.entrySet()) { error(sensorType, entry.getKey(), entry.getValue(), messageGetStrategy); } }
public void commit(BulkWriterResponse response) { commit(response.getSuccesses()); }
@Override public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception { messages.forEach(message -> output.add(message.toJSONString().getBytes())); BulkWriterResponse bulkWriterResponse = new BulkWriterResponse(); bulkWriterResponse.addAllSuccesses(tuples); return bulkWriterResponse; }
@Test public void shouldWriteSuccessfullyWhenMessageTimestampIsString() { List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); // the timestamp is a String, rather than a Long messages.get(0).put(Constants.Fields.TIMESTAMP.getName(), new Long(System.currentTimeMillis()).toString()); // create the document JSONObject message = messages.get(0); String timestamp = (String) message.get(Constants.Fields.TIMESTAMP.getName()); String guid = (String) message.get(Constants.GUID); String sensorType = (String) message.get(Constants.SENSOR_TYPE); TupleBasedDocument document = new TupleBasedDocument(message, guid, sensorType, Long.parseLong(timestamp), tuples.get(0)); // create a document writer which will successfully write that document BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addSuccess(document); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should only contain successes assertFalse(response.hasErrors()); assertTrue(response.getSuccesses().contains(tuples.get(0))); }
@Test public void shouldHandlePartialFailures() { // create a few tuples and the messages associated with the tuples int count = 2; List<Tuple> tuples = createTuples(count); List<JSONObject> messages = createMessages(count); Exception cause = new Exception(); // create a document writer that will fail one and succeed the other BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); results.addSuccess(createDocument(messages.get(1), tuples.get(1))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should contain some successes and some failures assertEquals(1, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); assertTrue(response.getErrors().get(cause).contains(tuples.get(0))); assertTrue(response.getSuccesses().contains(tuples.get(1))); }
@Override public BulkWriterResponse write(String sourceType, WriterConfiguration configurations, Iterable<Tuple> tuples, List<JSONObject> messages) throws Exception { String collection = getCollection(sourceType, configurations); BulkWriterResponse bulkResponse = new BulkWriterResponse(); Collection<SolrInputDocument> docs = toDocs(messages); try { Optional<SolrException> exceptionOptional = fromUpdateResponse(solr.add(collection, docs)); // Solr commits the entire batch or throws an exception for it. There's no way to get partial failures. if(exceptionOptional.isPresent()) { bulkResponse.addAllErrors(exceptionOptional.get(), tuples); } else { if (shouldCommit) { exceptionOptional = fromUpdateResponse(solr.commit(collection, waitFlush, waitSearcher, softCommit)); if(exceptionOptional.isPresent()) { bulkResponse.addAllErrors(exceptionOptional.get(), tuples); } } if(!exceptionOptional.isPresent()) { bulkResponse.addAllSuccesses(tuples); } } } catch(HttpSolrClient.RemoteSolrException sse) { bulkResponse.addAllErrors(sse, tuples); } return bulkResponse; }
BulkWriterResponse writerResponse = new BulkWriterResponse(); writerResponse.addAllSuccesses(tuples); when(batchWriter.write(any(), any(), any(), any())).thenReturn(writerResponse);
@Test public void shouldWriteSuccessfully() { // create a tuple and a message associated with that tuple List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); // create a document writer which will successfully write all BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addSuccess(createDocument(messages.get(0), tuples.get(0))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should only contain successes assertFalse(response.hasErrors()); assertTrue(response.getSuccesses().contains(tuples.get(0))); }
@Test public void shouldHandleManyWriteFailures() { // create a few tuples and the messages associated with the tuples int count = 3; List<Tuple> tuples = createTuples(count); List<JSONObject> messages = createMessages(count); Exception cause = new Exception(); // create a document writer which will fail all writes BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); results.addFailure(createDocument(messages.get(1), tuples.get(1)), cause, "error"); results.addFailure(createDocument(messages.get(2), tuples.get(2)), cause, "error"); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // the writer response should only contain failures assertEquals(0, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); Collection<Tuple> errors = response.getErrors().get(cause); assertTrue(errors.contains(tuples.get(0))); assertTrue(errors.contains(tuples.get(1))); assertTrue(errors.contains(tuples.get(2))); }
@Override public BulkWriterResponse write(String sourceType , WriterConfiguration configurations , Iterable<Tuple> tuples , List<JSONObject> messages ) throws Exception { BulkWriterResponse response = new BulkWriterResponse(); // Currently treating all the messages in a group for pass/failure. try { // Messages can all result in different HDFS paths, because of Stellar Expressions, so we'll need to iterate through for(JSONObject message : messages) { String path = getHdfsPathExtension( sourceType, (String)configurations.getSensorConfig(sourceType).getOrDefault(IndexingConfigurations.OUTPUT_PATH_FUNCTION_CONF, ""), message ); SourceHandler handler = getSourceHandler(sourceType, path, configurations); handler.handle(message, sourceType, configurations, syncPolicyCreator); } } catch (Exception e) { response.addAllErrors(e, tuples); } response.addAllSuccesses(tuples); return response; }
tupleList.add(tuple); messageList.add(fullMessageList.get(3)); BulkWriterResponse response = new BulkWriterResponse(); response.addAllSuccesses(tupleList); when(bulkMessageWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(tupleList) , argThat(new MessageListMatcher(messageList)))).thenReturn(response);