public static BulkResponse complete() { return complete(HttpStatus.OK, 0L, 0, 0, 0); }
public static BulkResponse partial(int httpStatus, long spent, int totalWrites, int docsSent, int docsSkipped, int docsAborted, List<BulkError> errors) { return new BulkResponse(BulkStatus.PARTIAL, httpStatus, spent, totalWrites, docsSent, docsSkipped, docsAborted, errors); }
private BulkProcessor getBulkProcessor(RestClient.BulkActionResponse... responses) { return new BulkProcessor(mockClientResponses(responses), resource, testSettings); }
@Test public void testBulk01_OneRetry() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(3, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk00_Empty() throws Exception { BulkProcessor processor = getBulkProcessor(); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(0, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(0, stats.docsAccepted); }
@Test(expected = EsHadoopException.class) public void testBulk09_HandlerStillReturnsGarbage() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "garbage"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".garbage", StillGarbageHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); fail("This should fail since the retry handler returned garbage"); }
private void processData(BulkProcessor processor) { BytesRef data = new BytesRef(); data.add(renderEntry("A")); processor.add(data); data.reset(); data.add(renderEntry("B")); processor.add(data); data.reset(); data.add(renderEntry("C")); processor.add(data); data.reset(); data.add(renderEntry("D")); processor.add(data); data.reset(); data.add(renderEntry("E")); processor.add(data); data.reset(); }
private void doWriteToIndex(BytesRef payload) { bulkProcessor.add(payload); payload.reset(); }
public BulkResponse tryFlush() { Assert.isTrue(writeInitialized, "Cannot flush non-initialized write operation"); return bulkProcessor.tryFlush(); }
@Override public Stats stats() { Stats copy = new Stats(stats); if (client != null) { // Aggregate stats if it's not already discarded copy.aggregate(client.stats()); } if (bulkProcessor != null) { // Aggregate stats if it's not already discarded copy.aggregate(bulkProcessor.stats()); } return copy; }
private void appendError(StringBuilder message, Throwable exception) { if(exception != null) { message.append(exception); if(exception.getCause() != null) { message.append(';'); appendError(message, exception.getCause()); } } }
public void flush() { Assert.isTrue(writeInitialized, "Cannot flush non-initialized write operation"); bulkProcessor.flush(); }
@Test public void testBulk01_ThreeRetries() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted());
@Test(expected = EsHadoopException.class) public void testBulk09_HandlerReturnsGarbage() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "garbage"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".garbage", GarbageHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); fail("This should fail since the retry handler returned garbage"); }
public static BulkResponse complete(int httpStatus, long spent, int totalWrites, int docsSent, int docsSkipped) { return new BulkResponse(BulkStatus.COMPLETE, httpStatus, spent, totalWrites, docsSent, docsSkipped, 0, Collections.<BulkError>emptyList()); }
@Test public void testBulk00_Success() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test(expected = EsHadoopException.class) public void testBulk07_HandlerThrowsExceptions() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "exception"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".exception", ExceptionThrowingHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); Assert.fail("Should have thrown exception and aborted handling data."); }
@Test public void testBulk00_SuccessFromEmptyResponse() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk02_RejectionAndExplicitFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addRejection("index") // This will be retried still .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(1, bulkResponse.getDocsAborted()); assertEquals("This data is bogus", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk02_OneExplicitFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(1, bulkResponse.getDocsAborted()); assertEquals("This data is bogus", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(4, stats.docsAccepted); }