public BulkResponse tryFlush() { Assert.isTrue(writeInitialized, "Cannot flush non-initialized write operation"); return bulkProcessor.tryFlush(); }
/** * Attempts a flush operation, handling failed documents based on configured error listeners. * @throws EsHadoopException in the event that the bulk operation fails, is aborted, or its errors could not be handled. */ public void flush() { BulkResponse bulk = tryFlush(); if (!bulk.getDocumentErrors().isEmpty()) { int maxErrors = 5; String header = String.format("Could not write all entries for bulk operation [%s/%s]. Error " + "sample (first [%s] error messages):\n", bulk.getDocumentErrors().size(), bulk.getTotalDocs(), maxErrors); StringBuilder message = new StringBuilder(header); int i = 0; for (BulkResponse.BulkError errors : bulk.getDocumentErrors()) { if (i >=maxErrors ) { break; } message.append("\t"); appendError(message, errors.getError()); message.append("\n"); message.append("\t") .append(errors.getDocument().toString()) .append("\n"); i++; } message.append("Bailing out..."); throw new EsHadoopException(message.toString()); } }
@Test(expected = EsHadoopException.class) public void testBulk09_HandlerStillReturnsGarbage() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "garbage"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".garbage", StillGarbageHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); fail("This should fail since the retry handler returned garbage"); }
@Test(expected = EsHadoopException.class) public void testBulk09_HandlerReturnsGarbage() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "garbage"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".garbage", GarbageHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); fail("This should fail since the retry handler returned garbage"); }
@Test(expected = EsHadoopException.class) public void testBulk07_HandlerThrowsExceptions() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "exception"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".exception", ExceptionThrowingHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); Assert.fail("Should have thrown exception and aborted handling data."); }
@Test public void testBulk00_Empty() throws Exception { BulkProcessor processor = getBulkProcessor(); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(0, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(0, stats.docsAccepted); }
@Test public void testBulk03_LogOneExplicitFailure() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "log"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER+".log."+ DropAndLog.CONF_LOGGER_NAME, this.getClass().getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(1, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk03_LogRejectionAndExplicitFailure() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "log"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER+".log."+ DropAndLog.CONF_LOGGER_NAME, this.getClass().getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addRejection("index") // This will be retried still .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(1, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk00_SuccessFromEmptyResponse() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk02_OneExplicitFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(1, bulkResponse.getDocsAborted()); assertEquals("This data is bogus", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
BulkResponse bulkResponse = processor.tryFlush();
@Test public void testBulk09_HandlerDropsNewline() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "drop"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".drop", NewlineDroppingHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk02_RejectionAndExplicitFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addRejection("index") // This will be retried still .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(1, bulkResponse.getDocsAborted()); assertEquals("This data is bogus", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk00_Success() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
BulkResponse bulkResponse = processor.tryFlush();
processor.tryFlush();
BulkResponse bulkResponse = processor.tryFlush();
@Test public void testBulk01_OneRetry() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(3, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
BulkResponse bulkResponse = processor.tryFlush();
BulkResponse bulkResponse = processor.tryFlush();