@Test(expected = EsHadoopException.class) public void testBulk09_HandlerStillReturnsGarbage() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "garbage"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".garbage", StillGarbageHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); fail("This should fail since the retry handler returned garbage"); }
generator.setInfo(resource, 56) .addRejection("index") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate() );
@Test public void testBulk00_Success() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk00_SuccessFromEmptyResponse() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test(expected = EsHadoopException.class) public void testBulk09_HandlerReturnsGarbage() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "garbage"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".garbage", GarbageHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); fail("This should fail since the retry handler returned garbage"); }
@Test(expected = EsHadoopException.class) public void testBulk07_HandlerThrowsExceptions() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "exception"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".exception", ExceptionThrowingHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); Assert.fail("Should have thrown exception and aborted handling data."); }
@Test public void testBulk02_OneRejectionBasedFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate() );
generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addRejection("index") .addRejection("index") .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addRejection("index") .addRejection("index") .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addRejection("index") .addSuccess("index", 201) .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .generate() );
generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addFailure("index", 401, "conflict", "This data is bogus") .addFailure("index", 401, "conflict", "This data is bogus") .addFailure("index", 401, "conflict", "This data is bogus") .addFailure("index", 401, "conflict", "This data is bogus") .generate(), generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() );
@Test public void testBulk01_ThreeRetries() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() );
generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addFailure("index", 402, "conflict", "This data is bogus") .generate(), generator.setInfo(resource, 56) .addFailure("index", 403, "conflict", "This data is bogus") .generate(), generator.setInfo(resource, 56) .addFailure("index", 404, "conflict", "This data is bogus") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() );
generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate() );
@Test public void testBulk01_OneRetry() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(3, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk03_LogRejectionAndExplicitFailure() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "log"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER+".log."+ DropAndLog.CONF_LOGGER_NAME, this.getClass().getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addRejection("index") // This will be retried still .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(1, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
generator.setInfo(resource, 56) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() );
@Test public void testBulk09_HandlerDropsNewline() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "drop"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".drop", NewlineDroppingHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk02_RejectionAndExplicitFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addRejection("index") // This will be retried still .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(1, bulkResponse.getDocsAborted()); assertEquals("This data is bogus", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk03_LogOneExplicitFailure() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "log"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER+".log."+ DropAndLog.CONF_LOGGER_NAME, this.getClass().getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(1, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk02_OneExplicitFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(1, bulkResponse.getDocsAborted()); assertEquals("This data is bogus", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(4, stats.docsAccepted); }