@Override public void close() { if (log.isDebugEnabled()) { log.debug("Closing repository and connection to Elasticsearch ..."); } // bail out if closed before if (client == null) { return; } try { if (bulkProcessor != null) { bulkProcessor.close(); // Aggregate stats before discarding them. stats.aggregate(bulkProcessor.stats()); bulkProcessor = null; } if (bulkEntryWriter != null) { bulkEntryWriter.close(); bulkEntryWriter = null; } } finally { client.close(); // Aggregate stats before discarding them. stats.aggregate(client.stats()); client = null; } }
@Test public void testBulk00_Empty() throws Exception { BulkProcessor processor = getBulkProcessor(); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(0, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(0, stats.docsAccepted); }
@Test public void testBulk03_LogOneExplicitFailure() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "log"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER+".log."+ DropAndLog.CONF_LOGGER_NAME, this.getClass().getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(1, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk03_LogRejectionAndExplicitFailure() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "log"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER+".log."+ DropAndLog.CONF_LOGGER_NAME, this.getClass().getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addRejection("index") // This will be retried still .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(1, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk00_SuccessFromEmptyResponse() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk02_OneExplicitFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(1, bulkResponse.getDocsAborted()); assertEquals("This data is bogus", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
@Test public void testBulk09_HandlerDropsNewline() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "drop"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".drop", NewlineDroppingHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
assertEquals("Abort the handler!!", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats();
@Test public void testBulk00_Success() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
@Test public void testBulk02_RejectionAndExplicitFailure() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addFailure("index", 401, "conflict", "This data is bogus") .addSuccess("index", 201) .addRejection("index") // This will be retried still .addSuccess("index", 201) .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(4, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(1, bulkResponse.getDocsAborted()); assertEquals("This data is bogus", bulkResponse.getDocumentErrors().get(0).getError().getMessage()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(1, stats.docsRetried); assertEquals(4, stats.docsAccepted); }
assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats();
assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats();
@Test public void testBulk01_OneRetry() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addSuccess("index", 201) .addRejection("index") .addSuccess("index", 201) .addRejection("index") .addRejection("index") .generate(), generator.setInfo(resource, 56) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(1, stats.bulkRetries); assertEquals(3, stats.docsRetried); assertEquals(5, stats.docsAccepted); }
assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats();
assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats();
assertEquals(1, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats();
assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats();
@Override public void close() { if (log.isDebugEnabled()) { log.debug("Closing repository and connection to Elasticsearch ..."); } // bail out if closed before if (client == null) { return; } try { if (bulkProcessor != null) { bulkProcessor.close(); // Aggregate stats before discarding them. stats.aggregate(bulkProcessor.stats()); bulkProcessor = null; } if (bulkEntryWriter != null) { bulkEntryWriter.close(); bulkEntryWriter = null; } } finally { client.close(); // Aggregate stats before discarding them. stats.aggregate(client.stats()); client = null; } }
@Override public void close() { if (log.isDebugEnabled()) { log.debug("Closing repository and connection to Elasticsearch ..."); } // bail out if closed before if (client == null) { return; } try { if (bulkProcessor != null) { bulkProcessor.close(); // Aggregate stats before discarding them. stats.aggregate(bulkProcessor.stats()); bulkProcessor = null; } if (bulkEntryWriter != null) { bulkEntryWriter.close(); bulkEntryWriter = null; } } finally { client.close(); // Aggregate stats before discarding them. stats.aggregate(client.stats()); client = null; } }
@Override public void close() { if (log.isDebugEnabled()) { log.debug("Closing repository and connection to Elasticsearch ..."); } // bail out if closed before if (client == null) { return; } try { if (bulkProcessor != null) { bulkProcessor.close(); // Aggregate stats before discarding them. stats.aggregate(bulkProcessor.stats()); bulkProcessor = null; } if (bulkEntryWriter != null) { bulkEntryWriter.close(); bulkEntryWriter = null; } } finally { client.close(); // Aggregate stats before discarding them. stats.aggregate(client.stats()); client = null; } }