@Override void beforeStart(BulkIndexer bulkIndexer) { this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", bulkIndexer.indexType.getIndex()), bulkIndexer.result.total, LOGGER) .setPluralLabel("requests"); this.progress.start(); Map<String, Object> temporarySettings = new HashMap<>(); GetSettingsResponse settingsResp = bulkIndexer.client.nativeClient().admin().indices().prepareGetSettings(bulkIndexer.indexType.getIndex()).get(); // deactivate replicas int initialReplicas = Integer.parseInt(settingsResp.getSetting(bulkIndexer.indexType.getIndex(), IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); if (initialReplicas > 0) { initialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas); temporarySettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0); } // deactivate periodical refresh String refreshInterval = settingsResp.getSetting(bulkIndexer.indexType.getIndex(), REFRESH_INTERVAL_SETTING); initialSettings.put(REFRESH_INTERVAL_SETTING, refreshInterval); temporarySettings.put(REFRESH_INTERVAL_SETTING, "-1"); updateSettings(bulkIndexer, temporarySettings); }
public void execute(MultiHandler handler) throws SQLException { checkState(select != null && !updates.isEmpty(), "SELECT or UPDATE(s) requests are not defined"); progress.start(); try { select.scroll(row -> callMultiHandler(handler, updates, row)); closeUpdates(); // log the total number of processed rows progress.log(); } finally { progress.stop(); } }
@Test public void create() { ProgressLogger progress = ProgressLogger.create(getClass(), new AtomicLong()); // default values assertThat(progress.getPeriodMs()).isEqualTo(60000L); assertThat(progress.getPluralLabel()).isEqualTo("rows"); // override values progress.setPeriodMs(10L); progress.setPluralLabel("issues"); assertThat(progress.getPeriodMs()).isEqualTo(10L); assertThat(progress.getPluralLabel()).isEqualTo("issues"); }
@Test(timeout = 5_000L) public void log_at_fixed_intervals() { AtomicLong counter = new AtomicLong(42L); ProgressLogger progress = new ProgressLogger("ProgressLoggerTest", counter, Loggers.get(getClass())); progress.setPeriodMs(1L); progress.start(); while (logTester.logs(LoggerLevel.INFO).size()<2) { Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS); } progress.stop(); assertThat(hasInfoLog("42 rows processed")).isTrue(); // ability to manual log, generally final status counter.incrementAndGet(); progress.log(); assertThat(hasInfoLog("43 rows processed")).isTrue(); }
public MassUpdate rowPluralName(String s) { this.progress.setPluralLabel(s); return this; }
@Override void afterStop(BulkIndexer bulkIndexer) { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ bulkIndexer.client.prepareForceMerge(bulkIndexer.indexType.getIndex()).get(); updateSettings(bulkIndexer, initialSettings); this.progress.stop(); }
public static ProgressLogger create(Class clazz, AtomicLong counter) { String threadName = String.format("ProgressLogger[%s]", clazz.getSimpleName()); Logger logger = Loggers.get(clazz); return new ProgressLogger(threadName, counter, logger); }
public MassUpdate rowPluralName(String s) { this.progress.setPluralLabel(s); return this; }
@Override void afterStop(BulkIndexer bulkIndexer) { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ bulkIndexer.client.prepareForceMerge(bulkIndexer.indexType.getIndex()).get(); updateSettings(bulkIndexer, initialSettings); this.progress.stop(); }
public static ProgressLogger create(Class clazz, AtomicLong counter) { String threadName = String.format("ProgressLogger[%s]", clazz.getSimpleName()); Logger logger = Loggers.get(clazz); return new ProgressLogger(threadName, counter, logger); }
public void execute(Handler handler) throws SQLException { checkState(select != null && !updates.isEmpty(), "SELECT or UPDATE requests are not defined"); checkState(updates.size() == 1, "There should be only one update when using a " + Handler.class.getName()); progress.start(); try { select.scroll(row -> callSingleHandler(handler, updates.iterator().next(), row)); closeUpdates(); // log the total number of processed rows progress.log(); } finally { progress.stop(); } }
@Override void beforeStart(BulkIndexer bulkIndexer) { this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", bulkIndexer.indexType.getIndex()), bulkIndexer.result.total, LOGGER) .setPluralLabel("requests"); this.progress.start(); Map<String, Object> temporarySettings = new HashMap<>(); GetSettingsResponse settingsResp = bulkIndexer.client.nativeClient().admin().indices().prepareGetSettings(bulkIndexer.indexType.getIndex()).get(); // deactivate replicas int initialReplicas = Integer.parseInt(settingsResp.getSetting(bulkIndexer.indexType.getIndex(), IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); if (initialReplicas > 0) { initialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas); temporarySettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0); } // deactivate periodical refresh String refreshInterval = settingsResp.getSetting(bulkIndexer.indexType.getIndex(), REFRESH_INTERVAL_SETTING); initialSettings.put(REFRESH_INTERVAL_SETTING, refreshInterval); temporarySettings.put(REFRESH_INTERVAL_SETTING, "-1"); updateSettings(bulkIndexer, temporarySettings); }
public void execute(MultiHandler handler) throws SQLException { checkState(select != null && !updates.isEmpty(), "SELECT or UPDATE(s) requests are not defined"); progress.start(); try { select.scroll(row -> callMultiHandler(handler, updates, row)); closeUpdates(); // log the total number of processed rows progress.log(); } finally { progress.stop(); } }
public void execute(Handler handler) throws SQLException { checkState(select != null && !updates.isEmpty(), "SELECT or UPDATE requests are not defined"); checkState(updates.size() == 1, "There should be only one update when using a " + Handler.class.getName()); progress.start(); try { select.scroll(row -> callSingleHandler(handler, updates.iterator().next(), row)); closeUpdates(); // log the total number of processed rows progress.log(); } finally { progress.stop(); } }