private void handleBatchRead(final long messsageId, final String clientId, Collection<String> keys) { int maxReadBatch = dataStoreConfig.dbMaxReadBatch(); List<String> keysToFetch = new ArrayList<>(maxReadBatch); for (String key : keys) { keysToFetch.add(key); if (keysToFetch.size() > maxReadBatch) { doHandleBatchRead(messsageId, clientId, new ArrayList<>(keysToFetch)); keysToFetch.clear(); } } if (keysToFetch.size() > 0) { doHandleBatchRead(messsageId, clientId, keysToFetch); } }
@Override public void batchRead(ReadBatchRequest request) { int maxReadBatch = dataStoreConfig.dbMaxReadBatch(); long messageId = request.messageId(); String clientId = request.clientId(); List<String> keysToFetch = new ArrayList<>(maxReadBatch); for (String key : request.keys()) { keysToFetch.add(key); if (keysToFetch.size() > maxReadBatch) { quickBatchRead(messageId, clientId, new ArrayList<>(keysToFetch)); keysToFetch.clear(); } } if (keysToFetch.size() > 0) { quickBatchRead(messageId, clientId, new ArrayList<>(keysToFetch)); } this.readOperationsQueue.offer(request); }
if (batch.size() > dataStoreConfig.dbMaxReadBatch()) {
public void init(final DataStoreConfig config, DataOutputQueue queue, DataStore nextReaderDataStore) { super.init(config, queue, nextReaderDataStore); url = config.dbUrl(); user = config.dbUser(); password = config.dbPassword(); table = config.dbTable(); numReaders = config.dbReaderCount(); numWriters = config.dbWriterCount(); maxReadBatch = config.dbMaxReadBatch(); minReadBatch = config.dbMinReadBatch(); maxWriteBatch = config.dbMaxWriteBatch(); sqlBatchWrite = config.sqlBatchWrite(); flushQueueInterval = config.dbReadFlushQueueIntervalMS(); if (debug) { logger.info(config); } initBlackListSet(config); if (this.keyblackList.size() > 0) { this.hasBlackList = true; } }
@Override public String toString() { return "DataStoreConfig{" + "dbUrl='" + dbUrl() + '\'' + ", dbUser='" + dbUser() + '\'' + ", dbPassword='" + dbPassword.length() + '\'' + ", dbTable='" + dbTable() + '\'' + ", dbMaxReadBatch=" + dbMaxReadBatch() + ", dbMinReadBatch=" + dbMinReadBatch() + ", dbMaxWriteBatch=" + dbMaxWriteBatch() + ", checkStatusEveryIntervalMS=" + checkStatusEveryIntervalMS() + ", threadErrorResumeTimeMS=" + threadErrorResumeTimeMS() + ", pollTimeout=" + pollTimeoutMS() + ", processQueueMaxBatchSize=" + processQueueMaxBatchSize() + ", outputDirectory='" + outputDirectory() + '\'' + ", levelDBFileName='" + levelDBFileName() + ", levelDBCacheSize=" + levelDBCacheSize() + ", levelDBWriteBufferSize=" + levelDBWriteBufferSize() + ", levelDBMaxOpenFiles=" + levelDBMaxOpenFiles() + ", dbReaderCount=" + dbReaderCount() + ", dbWriterCount=" + dbWriterCount() + ", dbReadFlushQueueIntervalMS=" + dbReadFlushQueueIntervalMS() + ", dbWriteFlushQueueIntervalMS=" + dbWriteFlushQueueIntervalMS() + '}'; }
@Override public String toString() { return "DataStoreConfig{" + "dbUrl='" + dbUrl() + '\'' + ", dbUser='" + dbUser() + '\'' + ", dbPassword='" + dbPassword.length() + '\'' + ", dbTable='" + dbTable() + '\'' + ", dbMaxReadBatch=" + dbMaxReadBatch() + ", dbMinReadBatch=" + dbMinReadBatch() + ", dbMaxWriteBatch=" + dbMaxWriteBatch() + ", checkStatusEveryIntervalMS=" + checkStatusEveryIntervalMS() + ", threadErrorResumeTimeMS=" + threadErrorResumeTimeMS() + ", pollTimeout=" + pollTimeoutMS() + ", processQueueMaxBatchSize=" + processQueueMaxBatchSize() + ", outputDirectory='" + outputDirectory() + '\'' + ", levelDBFileName='" + levelDBFileName() + ", levelDBCacheSize=" + levelDBCacheSize() + ", levelDBWriteBufferSize=" + levelDBWriteBufferSize() + ", levelDBMaxOpenFiles=" + levelDBMaxOpenFiles() + ", dbReaderCount=" + dbReaderCount() + ", dbWriterCount=" + dbWriterCount() + ", dbReadFlushQueueIntervalMS=" + dbReadFlushQueueIntervalMS() + ", dbWriteFlushQueueIntervalMS=" + dbWriteFlushQueueIntervalMS() + '}'; }
Action.NONE, "CONFIG dbMaxReadBatch", config.dbMaxReadBatch());