@Override public String toString() { return "DataStoreConfig{" + "dbUrl='" + dbUrl() + '\'' + ", dbUser='" + dbUser() + '\'' + ", dbPassword='" + dbPassword.length() + '\'' + ", dbTable='" + dbTable() + '\'' + ", dbMaxReadBatch=" + dbMaxReadBatch() + ", dbMinReadBatch=" + dbMinReadBatch() + ", dbMaxWriteBatch=" + dbMaxWriteBatch() + ", checkStatusEveryIntervalMS=" + checkStatusEveryIntervalMS() + ", threadErrorResumeTimeMS=" + threadErrorResumeTimeMS() + ", pollTimeout=" + pollTimeoutMS() + ", processQueueMaxBatchSize=" + processQueueMaxBatchSize() + ", outputDirectory='" + outputDirectory() + '\'' + ", levelDBFileName='" + levelDBFileName() + ", levelDBCacheSize=" + levelDBCacheSize() + ", levelDBWriteBufferSize=" + levelDBWriteBufferSize() + ", levelDBMaxOpenFiles=" + levelDBMaxOpenFiles() + ", dbReaderCount=" + dbReaderCount() + ", dbWriterCount=" + dbWriterCount() + ", dbReadFlushQueueIntervalMS=" + dbReadFlushQueueIntervalMS() + ", dbWriteFlushQueueIntervalMS=" + dbWriteFlushQueueIntervalMS() + '}'; }
public static DataStoreConfig config() { return new DataStoreConfig(); }
private static String getLogFilesDirectory() { DataStoreConfig dsc = DataStoreConfig.load(); return dsc.outputDirectory(); }
public void init(final DataStoreConfig config, DataOutputQueue queue, DataStore nextReaderDataStore) { super.init(config, queue, nextReaderDataStore); url = config.dbUrl(); user = config.dbUser(); password = config.dbPassword(); table = config.dbTable(); numReaders = config.dbReaderCount(); numWriters = config.dbWriterCount(); maxReadBatch = config.dbMaxReadBatch(); minReadBatch = config.dbMinReadBatch(); maxWriteBatch = config.dbMaxWriteBatch(); sqlBatchWrite = config.sqlBatchWrite(); flushQueueInterval = config.dbReadFlushQueueIntervalMS(); if (debug) { logger.info(config); } initBlackListSet(config); if (this.keyblackList.size() > 0) { this.hasBlackList = true; } }
Action.NONE, "CONFIG dbReaderCount", config.dbReaderCount()); Action.NONE, "CONFIG dbWriterCount", config.dbWriterCount()); Action.NONE, "CONFIG levelDBBatchReaderCount", config.levelDBBatchReaderCount()); Action.NONE, "CONFIG checkStatusEveryIntervalMS", config.checkStatusEveryIntervalMS()); Action.NONE, "CONFIG dbMaxReadBatch", config.dbMaxReadBatch()); Action.NONE, "CONFIG dbMaxWriteBatch", config.dbMaxWriteBatch()); Action.NONE, "CONFIG dbMinReadBatch", config.dbMinReadBatch()); Action.NONE,
public void init(final DataStoreConfig config, DataOutputQueue queue, final DataStore nextReaderDataStore ) { super.init(config, queue, nextReaderDataStore); this.executorService = SimpleExecutors.threadPool(config.levelDBBatchReaderCount(), "LevelDB BatchReader"); File file = new File(config.outputDirectory()); file.mkdirs(); file = new File(file, config.levelDBFileName()); Options options = new Options(); options.cacheSize(config.levelDBCacheSize()); options.writeBufferSize(config.levelDBWriteBufferSize()); options.maxOpenFiles(config.levelDBMaxOpenFiles()); try { store = new SimpleStringKeyValueStoreLevelDB(file.toString(), options, false); } catch (Exception ex) { store = StringKeyValueStoreNoOp.SINGLETON; if (debug) logger.error(ex, "UNABLE TO CONNECT TO LEVEL DB", ex.getMessage()); logger.debug(ex, "UNABLE TO CONNECT TO LEVEL DB", ex.getMessage()); logger.error("UNABLE TO CONNECT TO LEVEL DB", ex.getMessage()); } }
poll = loadQueue.poll(dataStoreConfig.pollTimeoutMS(), TimeUnit.MILLISECONDS); if (batch.size() > dataStoreConfig.dbMaxReadBatch()) {
public void init(DataStoreConfig config) { this.config = config; this.outputDir = IO.path(config.outputDirectory()); }
private void handleBatchRead(final long messsageId, final String clientId, Collection<String> keys) { int maxReadBatch = dataStoreConfig.dbMaxReadBatch(); List<String> keysToFetch = new ArrayList<>(maxReadBatch); for (String key : keys) { keysToFetch.add(key); if (keysToFetch.size() > maxReadBatch) { doHandleBatchRead(messsageId, clientId, new ArrayList<>(keysToFetch)); keysToFetch.clear(); } } if (keysToFetch.size() > 0) { doHandleBatchRead(messsageId, clientId, keysToFetch); } }
String sDir = masterDataStore.config().outputDirectory(); File dir = new File(sDir); if (dir.exists()) {
@Override public void batchRead(ReadBatchRequest request) { int maxReadBatch = dataStoreConfig.dbMaxReadBatch(); long messageId = request.messageId(); String clientId = request.clientId(); List<String> keysToFetch = new ArrayList<>(maxReadBatch); for (String key : request.keys()) { keysToFetch.add(key); if (keysToFetch.size() > maxReadBatch) { quickBatchRead(messageId, clientId, new ArrayList<>(keysToFetch)); keysToFetch.clear(); } } if (keysToFetch.size() > 0) { quickBatchRead(messageId, clientId, new ArrayList<>(keysToFetch)); } this.readOperationsQueue.offer(request); }
@Override public String toString() { return "DataStoreConfig{" + "dbUrl='" + dbUrl() + '\'' + ", dbUser='" + dbUser() + '\'' + ", dbPassword='" + dbPassword.length() + '\'' + ", dbTable='" + dbTable() + '\'' + ", dbMaxReadBatch=" + dbMaxReadBatch() + ", dbMinReadBatch=" + dbMinReadBatch() + ", dbMaxWriteBatch=" + dbMaxWriteBatch() + ", checkStatusEveryIntervalMS=" + checkStatusEveryIntervalMS() + ", threadErrorResumeTimeMS=" + threadErrorResumeTimeMS() + ", pollTimeout=" + pollTimeoutMS() + ", processQueueMaxBatchSize=" + processQueueMaxBatchSize() + ", outputDirectory='" + outputDirectory() + '\'' + ", levelDBFileName='" + levelDBFileName() + ", levelDBCacheSize=" + levelDBCacheSize() + ", levelDBWriteBufferSize=" + levelDBWriteBufferSize() + ", levelDBMaxOpenFiles=" + levelDBMaxOpenFiles() + ", dbReaderCount=" + dbReaderCount() + ", dbWriterCount=" + dbWriterCount() + ", dbReadFlushQueueIntervalMS=" + dbReadFlushQueueIntervalMS() + ", dbWriteFlushQueueIntervalMS=" + dbWriteFlushQueueIntervalMS() + '}'; }
public static DataStoreConfig config() { return new DataStoreConfig(); }
public static DataStoreConfig load() { try { if (IO.exists(FILE_LOCATION)) { return new JsonParserFactory().create().parseFile(DataStoreConfig.class, FILE_LOCATION); } else { return new DataStoreConfig(); } } catch (Exception ex) { ex.printStackTrace(); Exceptions.handle(ex, "Unable to read file from ", FILE_LOCATION); return null; } }