public MongoDBRiverDefinition build() { return new MongoDBRiverDefinition(this); } }
static String removePrefix(String prefix, String object) { return addRemovePrefix(prefix, object, false); }
public Indexer(MongoDBRiver river) { super(river); this.river = river; this.definition = river.definition; this.context = river.context; this.esClient = river.esClient; this.scriptService = river.scriptService; logger.debug( "Create bulk processor with parameters - bulk actions: {} - concurrent request: {} - flush interval: {} - bulk size: {}", definition.getBulk().getBulkActions(), definition.getBulk().getConcurrentRequests(), definition.getBulk() .getFlushInterval(), definition.getBulk().getBulkSize()); getBulkProcessor(definition.getIndexName(), definition.getTypeName()); }
List<ServerAddress> servers = shardServers != null ? shardServers : definition.getMongoServers(); if (!Strings.isNullOrEmpty(definition.getMongoLocalUser()) && !Strings.isNullOrEmpty(definition.getMongoLocalPassword())) { mongoCredentials.add(MongoCredential.createCredential( definition.getMongoLocalUser(), !Strings.isNullOrEmpty(definition.getMongoLocalAuthDatabase()) ? definition.getMongoLocalAuthDatabase() : MongoDBRiver.MONGODB_LOCAL_DATABASE, definition.getMongoLocalPassword().toCharArray())); if (!Strings.isNullOrEmpty(definition.getMongoAdminUser()) && !Strings.isNullOrEmpty(definition.getMongoAdminPassword())) { mongoCredentials.add(MongoCredential.createCredential( definition.getMongoAdminUser(), !Strings.isNullOrEmpty(definition.getMongoAdminAuthDatabase()) ? definition.getMongoAdminAuthDatabase() : MongoDBRiver.MONGODB_ADMIN_DATABASE, definition.getMongoAdminPassword().toCharArray())); MongoClientOptions mongoClientOptions = definition.getMongoClientOptions(); ClientCacheKey cacheKey = new ClientCacheKey(servers, mongoCredentials, mongoClientOptions);
private void isRiverStale(DBCursor cursor, Timestamp<?> time) throws SlurperException { if (cursor == null || time == null) { return; } if (definition.getInitialTimestamp() != null && time.equals(definition.getInitialTimestamp())) { return; } DBObject entry = cursor.next(); Timestamp<?> oplogTimestamp = Timestamp.on(entry); if (!time.equals(oplogTimestamp)) { MongoDBRiverHelper.setRiverStatus(esClient, definition.getRiverName(), Status.RIVER_STALE); throw new SlurperException("River out of sync with oplog.rs collection"); } }
logger.info( "MongoDB options: secondaryreadpreference [{}], drop_collection [{}], include_collection [{}], throttlesize [{}], gridfs [{}], filter [{}], db [{}], collection [{}], script [{}], indexing to [{}]/[{}]", definition.isMongoSecondaryReadPreference(), definition.isDropCollection(), definition.getIncludeCollection(), definition.getThrottleSize(), definition.isMongoGridFS(), definition.getMongoOplogFilter(), definition.getMongoDb(), definition.getMongoCollection(), definition.getScript(), definition.getIndexName(), definition.getTypeName()); for (ServerAddress server : definition.getMongoServers()) { logger.debug("Using MongoDB server(s): host [{}], port [{}]", server.getHost(), server.getPort()); if (!esClient.admin().indices().prepareExists(definition.getIndexName()).get().isExists()) { esClient.admin().indices().prepareCreate(definition.getIndexName()).get(); logger.error("failed to create index [{}], disabling river...", e, definition.getIndexName()); return; if (definition.isMongoGridFS()) { try { if (logger.isDebugEnabled()) { logger.debug("Set explicit attachment mapping."); esClient.admin().indices().preparePutMapping(definition.getIndexName()).setType(definition.getTypeName()) .setSource(getGridFSMapping()).get(); } catch (Exception e) { if (definition.getInitialTimestamp() != null) { startTimestamp = definition.getInitialTimestamp(); } else if (getLastProcessedTimestamp() != null) { startTimestamp = getLastProcessedTimestamp();
if (definition.isImportAllCollections()) { type = entry.getCollection(); } else { type = definition.getTypeName(); updateBulkRequest(entry.getData(), null, operation, definition.getIndexName(), type, null, null); } catch (IOException ioEx) { logger.error("Update bulk failed.", ioEx); updateBulkRequest(entry.getData(), objectId, operation, definition.getIndexName(), type, null, null); } catch (IOException ioEx) { logger.error("Update bulk failed.", ioEx); if (hasScript() && definition.isAdvancedTransformation()) { return applyAdvancedTransformation(entry, type); if (!definition.getIncludeCollection().isEmpty()) { logger.trace("About to include collection. set attribute {} / {} ", definition.getIncludeCollection(), definition.getMongoCollection()); entry.getData().put(definition.getIncludeCollection(), definition.getMongoCollection()); logger.trace("Script to be executed: {} - {}", definition.getScriptType(), definition.getScript()); logger.trace("Context before script executed: {}", ctx); ExecutableScript executableScript = scriptService.executable(definition.getScriptType(), definition.getScript(), ScriptService.ScriptType.INLINE, ImmutableMap.<String, Object>of("logger", logger)); executableScript.setNextVar("ctx", ctx); } catch (Exception e) {
if (!definition.getIncludeCollection().isEmpty()) { logger.trace("About to include collection. set attribute {} / {} ", definition.getIncludeCollection(), definition.getMongoCollection()); entry.getData().put(definition.getIncludeCollection(), definition.getMongoCollection()); document.put("id", objectId); document.put("_index", definition.getIndexName()); document.put("_type", type); document.put("operation", operation.getValue()); ExecutableScript executableScript = scriptService.executable(definition.getScriptType(), definition.getScript(), ScriptService.ScriptType.INLINE, ImmutableMap.<String, Object>of("logger", logger)); if (logger.isTraceEnabled()) { logger.trace("Script to be executed: {} - {}", definition.getScriptType(), definition.getScript()); logger.trace("Context before script executed: {}", ctx); } catch (Exception e) { logger.error("failed to script process {}, ignoring", e, ctx); MongoDBRiverHelper.setRiverStatus(esClient, definition.getRiverName(), Status.SCRIPT_IMPORT_FAILED);
DBCursor cursor = null; try { if (definition.isDisableIndexRefresh()) { updateIndexRefresh(definition.getIndexName(), -1L); if (!definition.isMongoGridFS()) { if (logger.isTraceEnabled()) { .find(getFilterForInitialImport(definition.getMongoCollectionFilter(), lastId)) .sort(new BasicDBObject("_id", 1)); while (cursor.hasNext() && context.getStatus() == Status.RUNNING) { GridFS grid = new GridFS(mongoClient.getDB(definition.getMongoDb()), definition.getMongoCollection()); cursor.close(); if (definition.isDisableIndexRefresh()) { updateIndexRefresh(definition.getIndexName(), TimeValue.timeValueSeconds(1));
public OplogSlurper(MongoDBRiver river, Timestamp<?> timestamp, MongoClient mongoClusterClient, MongoClient mongoShardClient) { super(river); this.river = river; this.timestamp = timestamp; this.definition = river.definition; this.context = river.context; this.esClient = river.esClient; this.mongoClusterClient = mongoClusterClient; this.mongoShardClient = mongoShardClient; this.findKeys = new BasicDBObject(); this.gridfsOplogNamespace = definition.getMongoOplogNamespace() + MongoDBRiver.GRIDFS_FILES_SUFFIX; this.cmdOplogNamespace = definition.getMongoDb() + "." + MongoDBRiver.OPLOG_NAMESPACE_COMMAND; if (definition.getExcludeFields() != null) { for (String key : definition.getExcludeFields()) { findKeys.put(key, 0); } } else if (definition.getIncludeFields() != null) { for (String key : definition.getIncludeFields()) { findKeys.put(key, 1); } } this.oplogDb = mongoShardClient.getDB(MongoDBRiver.MONGODB_LOCAL_DATABASE); this.oplogCollection = oplogDb.getCollection(MongoDBRiver.OPLOG_COLLECTION); this.oplogRefsCollection = oplogDb.getCollection(MongoDBRiver.OPLOG_REFS_COLLECTION); this.slurpedDb = mongoShardClient.getDB(definition.getMongoDb()); }
/** * Get the latest timestamp for a given namespace. */ @SuppressWarnings("unchecked") public static Timestamp<?> getLastTimestamp(Client client, MongoDBRiverDefinition definition) { client.admin().indices().prepareRefresh(definition.getRiverIndexName()).get(); GetResponse lastTimestampResponse = client.prepareGet(definition.getRiverIndexName(), definition.getRiverName(), definition.getMongoOplogNamespace()).get(); if (lastTimestampResponse.isExists()) { Map<String, Object> mongodbState = (Map<String, Object>) lastTimestampResponse.getSourceAsMap().get(TYPE); if (mongodbState != null) { Timestamp<?> lastTimestamp = Timestamp.on(mongodbState); if (lastTimestamp != null) { return lastTimestamp; } } } else { if (definition.getInitialTimestamp() != null) { return definition.getInitialTimestamp(); } } return null; }
public static long getIndexCount(Client client, MongoDBRiverDefinition definition) { if (client.admin().indices().prepareExists(definition.getIndexName()).get().isExists()) { if (definition.isImportAllCollections()) { return client.prepareCount(definition.getIndexName()).execute().actionGet().getCount(); } else { if (client.admin().indices().prepareTypesExists(definition.getIndexName()).setTypes(definition.getTypeName()).get() .isExists()) { return client.prepareCount(definition.getIndexName()).setTypes(definition.getTypeName()).get().getCount(); } } } return 0; }
mongoClientOptionsBuilder.socketFactory(getSSLSocketFactory()); if (mongoSettings.containsKey(FILTER_FIELD)) { String filter = XContentMapValues.nodeStringValue(mongoSettings.get(FILTER_FIELD), ""); filter = removePrefix("o.", filter); builder.mongoCollectionFilter(convertToBasicDBObject(filter)); builder.mongoOplogFilter(convertToBasicDBObject(removePrefix("o.", filter)));
private DBObject applyFieldFilter(DBObject object) { if (object instanceof GridFSFile) { GridFSFile file = (GridFSFile) object; DBObject metadata = file.getMetaData(); if (metadata != null) { file.setMetaData(applyFieldFilter(metadata)); } } else { object = MongoDBHelper.applyExcludeFields(object, definition.getExcludeFields()); object = MongoDBHelper.applyIncludeFields(object, definition.getIncludeFields()); } return object; }
private XContentBuilder build(final DBObject data, final String objectId) throws IOException { if (data instanceof GridFSDBFile) { logger.info("Add Attachment: {} to index {} / type {}", objectId, definition.getIndexName(), definition.getTypeName()); return MongoDBHelper.serialize((GridFSDBFile) data); } else { Map<String, Object> mapData = this.createObjectMap(data); return XContentFactory.jsonBuilder().map(mapData); } }
private String extractIndex(Map<String, Object> ctx) { String index = (String) ctx.get("_index"); if (index == null) { index = definition.getIndexName(); } return index; }
MongoDBRiverBulkProcessor(MongoDBRiver river, MongoDBRiverDefinition definition, Client client, String index, String type) { super(river); this.river = river; this.bulkProcessor = BulkProcessor.builder(client, listener).setBulkActions(definition.getBulk().getBulkActions()) .setConcurrentRequests(definition.getBulk().getConcurrentRequests()) .setFlushInterval(definition.getBulk().getFlushInterval()).setBulkSize(definition.getBulk().getBulkSize()).build(); this.definition = definition; this.client = client; this.index = index; this.type = type; this.bulkQueueSize = getBulkQueueSize(); }
private DBObject applyFieldFilter(DBObject object) { if (object instanceof GridFSFile) { GridFSFile file = (GridFSFile) object; DBObject metadata = file.getMetaData(); if (metadata != null) { file.setMetaData(applyFieldFilter(metadata)); } } else { object = MongoDBHelper.applyExcludeFields(object, definition.getExcludeFields()); object = MongoDBHelper.applyIncludeFields(object, definition.getIncludeFields()); } return object; }
startupThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "mongodb_river_startup:" + definition.getIndexName()).newThread( startupRunnable); startupThread.start();
@Override public void run() { while (context.getStatus() == Status.RUNNING) { try { Timestamp<?> lastTimestamp = null; // 1. Attempt to fill as much of the bulk request as possible QueueEntry entry = context.getStream().take(); lastTimestamp = processBlockingQueue(entry); while ((entry = context.getStream().poll(definition.getBulk().getFlushInterval().millis(), MILLISECONDS)) != null) { lastTimestamp = processBlockingQueue(entry); } // 2. Update the timestamp if (lastTimestamp != null) { river.setLastTimestamp(lastTimestamp, getBulkProcessor(definition.getIndexName(), definition.getTypeName()).getBulkProcessor()); } } catch (InterruptedException e) { logger.info("river-mongodb indexer interrupted"); releaseProcessors(); Thread.currentThread().interrupt(); break; } } }