/** * Adds an index request operation to a bulk request, updating the last * timestamp for a given namespace (ie: host:dbName.collectionName) * * @param bulk */ void setLastTimestamp(final Timestamp<?> time, final BulkProcessor bulkProcessor) { try { if (logger.isTraceEnabled()) { logger.trace("setLastTimestamp [{}] [{}]", definition.getMongoOplogNamespace(), time); } bulkProcessor.add(indexRequest(definition.getRiverIndexName()).type(definition.getRiverName()) .id(definition.getMongoOplogNamespace()).source(source(time))); } catch (IOException e) { logger.error("error updating last timestamp for namespace {}", definition.getMongoOplogNamespace()); } }
/** * Get the latest timestamp for a given namespace. */ @SuppressWarnings("unchecked") public static Timestamp<?> getLastTimestamp(Client client, MongoDBRiverDefinition definition) { client.admin().indices().prepareRefresh(definition.getRiverIndexName()).get(); GetResponse lastTimestampResponse = client.prepareGet(definition.getRiverIndexName(), definition.getRiverName(), definition.getMongoOplogNamespace()).get(); if (lastTimestampResponse.isExists()) { Map<String, Object> mongodbState = (Map<String, Object>) lastTimestampResponse.getSourceAsMap().get(TYPE); if (mongodbState != null) { Timestamp<?> lastTimestamp = Timestamp.on(mongodbState); if (lastTimestamp != null) { return lastTimestamp; } } } else { if (definition.getInitialTimestamp() != null) { return definition.getInitialTimestamp(); } } return null; }
public OplogSlurper(MongoDBRiver river, Timestamp<?> timestamp, MongoClient mongoClusterClient, MongoClient mongoShardClient) { super(river); this.river = river; this.timestamp = timestamp; this.definition = river.definition; this.context = river.context; this.esClient = river.esClient; this.mongoClusterClient = mongoClusterClient; this.mongoShardClient = mongoShardClient; this.findKeys = new BasicDBObject(); this.gridfsOplogNamespace = definition.getMongoOplogNamespace() + MongoDBRiver.GRIDFS_FILES_SUFFIX; this.cmdOplogNamespace = definition.getMongoDb() + "." + MongoDBRiver.OPLOG_NAMESPACE_COMMAND; if (definition.getExcludeFields() != null) { for (String key : definition.getExcludeFields()) { findKeys.put(key, 0); } } else if (definition.getIncludeFields() != null) { for (String key : definition.getIncludeFields()) { findKeys.put(key, 1); } } this.oplogDb = mongoShardClient.getDB(MongoDBRiver.MONGODB_LOCAL_DATABASE); this.oplogCollection = oplogDb.getCollection(MongoDBRiver.OPLOG_COLLECTION); this.oplogRefsCollection = oplogDb.getCollection(MongoDBRiver.OPLOG_REFS_COLLECTION); this.slurpedDb = mongoShardClient.getDB(definition.getMongoDb()); }
if (definition.getMongoOplogNamespace().equals(namespace)) { validNamespace = true;