private String addInsertToStream(final Timestamp<?> currentTimestamp, final DBObject data) throws InterruptedException { return addInsertToStream(currentTimestamp, data, definition.getMongoCollection()); }
private String addInsertToStream(final Timestamp<?> currentTimestamp, final DBObject data, final String collection) throws InterruptedException { totalDocuments.incrementAndGet(); addToStream(Operation.INSERT, currentTimestamp, data, collection); if (data == null) { return null; } else { return data.containsField(MongoDBRiver.MONGODB_ID_FIELD) ? data.get(MongoDBRiver.MONGODB_ID_FIELD).toString() : null; } }
try { if (definition.isDisableIndexRefresh()) { updateIndexRefresh(definition.getIndexName(), -1L); logger.trace("Collection {} - count: {}", collection.getName(), safeCount(collection, timestamp.getClass())); .find(getFilterForInitialImport(definition.getMongoCollectionFilter(), lastId)) .sort(new BasicDBObject("_id", 1)); while (cursor.hasNext() && context.getStatus() == Status.RUNNING) { count++; if (cursor.hasNext()) { lastId = addInsertToStream(null, applyFieldFilter(object), collection.getName()); } else { logger.debug("Last entry for initial import of {} - add timestamp: {}", collection.getFullName(), timestamp); lastId = addInsertToStream(timestamp, applyFieldFilter(object), collection.getName()); GridFSDBFile file = grid.findOne(new ObjectId(object.get(MongoDBRiver.MONGODB_ID_FIELD).toString())); if (cursor.hasNext()) { lastId = addInsertToStream(null, file); } else { logger.debug("Last entry for initial import of {} - add timestamp: {}", collection.getFullName(), timestamp); lastId = addInsertToStream(timestamp, file); updateIndexRefresh(definition.getIndexName(), TimeValue.timeValueSeconds(1));
if (!isIndexEmpty()) { if (name.length() < 7 || !name.substring(0, 7).equals("system.")) { DBCollection collection = slurpedDb.getCollection(name); importCollection(collection, timestamp); importCollection(collection, timestamp);
private void processAdminCommandOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) throws InterruptedException { if (logger.isTraceEnabled()) { logger.trace("processAdminCommandOplogEntry - [{}]", entry); } DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT); if (definition.isImportAllCollections()) { if (object.containsField(MongoDBRiver.OPLOG_RENAME_COLLECTION_COMMAND_OPERATION) && object.containsField(MongoDBRiver.OPLOG_TO)) { String to = object.get(MongoDBRiver.OPLOG_TO).toString(); if (to.startsWith(definition.getMongoDb())) { String newCollection = getCollectionFromNamespace(to); DBCollection coll = slurpedDb.getCollection(newCollection); CollectionSlurper importer = new CollectionSlurper(river, mongoClusterClient); importer.importCollection(coll, timestamp); } } } }
CollectionSlurper importer = new CollectionSlurper(MongoDBRiver.this, mongoClusterClient); importer.importInitial(initialImportTimestamp);
private DBObject applyFieldFilter(DBObject object) { if (object instanceof GridFSFile) { GridFSFile file = (GridFSFile) object; DBObject metadata = file.getMetaData(); if (metadata != null) { file.setMetaData(applyFieldFilter(metadata)); } } else { object = MongoDBHelper.applyExcludeFields(object, definition.getExcludeFields()); object = MongoDBHelper.applyIncludeFields(object, definition.getIncludeFields()); } return object; }