/** * Register the MongoDB river */ public void onModule(RiversModule module) { module.registerRiver(MongoDBRiver.TYPE, MongoDBRiverModule.class); }
@Inject public MongoDBRiver(RiverName riverName, RiverSettings settings, @RiverIndexName String riverIndexName, Client esClient, ScriptService scriptService, MongoClientService mongoClientService) { super(riverName, settings); if (logger.isTraceEnabled()) { logger.trace("Initializing"); } this.esClient = esClient; this.scriptService = scriptService; this.mongoClientService = mongoClientService; this.definition = MongoDBRiverDefinition.parseSettings(riverName.name(), riverIndexName, settings, scriptService); BlockingQueue<QueueEntry> stream = definition.getThrottleSize() == -1 ? new LinkedTransferQueue<QueueEntry>() : new ArrayBlockingQueue<QueueEntry>(definition.getThrottleSize()); this.context = new SharedContext(stream, Status.STOPPED); }
@Override public void start() { // http://stackoverflow.com/questions/5270611/read-maven-properties-file-inside-jar-war-file logger.info("{} - {}", DESCRIPTION, MongoDBHelper.getRiverVersion()); Status status = MongoDBRiverHelper.getRiverStatus(esClient, riverName.getName()); if (status == Status.IMPORT_FAILED || status == Status.INITIAL_IMPORT_FAILED || status == Status.SCRIPT_IMPORT_FAILED || status == Status.START_FAILED) { logger.error("Cannot start. Current status is {}", status); return; } if (status == Status.STOPPED) { // Leave the current status of the river alone, but set the context status to 'stopped'. // Enabling the river via REST will trigger the actual start. context.setStatus(Status.STOPPED); logger.info("River is currently disabled and will not be started"); } else { // Mark the current status as "waiting for full start" context.setStatus(Status.START_PENDING); // Request start of the river in the next iteration of the status thread MongoDBRiverHelper.setRiverStatus(esClient, riverName.getName(), Status.RUNNING); logger.info("Startup pending"); } statusThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "mongodb_river_status:" + definition.getIndexName()).newThread( new StatusChecker(this, definition, context)); statusThread.start(); }
Builder builder = new Builder(); builder.riverName(riverName); builder.riverIndexName(riverIndexName); if (settings.settings().containsKey(MongoDBRiver.TYPE)) { Map<String, Object> mongoSettings = (Map<String, Object>) settings.settings().get(MongoDBRiver.TYPE); if (mongoSettings.containsKey(SERVERS_FIELD)) { Object mongoServersSettings = mongoSettings.get(SERVERS_FIELD); builder.mongoServers(mongoServers); Map<String, Object> mongoOptionsSettings = (Map<String, Object>) mongoSettings.get(OPTIONS_FIELD); logger.trace("mongoOptionsSettings: " + mongoOptionsSettings); builder.mongoSecondaryReadPreference(XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(SECONDARY_READ_PREFERENCE_FIELD), false)); builder.connectTimeout(XContentMapValues.nodeIntegerValue(mongoOptionsSettings.get(CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT)); builder.socketTimeout(XContentMapValues.nodeIntegerValue(mongoOptionsSettings.get(SOCKET_TIMEOUT), DEFAULT_SOCKET_TIMEOUT)); builder.dropCollection(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(DROP_COLLECTION_FIELD), false)); String isMongos = XContentMapValues.nodeStringValue(mongoOptionsSettings.get(IS_MONGOS_FIELD), null); if (isMongos != null) { builder.isMongos(Boolean.valueOf(isMongos)); builder.mongoUseSSL(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(SSL_CONNECTION_FIELD), false)); builder.mongoSSLVerifyCertificate(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(SSL_VERIFY_CERT_FIELD), true)); builder.advancedTransformation(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(ADVANCED_TRANSFORMATION_FIELD), false)); builder.skipInitialImport(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(SKIP_INITIAL_IMPORT_FIELD), false)); builder.connectionsPerHost(XContentMapValues.nodeIntegerValue(mongoOptionsSettings.get(CONNECTIONS_PER_HOST), DEFAULT_CONNECTIONS_PER_HOST));
logger.info( "MongoDB options: secondaryreadpreference [{}], drop_collection [{}], include_collection [{}], throttlesize [{}], gridfs [{}], filter [{}], db [{}], collection [{}], script [{}], indexing to [{}]/[{}]", definition.isMongoSecondaryReadPreference(), definition.isDropCollection(), definition.getIncludeCollection(), definition.getThrottleSize(), definition.isMongoGridFS(), definition.getMongoOplogFilter(), definition.getMongoDb(), definition.getMongoCollection(), definition.getScript(), definition.getIndexName(), definition.getTypeName()); for (ServerAddress server : definition.getMongoServers()) { logger.debug("Using MongoDB server(s): host [{}], port [{}]", server.getHost(), server.getPort()); if (!esClient.admin().indices().prepareExists(definition.getIndexName()).get().isExists()) { esClient.admin().indices().prepareCreate(definition.getIndexName()).get(); logger.error("failed to create index [{}], disabling river...", e, definition.getIndexName()); return; if (definition.isMongoGridFS()) { try { if (logger.isDebugEnabled()) { logger.debug("Set explicit attachment mapping."); esClient.admin().indices().preparePutMapping(definition.getIndexName()).setType(definition.getTypeName()) .setSource(getGridFSMapping()).get(); } catch (Exception e) { MongoClient mongoClusterClient = mongoClientService.getMongoClusterClient(definition); MongoConfigProvider configProvider = new MongoConfigProvider(MongoDBRiver.this, mongoClientService); MongoConfig config; while (true) { try {
context.setStatus(Status.STARTING); startupThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "mongodb_river_startup:" + definition.getIndexName()).newThread( startupRunnable); startupThread.start();
Map<String, Object> source = new HashMap<String, Object>(); String riverName = hit.getType(); RiverSettings riverSettings = new RiverSettings(null, hit.getSource()); MongoDBRiverDefinition definition = MongoDBRiverDefinition.parseSettings(riverName, riverIndexName, riverSettings, null); Timestamp<?> ts = MongoDBRiver.getLastTimestamp(esClient, definition); Long lastTimestamp = null; if (ts != null) { lastTimestamp = ts.getTime(); source.put("status", MongoDBRiverHelper.getRiverStatus(esClient, riverName)); source.put("settings", hit.getSource()); source.put("lastTimestamp", lastTimestamp); source.put("indexCount", MongoDBRiver.getIndexCount(esClient, definition)); if (logger.isTraceEnabled()) { logger.trace("source: {}", hit.getSourceAsString());
@Inject public RestMongoDBRiverAction(Settings settings, Client esClient, RestController controller, @RiverIndexName String riverIndexName) { super(settings, controller, esClient); this.riverIndexName = riverIndexName; String baseUrl = "/" + riverIndexName + "/" + MongoDBRiver.TYPE; logger.trace("RestMongoDBRiverAction - baseUrl: {}", baseUrl); controller.registerHandler(RestRequest.Method.GET, baseUrl + "/{action}", this); controller.registerHandler(RestRequest.Method.POST, baseUrl + "/{river}/{action}", this); }
/** * Prepare base REST URL for JIRA river management operations. <code>riverName</code> request parameter is defined * here. * * @return base REST management url ending by <code>/</code> */ protected String baseRestMgmUrl() { return "/" + RiverIndexName.Conf.indexName(settings) + "/{riverName}/_mgm_jr/"; }
/** * Registers the HBaseRiverModule as "hbase" river. * * @param module */ public void onModule(final RiversModule module) { module.registerRiver("hbase", HBaseRiverModule.class); } }
/** * Registers the {@link SolrRiverModule} * @param module the elasticsearch module used to handle rivers */ public void onModule(RiversModule module) { module.registerRiver("solr", SolrRiverModule.class); } }
/** * Registers the {@link CassandraRiverModule} * @param module the elasticsearch module used to handle rivers */ public void onModule(RiversModule module) { module.registerRiver("cassandra", CassandraRiverModule.class); } }
@Override public void processModule(Module module) { if (module instanceof RiversModule) { ((RiversModule) module).registerRiver("email", EmailRiverModule.class); } } }
public void onModule(RiversModule module) { module.registerRiver("mysql", MysqlRiverModule.class); } }
public void onModule(RiversModule module){ module.registerRiver("redis", RedisRiverModule.class); } }
public void onModule(RiversModule module) { module.registerRiver("json", JsonRiverModule.class); } }
public void onModule(RiversModule module) { module.registerRiver("cassandra", CassandraRiverModule.class); } }
public void onModule(RiversModule module) { module.registerRiver("github", GitHubRiverModule.class); } }
public void onModule(RiversModule module) { module.registerRiver("arangodb", ArangoDbRiverModule.class); } }
public void onModule(RiversModule module) { module.registerRiver("jira", JiraRiverModule.class); }