@Override protected void registerDerivedNodeNameWithLogger(String nodeName) { LogConfigurator.setNodeName(nodeName); }
private void checkBulkProcessorAvailability() { while (!isBulkProcessorAvailable()) { try { if (logger.isDebugEnabled()) { logger.debug("Waiting for bulk queue to empty..."); } Thread.sleep(2000); } catch (InterruptedException e) { logger.warn("checkIndexStatistics interrupted", e); } } }
@Override public void close() { logger.info("Closing river"); // Stop the status thread completely, it will be re-started by #start() if (statusThread != null) { statusThread.interrupt(); statusThread = null; } // Cleanup the other parts (the status thread is gone, and can't do that for us anymore) internalStopRiver(); }
/** * Execute actions to stop this river. * * The status thread will not be touched, and the river can be restarted by setting its status again * to {@link Status#RUNNING}. */ void internalStopRiver() { logger.info("Stopping"); try { if (startupThread != null) { startupThread.interrupt(); startupThread = null; } for (Thread thread : tailerThreads) { thread.interrupt(); thread = null; } tailerThreads.clear(); if (indexerThread != null) { indexerThread.interrupt(); indexerThread = null; } logger.info("Stopped"); } catch (Throwable t) { logger.error("Failed to stop", t); } finally { this.context.setStatus(Status.STOPPED); } }
@Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { if (failure.getClass().equals(ActionRequestValidationException.class)) { if (logger.isTraceEnabled()) { logger.trace("Ignore ActionRequestValidationException : {}", failure); } } else { logger.error("afterBulk - Bulk request failed: {} - {} - {}", executionId, request, failure); MongoDBRiverHelper.setRiverStatus(client, definition.getRiverName(), Status.IMPORT_FAILED); request.requests().clear(); bulkProcessor.close(); river.close(); } }
private List<ServerAddress> getServerAddressForReplica(DBObject item) { String definition = item.get("host").toString(); if (definition.contains("/")) { definition = definition.substring(definition.indexOf("/") + 1); } if (logger.isDebugEnabled()) { logger.debug("getServerAddressForReplica - definition: {}", definition); } List<ServerAddress> servers = new ArrayList<ServerAddress>(); for (String server : definition.split(",")) { servers.add(new ServerAddress(server)); } return servers; }
@Override protected void handleRequest(RestRequest request, RestChannel channel, Client esClient) throws Exception { logger.debug("uri: {}", request.uri()); logger.debug("action: {}", request.param("action")); if (request.path().endsWith("list")) { list(request, channel, esClient); return; } else if (request.path().endsWith("start")) { start(request, channel, esClient); return; } else if (request.path().endsWith("stop")) { stop(request, channel, esClient); return; } else if (request.path().endsWith("delete")) { delete(request, channel, esClient); return; } respondError(request, channel, "action not found: " + request.uri(), RestStatus.OK); }
private void addQueryToStream(final Operation operation, final Timestamp<?> currentTimestamp, final DBObject update, final String collection) throws InterruptedException { if (logger.isTraceEnabled()) { logger.trace("addQueryToStream - operation [{}], currentTimestamp [{}], update [{}]", operation, currentTimestamp, update); } if (collection == null) { for (String name : slurpedDb.getCollectionNames()) { DBCollection slurpedCollection = slurpedDb.getCollection(name); addQueryToStream(operation, currentTimestamp, update, name, slurpedCollection); } } else { DBCollection slurpedCollection = slurpedDb.getCollection(collection); addQueryToStream(operation, currentTimestamp, update, collection, slurpedCollection); } }
private static SocketFactory getSSLSocketFactory() { SocketFactory sslSocketFactory; try { final TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { @Override public X509Certificate[] getAcceptedIssuers() { return null; } @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { } @Override public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { } } }; final SSLContext sslContext = SSLContext.getInstance("SSL"); sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); // Create an ssl socket factory with our all-trusting manager sslSocketFactory = sslContext.getSocketFactory(); return sslSocketFactory; } catch (Exception ex) { logger.warn("Unable to build ssl socket factory without certificate validation, using default instead.", ex); } return SSLSocketFactory.getDefault(); }
@Inject public RestMongoDBRiverAction(Settings settings, Client esClient, RestController controller, @RiverIndexName String riverIndexName) { super(settings, controller, esClient); this.riverIndexName = riverIndexName; String baseUrl = "/" + riverIndexName + "/" + MongoDBRiver.TYPE; logger.trace("RestMongoDBRiverAction - baseUrl: {}", baseUrl); controller.registerHandler(RestRequest.Method.GET, baseUrl + "/{action}", this); controller.registerHandler(RestRequest.Method.POST, baseUrl + "/{river}/{action}", this); }
public static void setRiverStatus(Client client, String riverName, Status status) { logger.info("setRiverStatus called with {} - {}", riverName, status); XContentBuilder xb; try { xb = jsonBuilder().startObject().startObject(MongoDBRiver.TYPE).field(MongoDBRiver.STATUS_FIELD, status).endObject() .endObject(); client.prepareIndex("_river", riverName, MongoDBRiver.STATUS_ID).setSource(xb).get(); } catch (IOException ioEx) { logger.error("setRiverStatus failed for river {}", ioEx, riverName); } }
/** * Adds an index request operation to a bulk request, updating the last * timestamp for a given namespace (ie: host:dbName.collectionName) * * @param bulk */ void setLastTimestamp(final Timestamp<?> time, final BulkProcessor bulkProcessor) { try { if (logger.isTraceEnabled()) { logger.trace("setLastTimestamp [{}] [{}]", definition.getMongoOplogNamespace(), time); } bulkProcessor.add(indexRequest(definition.getRiverIndexName()).type(definition.getRiverName()) .id(definition.getMongoOplogNamespace()).source(source(time))); } catch (IOException e) { logger.error("error updating last timestamp for namespace {}", definition.getMongoOplogNamespace()); } }
private String getCollectionFromNamespace(String namespace) { if (namespace.startsWith(definition.getMongoDb() + '.')) { return namespace.substring(definition.getMongoDb().length() + 1); } logger.error("Cannot get collection from namespace [{}]", namespace); return null; }
private XContentBuilder getGridFSMapping() throws IOException { XContentBuilder mapping = jsonBuilder() .startObject() .startObject(definition.getTypeName()) .startObject("properties") .startObject("content").field("type", "attachment").endObject() .startObject("filename").field("type", "string").endObject() .startObject("contentType").field("type", "string").endObject() .startObject("md5").field("type", "string").endObject() .startObject("length").field("type", "long").endObject() .startObject("chunkSize").field("type", "long").endObject() .endObject() .endObject() .endObject(); logger.info("GridFS Mapping: {}", mapping.string()); return mapping; }
public Indexer(MongoDBRiver river) { super(river); this.river = river; this.definition = river.definition; this.context = river.context; this.esClient = river.esClient; this.scriptService = river.scriptService; logger.debug( "Create bulk processor with parameters - bulk actions: {} - concurrent request: {} - flush interval: {} - bulk size: {}", definition.getBulk().getBulkActions(), definition.getBulk().getConcurrentRequests(), definition.getBulk() .getFlushInterval(), definition.getBulk().getBulkSize()); getBulkProcessor(definition.getIndexName(), definition.getTypeName()); }
@Inject public MongoDBRiver(RiverName riverName, RiverSettings settings, @RiverIndexName String riverIndexName, Client esClient, ScriptService scriptService, MongoClientService mongoClientService) { super(riverName, settings); if (logger.isTraceEnabled()) { logger.trace("Initializing"); } this.esClient = esClient; this.scriptService = scriptService; this.mongoClientService = mongoClientService; this.definition = MongoDBRiverDefinition.parseSettings(riverName.name(), riverIndexName, settings, scriptService); BlockingQueue<QueueEntry> stream = definition.getThrottleSize() == -1 ? new LinkedTransferQueue<QueueEntry>() : new ArrayBlockingQueue<QueueEntry>(definition.getThrottleSize()); this.context = new SharedContext(stream, Status.STOPPED); }
public void deleteBulkRequest(String id, String routing, String parent) { logger.trace("deleteBulkRequest - id: {} - index: {} - type: {} - routing: {} - parent: {}", id, index, type, routing, parent); bulkProcessor.add(deleteRequest(index).type(type).id(id).routing(routing).parent(parent)); deletedDocuments.incrementAndGet(); }
@SuppressWarnings("unchecked") private void flattenOps(DBObject entry) { Object ref = entry.removeField(MongoDBRiver.OPLOG_REF); Object ops = ref == null ? entry.removeField(MongoDBRiver.OPLOG_OPS) : getRefOps(ref); if (ops != null) { try { for (DBObject op : (List<DBObject>) ops) { String operation = (String) op.get(MongoDBRiver.OPLOG_OPERATION); if (operation.equals(MongoDBRiver.OPLOG_COMMAND_OPERATION)) { DBObject object = (DBObject) op.get(MongoDBRiver.OPLOG_OBJECT); if (object.containsField(MongoDBRiver.OPLOG_CREATE_COMMAND)) { continue; } } entry.putAll(op); } } catch (ClassCastException e) { logger.error(e.toString(), e); } } }
private XContentBuilder build(final DBObject data, final String objectId) throws IOException { if (data instanceof GridFSDBFile) { logger.info("Add Attachment: {} to index {} / type {}", objectId, definition.getIndexName(), definition.getTypeName()); return MongoDBHelper.serialize((GridFSDBFile) data); } else { Map<String, Object> mapData = this.createObjectMap(data); return XContentFactory.jsonBuilder().map(mapData); } }