protected void initSearchAlias(ElasticSearchIndexConfig conf) { // init the search alias String searchAlias = conf.getName(); String searchIndex = getClient().getFirstIndexForAlias(searchAlias); String writeAlias = conf.writeIndexOrAlias(); String writeIndex = getClient().getFirstIndexForAlias(writeAlias); if (searchIndex == null) { if (getClient().indexExists(searchAlias)) { if (Framework.isTestModeSet()) { // in test mode we drop an index that have the target alias name getClient().deleteIndex(searchAlias, TIMEOUT_DELETE_SECOND); } searchIndex = searchAlias; } else { // search alias is not created, point to the write index getClient().updateAlias(searchAlias, writeIndex); searchIndex = writeIndex; } } log.info(String.format("Managed index aliases: Alias: %s -> index: %s, alias: %s -> index: %s", searchAlias, searchIndex, writeAlias, writeIndex)); }
protected void clearScroll(String scrollId) { if (log.isDebugEnabled()) { log.debug(String.format( "Clear scroll : curl -XDELETE 'http://localhost:9200/_search/scroll' -d '{\"scroll_id\" : [\"%s\"]}'", scrollId)); } ClearScrollRequest request = new ClearScrollRequest(); request.addScrollId(scrollId); esa.getClient().clearScroll(request); }
@Override public void onApplicationStopped() { if (esClient == null) { return; } try { esClient.close(); cursorService.clear(); } catch (Exception e) { log.warn("Fail to close esClient: " + e.getMessage(), e); } finally { esClient = null; cursorService = null; } }
protected void initWriteAlias(ElasticSearchIndexConfig conf, boolean dropIfExists) { // init the write index and alias String writeAlias = conf.writeIndexOrAlias(); String writeIndex = getClient().getFirstIndexForAlias(writeAlias); String nextWriteIndex = conf.newWriteIndexForAlias(conf.getName(), writeIndex); if (writeIndex != null && !dropIfExists) { // alias exists make sure the index is well configured initIndex(writeIndex, conf, false); } else { // create a new write index and update the alias, we don't drop anything if (getClient().indexExists(nextWriteIndex)) { throw new IllegalStateException( String.format("New index name %s for the alias %s already exists", nextWriteIndex, writeAlias)); } initIndex(nextWriteIndex, conf, false); getClient().updateAlias(writeAlias, nextWriteIndex); } }
conf.getType())); boolean mappingExists = false; boolean indexExists = getClient().indexExists(indexName); if (indexExists) { if (!dropIfExists) { log.debug("Index " + indexName + " already exists"); mappingExists = getClient().mappingExists(indexName, conf.getType()); if (conf.isDocumentIndex()) { String realIndexForAlias = getClient().getFirstIndexForAlias(conf.getName()); if (realIndexForAlias != null) { repoNames.put(realIndexForAlias, conf.getRepositoryName()); indexName, conf.getType())); getClient().deleteIndex(indexName, TIMEOUT_DELETE_SECOND); indexExists = false; log.debug("Using settings: " + conf.getSettings()); getClient().createIndex(indexName, conf.getSettings()); log.debug("Using mapping: " + conf.getMapping()); getClient().createMapping(indexName, conf.getType(), conf.getMapping()); if (!dropIfExists && conf.getRepositoryName() != null) { repositoryInitialized.add(conf.getRepositoryName());
/** * Ensures the audit sequence returns an UID greater or equal than the maximum log entry id. */ protected void ensureUIDSequencer(ESClient esClient) { boolean auditIndexExists = esClient.indexExists(getESIndexName()); if (!auditIndexExists) { return; } // Get max log entry id SearchRequest request = createSearchRequest(); request.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) .aggregation(AggregationBuilders.max("maxAgg").field("id"))); SearchResponse searchResponse = esClient.search(request); Max agg = searchResponse.getAggregations().get("maxAgg"); long maxLogEntryId = (long) agg.getValue(); // Get next sequence id UIDGeneratorService uidGeneratorService = Framework.getService(UIDGeneratorService.class); UIDSequencer seq = uidGeneratorService.getSequencer(); seq.init(); long nextSequenceId = seq.getNextLong(SEQ_NAME); // Increment sequence to max log entry id if needed if (nextSequenceId < maxLogEntryId) { log.info(String.format("Next UID returned by %s sequence is %d, initializing sequence to %d", SEQ_NAME, nextSequenceId, maxLogEntryId)); seq.initSequence(SEQ_NAME, maxLogEntryId); } }
SearchResponse runRequest(SearchRequest request) { logSearchRequest(request); SearchResponse response = esClient.search(request); logSearchResponse(response); return response; }
@Override public void init() { if (esClient != null) { return; } ElasticSearchAdmin esa = Framework.getService(ElasticSearchAdmin.class); esClient = esa.getClient(); indexName = esa.getIndexNameForType(ElasticSearchConstants.SEQ_ID_TYPE); try { boolean indexExists = esClient.indexExists(indexName); if (!indexExists) { throw new NuxeoException( String.format("Sequencer %s needs an elasticSearchIndex contribution with type %s", getName(), ElasticSearchConstants.SEQ_ID_TYPE)); } } catch (NoSuchElementException | NuxeoException e) { dispose(); throw e; } }
SearchResponse runNextScroll(String scrollId, TimeValue keepAlive) { if (log.isDebugEnabled()) { log.debug(String.format( "Scroll request: -XGET 'localhost:9200/_search/scroll' -d '{\"scroll\": \"%s\", \"scroll_id\": \"%s\" }'", keepAlive, scrollId)); } SearchResponse response = esClient.searchScroll(new SearchScrollRequest(scrollId).scroll(keepAlive)); logSearchResponse(response); return response; }
@Override public long getNextLong(String sequenceName) { String source = "{ \"ts\" : " + System.currentTimeMillis() + "}"; IndexResponse res = esClient.index( new IndexRequest(indexName, ElasticSearchConstants.SEQ_ID_TYPE, sequenceName).source(source, XContentType.JSON)); return res.getVersion(); }
void sendBulkCommand(BulkRequest bulkRequest, int bulkSize) { if (bulkRequest.numberOfActions() > 0) { if (log.isDebugEnabled()) { logDebugMessageTruncated(String.format( "Index %d docs (%d bytes) in bulk request: curl -XPOST 'http://localhost:9200/_bulk' -d '%s'", bulkRequest.numberOfActions(), bulkSize, bulkRequest.requests().toString()), MAX_CURL_LINE); } BulkResponse response = esa.getClient().bulk(bulkRequest); if (response.hasFailures()) { logBulkFailure(response); } } }
@Override public LogEntry getLogEntryByID(long id) { GetResponse ret = esClient.get( new GetRequest(getESIndexName(), ElasticSearchConstants.ENTRY_TYPE, String.valueOf(id))); if (!ret.isExists()) { return null; } try { return new ObjectMapper().readValue(ret.getSourceAsString(), LogEntryImpl.class); } catch (IOException e) { throw new NuxeoException("Unable to read Entry for id " + id, e); } }
@Override public void refreshRepositoryIndex(String repositoryName) { if (log.isDebugEnabled()) { log.debug("Refreshing index associated with repo: " + repositoryName); } getClient().refresh(getWriteIndexName(getIndexNameForRepository(repositoryName))); if (log.isDebugEnabled()) { log.debug("Refreshing index done"); } }
protected SearchResponse search(NxQueryBuilder query) { try (Context ignored = searchTimer.time()) { SearchType searchType = SearchType.DFS_QUERY_THEN_FETCH; SearchRequest request = buildEsSearchRequest(query, searchType); logSearchRequest(request, query, searchType); SearchResponse response = esa.getClient().search(request); logSearchResponse(response); return response; } }
SearchResponse runNextScroll(SearchResponse response, TimeValue keepAlive) { if (log.isDebugEnabled()) { log.debug(String.format( "Scroll request: -XGET 'localhost:9200/_search/scroll' -d '{\"scroll\": \"%s\", \"scroll_id\": \"%s\" }'", keepAlive, response.getScrollId())); } SearchScrollRequest request = new SearchScrollRequest(response.getScrollId()).scroll(keepAlive); return esa.getClient().searchScroll(request); }
@Override public void initSequence(String key, long id) { String source = "{ \"ts\" : " + System.currentTimeMillis() + "}"; IndexResponse res = esClient.index( new IndexRequest(indexName, ElasticSearchConstants.SEQ_ID_TYPE, key).versionType(VersionType.EXTERNAL) .version(id) .source(source, XContentType.JSON)); }
@Override public void append(List<String> jsonEntries) { BulkRequest bulkRequest = new BulkRequest(); for (String json : jsonEntries) { try { String entryId = new JSONObject(json).getString(LOG_ID); if (StringUtils.isBlank(entryId)) { throw new NuxeoException("A json entry has an empty id. entry=" + json); } IndexRequest request = new IndexRequest(getESIndexName(), ElasticSearchConstants.ENTRY_TYPE, entryId); request.source(json, XContentType.JSON); bulkRequest.add(request); } catch (JSONException e) { throw new NuxeoException("Unable to deserialize json entry=" + json, e); } } esClient.bulk(bulkRequest); }
/** * Return the ecm:path of an ES document or null if not found. */ String getPathOfDocFromEs(String repository, String docId) { String indexName = getWriteIndexForRepository(repository); GetRequest request = new GetRequest(indexName, DOC_TYPE, docId).fetchSourceContext( new FetchSourceContext(true, new String[] { PATH_FIELD }, null)); if (log.isDebugEnabled()) { log.debug(String.format("Get path of doc: curl -XGET 'http://localhost:9200/%s/%s/%s?fields=%s'", indexName, DOC_TYPE, docId, PATH_FIELD)); } GetResponse ret = esa.getClient().get(request); if (!ret.isExists() || ret.getSource() == null || ret.getSource().get(PATH_FIELD) == null) { // doc not found return null; } return ret.getSource().get(PATH_FIELD).toString(); }
esa.refreshRepositoryIndex(repo.getRepositoryName()); if (waitForAuditStoredInEs) { esa.getClient().refresh(esa.getIndexNameForType(ElasticSearchConstants.ENTRY_TYPE));