/** * Registers the input {@link CursorResult} and generates a new <code>scrollId</code> to associate with. * * @return the scrollId associated to the cursor result. */ public String registerCursorResult(CursorResult<C, O> cursorResult) { String scrollId = UUID.randomUUID().toString(); return registerCursorResult(scrollId, cursorResult); }
protected boolean isScrollTimedOut(String scrollId, CursorResult<C, O> cursorResult) { if (cursorResult.timedOut()) { if (unregisterCursor(scrollId)) { log.warn("Scroll '" + scrollId + "' timed out"); } return true; } return false; }
@Override public ScrollResult<String> scroll(String scrollId) { return cursorService.scroll(scrollId); }
@Override public void onApplicationStarted() { activatePersistenceProvider(); cursorService = new CursorService<>(entry -> { try { return OBJECT_MAPPER.writeValueAsString(entry); } catch (IOException e) { throw new NuxeoException("Unable to serialize entry"); } }); }
@Override public void onApplicationStopped() { if (esClient == null) { return; } try { esClient.close(); cursorService.clear(); } catch (Exception e) { log.warn("Fail to close esClient: " + e.getMessage(), e); } finally { esClient = null; cursorService = null; } }
if (cursorResult == null) { throw new NuxeoException("Unknown or timed out scrollId"); } else if (isScrollTimedOut(scrollId, cursorResult)) { throw new NuxeoException("Timed out scrollId"); synchronized (cursorResult) { if (!cursorResult.hasNext()) { unregisterCursor(scrollId); return emptyResult();
@Override public ScrollResult<String> scroll(DBSExpressionEvaluator evaluator, int batchSize, int keepAliveSeconds) { cursorService.checkForTimedOutScroll(); MongoDBRepositoryQueryBuilder builder = new MongoDBRepositoryQueryBuilder(this, evaluator.getExpression(), evaluator.getSelectClause(), null, evaluator.pathResolver, evaluator.fulltextSearchDisabled); builder.walk(); if (builder.hasFulltext && isFulltextSearchDisabled()) { throw new QueryParseException("Fulltext search disabled by configuration"); } Bson filter = builder.getQuery(); addPrincipals((Document) filter, evaluator.principals); Bson keys = builder.getProjection(); if (log.isTraceEnabled()) { logQuery(filter, keys, null, 0, 0); } MongoCursor<Document> cursor = coll.find(filter).projection(keys).batchSize(batchSize).iterator(); String scrollId = cursorService.registerCursor(cursor, batchSize, keepAliveSeconds); return scroll(scrollId); }
@Override public ScrollResult<String> scroll(String scrollId) { return cursorService.scroll(scrollId); }
public MongoDBRepository(ConnectionManager cm, MongoDBRepositoryDescriptor descriptor) { super(cm, descriptor.name, descriptor); MongoDBConnectionService mongoService = Framework.getService(MongoDBConnectionService.class); // prefix with repository/ to group repository connection MongoDatabase database = mongoService.getDatabase(REPOSITORY_CONNECTION_PREFIX + descriptor.name); coll = database.getCollection(descriptor.name); countersColl = database.getCollection(descriptor.name + ".counters"); if (Boolean.TRUE.equals(descriptor.nativeId)) { idKey = MONGODB_ID; } else { idKey = KEY_ID; } useCustomId = KEY_ID.equals(idKey); if (idType == IdType.sequence || DEBUG_UUIDS) { Integer sbs = descriptor.sequenceBlockSize; sequenceBlockSize = sbs == null ? 1 : sbs.longValue(); sequenceLeft = 0; } converter = new MongoDBConverter(useCustomId ? null : KEY_ID); cursorService = new CursorService<>(ob -> (String) ob.get(converter.keyToBson(KEY_ID))); initRepository(); }
@Override public void shutdown() { super.shutdown(); cursorService.clear(); }
/** * Registers the input {@link C} and generates a new <code>scrollId</code> to associate with. * * @return the scrollId associated to the cursor. */ public String registerCursor(C cursor, int batchSize, int keepAliveSeconds) { return registerCursorResult(new CursorResult<>(cursor, batchSize, keepAliveSeconds)); }
@Override public ScrollResult<String> scroll(String scrollId) { return cursorService.scroll(scrollId); }
@Override public void onApplicationStarted() { esClient = getClient(); if (Boolean.parseBoolean(Framework.getProperty(MIGRATION_FLAG_PROP))) { if (!isMigrationDone()) { log.info(String.format( "Property %s is true and migration is not done yet, processing audit migration from SQL to Elasticsearch index", MIGRATION_FLAG_PROP)); // Drop audit index first in case of a previous bad migration ElasticSearchAdmin esa = Framework.getService(ElasticSearchAdmin.class); esa.dropAndInitIndex(getESIndexName()); int batchSize = MIGRATION_DEFAULT_BACTH_SIZE; String batchSizeProp = Framework.getProperty(MIGRATION_BATCH_SIZE_PROP); if (batchSizeProp != null) { batchSize = Integer.parseInt(batchSizeProp); } migrate(batchSize); } else { log.warn(String.format( "Property %s is true but migration is already done, please set this property to false", MIGRATION_FLAG_PROP)); } } else { log.debug(String.format("Property %s is false, not processing any migration", MIGRATION_FLAG_PROP)); } cursorService = new CursorService<>(SearchHit::getSourceAsString); }
/** * Registers the input {@link C} associated to the input <code>scrollId</code>. * * @return the scrollId associated to the cursor. */ public String registerCursor(String scrollId, C cursor, int batchSize, int keepAliveSeconds) { return registerCursorResult(scrollId, new CursorResult<>(cursor, batchSize, keepAliveSeconds)); }
@Override public ScrollResult<String> scroll(QueryBuilder builder, int batchSize, int keepAliveSeconds) { // as we're using pages to scroll audit, we need to add an order to make results across pages deterministic builder.orders(OrderByExprs.asc(LOG_ID), builder.orders().toArray(new OrderByExpr[0])); String scrollId = cursorService.registerCursorResult( new SQLAuditCursorResult(builder, batchSize, keepAliveSeconds)); return scroll(scrollId); }
@Override public ScrollResult<String> scroll(org.nuxeo.ecm.core.query.sql.model.QueryBuilder builder, int batchSize, int keepAliveSeconds) { // prepare parameters MultiExpression predicate = builder.predicate(); OrderByList orders = builder.orders(); // create source SearchSourceBuilder source = createSearchRequestSource(predicate, orders); source.size(batchSize); // create request SearchRequest request = createSearchRequest(); request.source(source).scroll(TimeValue.timeValueSeconds(keepAliveSeconds)); SearchResponse response = runRequest(request); // register cursor String scrollId = cursorService.registerCursorResult(new ESCursorResult(response, batchSize, keepAliveSeconds)); return scroll(scrollId); }