/** * <pre> * { field: <field>, op: <op>, rvalue: <value> } * </pre> */ public static Query withValue(String field, BinOp op, int value) { return withValue(field, op, Literal.value(value)); }
@Override public String createRangeQuery(Date startDate, Date endDate) { StringTokenizer tkz = new StringTokenizer(getMigrationConfiguration().getTimestampFieldName(), ", "); List<Query> ql = new ArrayList<>(); while (tkz.hasMoreTokens()) { String tok = tkz.nextToken(); ql.add(Query.and(Query.withValue(tok, Query.gte, startDate), Query.withValue(tok, Query.lt, endDate))); } if (ql.size() == 1) { return ql.get(0).toString(); } else { return Query.or(ql).toString(); } }
/** * <pre> * { "$or": [{ field: <field>, regex: <^string$>, caseInsensitive: <caseInsensitive>, ... }, ... ]} * </pre> */ public static Query withStrings(String field, String[] values, boolean caseInsensitive) { if (caseInsensitive) { List<Query> regexList = new ArrayList<Query>(); for (String value : values) { regexList.add(withString(field, value, true)); } return Query.or(regexList); } else { return Query.withValues(field, Query.in, Literal.values(values)); } }
/** * <pre> * { field: <field>, regex: <pattern>, caseInsensitive: <caseInsensitive>, ... } * </pre> */ public static Query withMatchingString(String field, String value, boolean caseInsensitive) { return caseInsensitive ? regex(field, escapeRegExPattern(value), caseInsensitive, false, false, false) : withValue(field, eq, value); }
public void cleanupOldJobs(LightblueClient cli, Date cleanupBefore) { DataFindRequest findRequest = new DataFindRequest("migrationJob", null); Query q = Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_COMPLETED), Query.withValue("generated", Query.eq, true), Query.not(Query.arrayMatch("jobExecutions", Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_COMPLETED), Query.withValue("actualEndDate", Query.gt, new Literal(cleanupBefore)))))); LOGGER.debug("Deleting {} jobs", jobs.length); DataDeleteRequest del = new DataDeleteRequest("migrationJob", null); del.where(Query.withValues("_id", Query.in, ids(jobs))); cli.data(del); } else {
MigrationJob.ConsistencyChecker c = j.getConsistencyChecker(); if (c != null) { qlist.add(Query.and(Query.withValue("consistencyChecker.jobRangeBegin", Query.eq, c.getJobRangeBegin()), Query.withValue("consistencyChecker.jobRangeEnd", Query.eq, c.getJobRangeEnd()), Query.withValue("consistencyChecker.configurationName", Query.eq, c.getConfigurationName()))); req.where(Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_AVAILABLE), Query.or(qlist))); req.select(Projection.includeFieldRecursively("*")); MigrationJob[] dups = null;
/** * <pre> * { field:<field>, op:<op>, rvalue:<value> } * { field:<field>, op:<op>, values:[values] } * </pre> */ public static Query withValue(String expression) { String[] parts = split(expression); if (parts != null) { String field = parts[0]; String operator = parts[1]; String value = parts[2]; BinOp binOp = BinOp.getOp(operator); if (binOp != null) { return withValue(field, binOp, value); } NaryOp naryOp = NaryOp.getOp(operator); if (naryOp != null) { Literal[] values = Literal.values(value.substring(1, value.length() - 1).split("\\s*,\\s*")); return withValues(field, naryOp, values); } } throw new IllegalArgumentException("'" + expression + "' is incorrect"); }
/** * Retrieves jobs that are available, and their scheduled time has passed. * Returns at most batchSize jobs starting at startIndex */ public MigrationJob[] retrieveJobs(int batchSize, int startIndex, JobType jobType) throws IOException, LightblueException { LOGGER.debug("Retrieving jobs: batchSize={}, startIndex={}", batchSize, startIndex); DataFindRequest findRequest = new DataFindRequest("migrationJob", null); List<Query> conditions = new ArrayList<>(Arrays.asList(new Query[] { // get jobs for this configuration Query.withValue("configurationName", Query.eq, migrationConfiguration.getConfigurationName()), // get jobs whose state ara available Query.withValue("status", Query.eq, "available"), // only get jobs that are Query.withValue("scheduledDate", Query.lte, new Date()) })); if (jobType == JobType.GENERATED) { LOGGER.debug("Looking for generated job"); conditions.add(Query.withValue("generated", Query.eq, true)); } else if (jobType == JobType.NONGENERATED) { LOGGER.debug("Looking for non generated job"); conditions.add(Query.withValue("generated", Query.eq, false)); } findRequest.where(Query.and(conditions)); findRequest.select(Projection.includeField("*")); findRequest.range(startIndex, startIndex + batchSize - 1); LOGGER.debug("Finding Jobs to execute: {}", findRequest.getBody()); return lbClient.data(findRequest, MigrationJob[].class); }
/** * Return a query from a json node */ public static Query query(ContainerNode query) { return new Query(query); }
public void enableStuckJobs(LightblueClient cli, Date enableBefore) { DataFindRequest findRequest = new DataFindRequest("migrationJob", null); // Find active jobs that's been sitting for too long findRequest.where(Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_ACTIVE), Query.arrayMatch("jobExecutions", Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_ACTIVE), Query.withValue("actualStartDate", Query.lt, new Literal(enableBefore)))))); findRequest.select(Projection.includeField("_id")); findRequest.range(0, 250); LOGGER.debug("Re-enabling stuck jobs"); for (int loop = 0; loop < 10; loop++) { try { MigrationJob[] jobs = cli.data(findRequest, MigrationJob[].class); if (jobs != null && jobs.length > 0) { LOGGER.warn("Re-enabling {} active stuck jobs", jobs.length); DataUpdateRequest upd = new DataUpdateRequest("migrationJob", null); upd.where(Query.withValues("_id", Query.in, ids(jobs))); upd.updates(Update.set("status", MigrationJob.STATE_AVAILABLE)); LOGGER.debug("update:{}", upd.getBodyJson()); cli.data(upd); } else { break; } } catch (Exception e) { LOGGER.error("Error re-activating jobs", e); } } }
private void doDestinationDocumentFetch(List<Identity> ids, List<JsonNode> dest) throws Exception { if (ids != null && !ids.isEmpty()) { DataFindRequest destinationRequest = new DataFindRequest(getMigrationConfiguration().getDestinationEntityName(), getMigrationConfiguration().getDestinationEntityVersion()); List<Query> requestConditions = new ArrayList<>(); for (Identity id : ids) { List<Query> docConditions = new ArrayList<>(); int i = 0; for (String keyField : getMigrationConfiguration().getDestinationIdentityFields()) { Object v = id.get(i); Query docQuery = Query.withValue(keyField, Query.eq, v == null ? null : v.toString()); docConditions.add(docQuery); i++; } requestConditions.add(Query.and(docConditions)); } destinationRequest.where(Query.or(requestConditions)); destinationRequest.select(Projection.includeFieldRecursively("*"), Projection.excludeField("objectType")); LOGGER.debug("Fetching destination docs {}", destinationRequest.getBody()); JsonNode[] nodes = getDestCli().data(destinationRequest, JsonNode[].class); if (nodes != null) { LOGGER.debug("There are {} destination docs", nodes.length); for (JsonNode node : nodes) { dest.add(node); } } } }
/** * <pre> * { field: <field>, regex: <^string$>, caseInsensitive: <caseInsensitive>, ... } * </pre> */ public static Query withString(String field, String value, boolean caseInsensitive) { return caseInsensitive ? regex(field, "^" + escapeRegExPattern(value) + "$", caseInsensitive, false, false, false) : withValue(field, eq, value); }
private boolean migrationJobsExist() { LOGGER.debug("Checking if there are migration jobs for {}", migrationConfiguration.getConfigurationName()); DataFindRequest req = new DataFindRequest("migrationJob", null); req.where(Query.and(Query.withValue("configurationName", Query.eq, migrationConfiguration.getConfigurationName()), Query.withValue("generated", Query.eq, false), Query.withValue("status", Query.eq, MigrationJob.STATE_AVAILABLE))); req.select(Projection.includeField("_id")); req.range(1, 1); try { LightblueDataResponse resp = lbClient.data(req); return resp.parseMatchCount() > 0; } catch (Exception e) { LOGGER.error("Cannot query migration jobs:{}", e, e); return true; } }
/** * <pre> * { field: <field>, op: <op>, rvalue: <value> } * </pre> */ public static Query withValue(String field, BinOp op, long value) { return withValue(field, op, Literal.value(value)); }
/** * Read a configuration from the database whose name matches the the given * configuration name */ public MigrationConfiguration getMigrationConfiguration(String configurationName) throws IOException, LightblueException { DataFindRequest findRequest = new DataFindRequest("migrationConfiguration", null); findRequest.where(Query.and( Query.withValue("configurationName", Query.eq, configurationName), Query.withValue("consistencyCheckerName", Query.eq, cfg.getName())) ); findRequest.select(Projection.includeFieldRecursively("*")); LOGGER.debug("Loading configuration:{}", findRequest.getBody()); return lightblueClient.data(findRequest, MigrationConfiguration.class); }