@Override public String createRangeQuery(Date startDate, Date endDate) { StringTokenizer tkz = new StringTokenizer(getMigrationConfiguration().getTimestampFieldName(), ", "); List<Query> ql = new ArrayList<>(); while (tkz.hasMoreTokens()) { String tok = tkz.nextToken(); ql.add(Query.and(Query.withValue(tok, Query.gte, startDate), Query.withValue(tok, Query.lt, endDate))); } if (ql.size() == 1) { return ql.get(0).toString(); } else { return Query.or(ql).toString(); } }
/** * Retrieves jobs that are available, and their scheduled time has passed. * Returns at most batchSize jobs starting at startIndex */ public MigrationJob[] retrieveJobs(int batchSize, int startIndex, JobType jobType) throws IOException, LightblueException { LOGGER.debug("Retrieving jobs: batchSize={}, startIndex={}", batchSize, startIndex); DataFindRequest findRequest = new DataFindRequest("migrationJob", null); List<Query> conditions = new ArrayList<>(Arrays.asList(new Query[] { // get jobs for this configuration Query.withValue("configurationName", Query.eq, migrationConfiguration.getConfigurationName()), // get jobs whose state ara available Query.withValue("status", Query.eq, "available"), // only get jobs that are Query.withValue("scheduledDate", Query.lte, new Date()) })); if (jobType == JobType.GENERATED) { LOGGER.debug("Looking for generated job"); conditions.add(Query.withValue("generated", Query.eq, true)); } else if (jobType == JobType.NONGENERATED) { LOGGER.debug("Looking for non generated job"); conditions.add(Query.withValue("generated", Query.eq, false)); } findRequest.where(Query.and(conditions)); findRequest.select(Projection.includeField("*")); findRequest.range(startIndex, startIndex + batchSize - 1); LOGGER.debug("Finding Jobs to execute: {}", findRequest.getBody()); return lbClient.data(findRequest, MigrationJob[].class); }
MigrationJob.ConsistencyChecker c = j.getConsistencyChecker(); if (c != null) { qlist.add(Query.and(Query.withValue("consistencyChecker.jobRangeBegin", Query.eq, c.getJobRangeBegin()), Query.withValue("consistencyChecker.jobRangeEnd", Query.eq, c.getJobRangeEnd()), Query.withValue("consistencyChecker.configurationName", Query.eq, c.getConfigurationName()))); req.where(Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_AVAILABLE), Query.or(qlist))); req.select(Projection.includeFieldRecursively("*"));
private void doDestinationDocumentFetch(List<Identity> ids, List<JsonNode> dest) throws Exception { if (ids != null && !ids.isEmpty()) { DataFindRequest destinationRequest = new DataFindRequest(getMigrationConfiguration().getDestinationEntityName(), getMigrationConfiguration().getDestinationEntityVersion()); List<Query> requestConditions = new ArrayList<>(); for (Identity id : ids) { List<Query> docConditions = new ArrayList<>(); int i = 0; for (String keyField : getMigrationConfiguration().getDestinationIdentityFields()) { Object v = id.get(i); Query docQuery = Query.withValue(keyField, Query.eq, v == null ? null : v.toString()); docConditions.add(docQuery); i++; } requestConditions.add(Query.and(docConditions)); } destinationRequest.where(Query.or(requestConditions)); destinationRequest.select(Projection.includeFieldRecursively("*"), Projection.excludeField("objectType")); LOGGER.debug("Fetching destination docs {}", destinationRequest.getBody()); JsonNode[] nodes = getDestCli().data(destinationRequest, JsonNode[].class); if (nodes != null) { LOGGER.debug("There are {} destination docs", nodes.length); for (JsonNode node : nodes) { dest.add(node); } } } }
private boolean migrationJobsExist() { LOGGER.debug("Checking if there are migration jobs for {}", migrationConfiguration.getConfigurationName()); DataFindRequest req = new DataFindRequest("migrationJob", null); req.where(Query.and(Query.withValue("configurationName", Query.eq, migrationConfiguration.getConfigurationName()), Query.withValue("generated", Query.eq, false), Query.withValue("status", Query.eq, MigrationJob.STATE_AVAILABLE))); req.select(Projection.includeField("_id")); req.range(1, 1); try { LightblueDataResponse resp = lbClient.data(req); return resp.parseMatchCount() > 0; } catch (Exception e) { LOGGER.error("Cannot query migration jobs:{}", e, e); return true; } }
/** * Read a configuration from the database whose name matches the the given * configuration name */ public MigrationConfiguration getMigrationConfiguration(String configurationName) throws IOException, LightblueException { DataFindRequest findRequest = new DataFindRequest("migrationConfiguration", null); findRequest.where(Query.and( Query.withValue("configurationName", Query.eq, configurationName), Query.withValue("consistencyCheckerName", Query.eq, cfg.getName())) ); findRequest.select(Projection.includeFieldRecursively("*")); LOGGER.debug("Loading configuration:{}", findRequest.getBody()); return lightblueClient.data(findRequest, MigrationConfiguration.class); }
public void cleanupOldJobs(LightblueClient cli, Date cleanupBefore) { DataFindRequest findRequest = new DataFindRequest("migrationJob", null); Query q = Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_COMPLETED), Query.withValue("generated", Query.eq, true), Query.not(Query.arrayMatch("jobExecutions", Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_COMPLETED), Query.withValue("actualEndDate", Query.gt, new Literal(cleanupBefore))))));
public void enableStuckJobs(LightblueClient cli, Date enableBefore) { DataFindRequest findRequest = new DataFindRequest("migrationJob", null); // Find active jobs that's been sitting for too long findRequest.where(Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_ACTIVE), Query.arrayMatch("jobExecutions", Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_ACTIVE), Query.withValue("actualStartDate", Query.lt, new Literal(enableBefore)))))); findRequest.select(Projection.includeField("_id")); findRequest.range(0, 250); LOGGER.debug("Re-enabling stuck jobs"); for (int loop = 0; loop < 10; loop++) { try { MigrationJob[] jobs = cli.data(findRequest, MigrationJob[].class); if (jobs != null && jobs.length > 0) { LOGGER.warn("Re-enabling {} active stuck jobs", jobs.length); DataUpdateRequest upd = new DataUpdateRequest("migrationJob", null); upd.where(Query.withValues("_id", Query.in, ids(jobs))); upd.updates(Update.set("status", MigrationJob.STATE_AVAILABLE)); LOGGER.debug("update:{}", upd.getBodyJson()); cli.data(upd); } else { break; } } catch (Exception e) { LOGGER.error("Error re-activating jobs", e); } } }