/** * Retrieves jobs that are available, and their scheduled time has passed. * Returns at most batchSize jobs starting at startIndex */ public MigrationJob[] retrieveJobs(int batchSize, int startIndex, JobType jobType) throws IOException, LightblueException { LOGGER.debug("Retrieving jobs: batchSize={}, startIndex={}", batchSize, startIndex); DataFindRequest findRequest = new DataFindRequest("migrationJob", null); List<Query> conditions = new ArrayList<>(Arrays.asList(new Query[] { // get jobs for this configuration Query.withValue("configurationName", Query.eq, migrationConfiguration.getConfigurationName()), // get jobs whose state ara available Query.withValue("status", Query.eq, "available"), // only get jobs that are Query.withValue("scheduledDate", Query.lte, new Date()) })); if (jobType == JobType.GENERATED) { LOGGER.debug("Looking for generated job"); conditions.add(Query.withValue("generated", Query.eq, true)); } else if (jobType == JobType.NONGENERATED) { LOGGER.debug("Looking for non generated job"); conditions.add(Query.withValue("generated", Query.eq, false)); } findRequest.where(Query.and(conditions)); findRequest.select(Projection.includeField("*")); findRequest.range(startIndex, startIndex + batchSize - 1); LOGGER.debug("Finding Jobs to execute: {}", findRequest.getBody()); return lbClient.data(findRequest, MigrationJob[].class); }
private LightblueResponse saveBatch(List<JsonNode> documentsToOverwrite) throws LightblueResponseException { // LightblueClient - save & overwrite documents DataSaveRequest saveRequest = new DataSaveRequest(getMigrationConfiguration().getDestinationEntityName(), getMigrationConfiguration().getDestinationEntityVersion()); saveRequest.setUpsert(true); saveRequest.create(documentsToOverwrite.toArray()); saveRequest.returns(Projection.includeField("*")); LightblueResponse response; try { response = getDestCli().data(saveRequest); } catch (LightblueException ex) { // bad things happened, bail! throw new RuntimeException(ex); } return response; }
private boolean migrationJobsExist() { LOGGER.debug("Checking if there are migration jobs for {}", migrationConfiguration.getConfigurationName()); DataFindRequest req = new DataFindRequest("migrationJob", null); req.where(Query.and(Query.withValue("configurationName", Query.eq, migrationConfiguration.getConfigurationName()), Query.withValue("generated", Query.eq, false), Query.withValue("status", Query.eq, MigrationJob.STATE_AVAILABLE))); req.select(Projection.includeField("_id")); req.range(1, 1); try { LightblueDataResponse resp = lbClient.data(req); return resp.parseMatchCount() > 0; } catch (Exception e) { LOGGER.error("Cannot query migration jobs:{}", e, e); return true; } }
r.select(new Projection[] { Projection.includeField("_id"), Projection.array("subscriptionProducts", Query.withValue("inactiveDate", Query.eq, Literal.value(null))) }); r.where(Query.withValue("_id=0"));
public void enableStuckJobs(LightblueClient cli, Date enableBefore) { DataFindRequest findRequest = new DataFindRequest("migrationJob", null); // Find active jobs that's been sitting for too long findRequest.where(Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_ACTIVE), Query.arrayMatch("jobExecutions", Query.and(Query.withValue("status", Query.eq, MigrationJob.STATE_ACTIVE), Query.withValue("actualStartDate", Query.lt, new Literal(enableBefore)))))); findRequest.select(Projection.includeField("_id")); findRequest.range(0, 250); LOGGER.debug("Re-enabling stuck jobs"); for (int loop = 0; loop < 10; loop++) { try { MigrationJob[] jobs = cli.data(findRequest, MigrationJob[].class); if (jobs != null && jobs.length > 0) { LOGGER.warn("Re-enabling {} active stuck jobs", jobs.length); DataUpdateRequest upd = new DataUpdateRequest("migrationJob", null); upd.where(Query.withValues("_id", Query.in, ids(jobs))); upd.updates(Update.set("status", MigrationJob.STATE_AVAILABLE)); LOGGER.debug("update:{}", upd.getBodyJson()); cli.data(upd); } else { break; } } catch (Exception e) { LOGGER.error("Error re-activating jobs", e); } } }
findRequest.select(Projection.includeField("_id")); findRequest.range(0, 250); LOGGER.debug("Cleaning up old jobs");
DataUpdateRequest updateRequest = new DataUpdateRequest("migrationJob", null); updateRequest.where(Query.withValue("_id", Query.eq, migrationJob.get_id())); updateRequest.returns(Projection.includeField("_id")); updateRequest.returns(Projection.includeField("_id")); if (execution.getErrorMsg() != null) { execution.setStatus(MigrationJob.STATE_FAILED);