/** * Default value is "NORMAL". Which implies {@link #get(Comparable)} will always return Priotity.NORMAL * * @return "NORMAL" */ @Override protected String getDefaultConf() { return Priority.NORMAL.toString(); } }
/** * Parse value method * * @param s * @return parsed Priority from String s * @see RangeConf#parseValue(String) */ @Override protected Priority parseValue(String s) { return Priority.valueOf(s); }
private String canLaunchWithPriorityConstraint(QueryContext candidateQuery, Set<QueryContext> launchedQueries) { if (maxConcurrentQueriesPerPriority == null) { return null; } Priority priority = candidateQuery.getPriority(); Integer limit = maxConcurrentQueriesPerPriority.get(priority); if (limit == null) { return null; } int launchedOnPriority = 0; for (QueryContext context : launchedQueries) { if (context.getPriority().equals(priority)) { launchedOnPriority++; } } if (launchedOnPriority >= limit) { return launchedOnPriority + "/" + limit + " queries running with priority " + priority; } return null; }
@Override public int compare(final QueryContext o1, final QueryContext o2) { return o1.getPriority().compareTo(o2.getPriority()); } }
@Test public void testRemoveMethodMustChangeQueryPriorityIndices() { Priority[] priorities = Priority.values(); /* Initialization */ int numberOfQueries = priorities.length; QueryCollection collection = createQueriesTreeSetWithQueryHandleAndPriorityStubbing(priorities, MOCK_HANDLE); QueryContext completedQuery = getMockedQueryFromQueries(collection.getQueries(), MOCK_HANDLE, 1); QueryContext queuedQuery = getMockedQueryFromQueries(collection.getQueries(), MOCK_HANDLE, 5); /* Verification 1: Verifies that all queries were added into the collection*/ assertEquals(collection.getQueriesCount(), numberOfQueries); /* Execution */ collection.remove(completedQuery); /* Verification 2: Verifies that queries were removed from the collection */ assertEquals(collection.getQueriesCount(), numberOfQueries - 1); /* Verification 3: Verifies that query index is decreased after removal of queries which were present before them in the queries list */ assertEquals(collection.getQueryIndex(queuedQuery).intValue(), 4); /* Verification 4: Verifies that query index is increasing when query with existing priority added to list */ completedQuery.setPriority(Priority.NORMAL); collection.add(completedQuery); assertEquals(collection.getQueryIndex(queuedQuery).intValue(), 5); }
private String canLaunchWithPriorityConstraint(QueryContext candidateQuery, Set<QueryContext> launchedQueries) { if (maxConcurrentQueriesPerPriority == null) { return null; } Priority priority = candidateQuery.getPriority(); Integer limit = maxConcurrentQueriesPerPriority.get(priority); if (limit == null) { return null; } int launchedOnPriority = 0; for (QueryContext context : launchedQueries) { if (context.getPriority().equals(priority)) { launchedOnPriority++; } } if (launchedOnPriority >= limit) { return launchedOnPriority + "/" + limit + " queries running with priority " + priority; } return null; }
@Override public int compare(final QueryContext o1, final QueryContext o2) { return o1.getPriority().compareTo(o2.getPriority()); } }
/** * Default value is "NORMAL". Which implies {@link #get(Comparable)} will always return Priotity.NORMAL * * @return "NORMAL" */ @Override protected String getDefaultConf() { return Priority.NORMAL.toString(); } }
/** * Parse value method * * @param s * @return parsed Priority from String s * @see RangeConf#parseValue(String) */ @Override protected Priority parseValue(String s) { return Priority.valueOf(s); }
Priority decidePriority(AbstractQueryContext ctx, QueryPriorityDecider queryPriorityDecider) { if (whetherCalculatePriority && ctx.getPriority() == null) { try { // On-demand re-computation of cost, in case it's not alredy set by a previous estimate call. // In driver test cases, estimate doesn't happen. Hence this code path ensures cost is computed and // priority is set based on correct cost. if (ctx.getDriverQueryCost(this) == null) { ctx.setDriverCost(this, this.estimate(ctx)); } // Inside try since non-data fetching queries can also be executed by async method. Priority priority = queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(this)); String priorityStr = priority.toString(); ctx.getDriverConf(this).set("mapred.job.priority", priorityStr); Map<String, String> confUpdate = new HashMap<>(); confUpdate.put("mapred.job.priority", priorityStr); ctx.updateConf(confUpdate); log.info("set priority to {}", priority); return priority; } catch (Exception e) { // not failing query launch when setting priority fails // priority will be set to usually NORMAL - the default in underlying system. log.error("could not set priority for lens session id:{} User query: {}", ctx.getLensSessionIdentifier(), ctx.getUserQuery(), e); return null; } } return null; }
@Override public Priority parseKey(String str) { return Priority.valueOf(str.toUpperCase()); }
Priority decidePriority(AbstractQueryContext ctx, QueryPriorityDecider queryPriorityDecider) { if (whetherCalculatePriority && ctx.getPriority() == null) { try { // On-demand re-computation of cost, in case it's not alredy set by a previous estimate call. // In driver test cases, estimate doesn't happen. Hence this code path ensures cost is computed and // priority is set based on correct cost. if (ctx.getDriverQueryCost(this) == null) { ctx.setDriverCost(this, this.estimate(ctx)); } // Inside try since non-data fetching queries can also be executed by async method. Priority priority = queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(this)); String priorityStr = priority.toString(); ctx.getDriverConf(this).set("mapred.job.priority", priorityStr); Map<String, String> confUpdate = new HashMap<>(); confUpdate.put("mapred.job.priority", priorityStr); ctx.updateConf(confUpdate); log.info("set priority to {}", priority); return priority; } catch (Exception e) { // not failing query launch when setting priority fails // priority will be set to usually NORMAL - the default in underlying system. log.error("could not set priority for lens session id:{} User query: {}", ctx.getLensSessionIdentifier(), ctx.getUserQuery(), e); return null; } } return null; }
@Override public Priority parseKey(String str) { return Priority.valueOf(str.toUpperCase()); }
@Test(enabled=true, dataProvider = "priority_check") public void testPriority(String query, Priority priority) throws Exception { QueryHandle qh = (QueryHandle) qHelper.executeQuery(query).getData(); qHelper.waitForQueryToRun(qh); LensQuery lq = qHelper.getLensQuery(sessionHandleString, qh); String progressMsg = lq.getStatus().getProgressMessage(); logger.info("Progress msg : " + progressMsg); String jobId = Util.getJobIdFromProgressMsg(progressMsg); Assert.assertEquals(lq.getPriority(), priority); Assert.assertEquals(Util.getMapredJobPrority(jobUrl, jobId), priority.toString()); }
@DataProvider public Object[][] priorityDataProvider() throws IOException, ParseException { BufferedReader br = new BufferedReader(new InputStreamReader( TestHiveDriver.class.getResourceAsStream("/priority_tests.data"))); String line; int numTests = Integer.parseInt(br.readLine()); Object[][] data = new Object[numTests][2]; for (int i = 0; i < numTests; i++) { String[] kv = br.readLine().split("\\s*:\\s*"); final Set<FactPartition> partitions = getFactParts(Arrays.asList(kv[0].trim().split("\\s*,\\s*"))); final Priority expected = Priority.valueOf(kv[1]); data[i] = new Object[]{partitions, expected}; } return data; }
Priority priority = driver.decidePriority(ctx); assertEquals(priority, expected, "cost: " + ctx.getDriverQueryCost(driver) + "priority: " + priority); assertEquals(ctx.getConf().get("mapred.job.priority"), priority.toString()); assertEquals(driver.decidePriority(ctx, alwaysNormalPriorityDecider), Priority.NORMAL);
public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) { if (null == selectedDriver && null != driverName) { selectedDriver = getDriverFromName(drivers); } QueryContext qctx = new QueryContext(userQuery, submitter, this.conf, conf, drivers, selectedDriver, submissionTime, false); qctx.setQueryHandle(QueryHandle.fromString(handle)); qctx.setLaunchTime(this.startTime); qctx.setEndTime(getEndTime()); qctx.setStatusSkippingTransitionTest(new QueryStatus(0.0, null, QueryStatus.Status.valueOf(getStatus()), null, getResult() != null, null, getErrorMessage() == null ? "" : getErrorMessage(), null)); qctx.getDriverStatus().setDriverStartTime(getDriverStartTime()); qctx.getDriverStatus().setDriverFinishTime(getDriverEndTime()); qctx.setResultSetPath(getResult()); qctx.setQueryName(getQueryName()); if (null != driverQuery){ qctx.setSelectedDriverQuery(driverQuery); } if (getPriority() != null) { qctx.setPriority(Priority.valueOf(getPriority())); } qctx.setFailedAttempts(getFailedAttempts()); return qctx; }
this.priority = ctx.getPriority().toString();