@Override public long getDelay(TimeUnit unit) { return unit.convert(expireTimeMillis - clock.getTime(), TimeUnit.MILLISECONDS); }
@Override public long getDelay(TimeUnit unit) { return unit.convert(localityDelayTimeout - clock.getTime(), TimeUnit.MILLISECONDS); }
@Override public synchronized long getTime() { if (time != -1) { return time; } return actualClock.getTime(); }
/** * Returns the current time, using the underlying clock in UTC time. */ public LocalDateTime getNow() { // Use UTC date to ensure reader date is same on all timezones. return LocalDateTime.ofEpochSecond(clock.getTime() / 1000, 0, ZoneOffset.UTC); }
@Override public void fragmentCompleting(String fragmentId, State state) { int val = 0; do { // Tez internals may register the same task as completing multiple times. val = runningFragmentCount.get(); if (val == 0) { LOG.warn("RunningFragmentCount is already 0. Multiple calls for the same completion."); return; } } while (!runningFragmentCount.compareAndSet(val, val - 1)); completingFragmentMap.put(fragmentId, new FragmentCompletion(state, clock.getTime())); }
@Override public void registerContainer(XLearningContainerId containerId, String role) { containerId2Status.put(containerId, XLearningContainerStatus.UNDEFINED); containerId2Role.put(containerId, role); reporterProgress.put(containerId, ""); containersAppStartTimeMap.put(containerId, ""); containersAppFinishTimeMap.put(containerId, ""); containersCpuMetrics.put(containerId, new ConcurrentHashMap<String, LinkedBlockingDeque<Object>>()); containersCpuStatistics.put(containerId, new ConcurrentHashMap<String, ContainerMetricsStatisticsTuple>()); if (role.equals(XLearningConstants.WORKER) || (role.equals(XLearningConstants.PS) && (xlearningAppType.equals("TENSORFLOW") || xlearningAppType.equals("LIGHTLDA")))) { containerId2InnerModel.put(containerId, new InnerModelSavedPair()); } runningContainers.put(containerId, new LastTime(clock.getTime())); }
void disableNode(boolean commFailure) { long duration = blacklistConf.minDelay; long currentTime = clock.getTime(); this.hadCommFailure = commFailure; disabled = true; if (numSuccessfulTasksAtLastBlacklist == numSuccessfulTasks) { // Relying on a task succeeding to reset the exponent. // There's no notifications on whether a task gets accepted or not. That would be ideal to // reset this. cumulativeBackoffFactor = cumulativeBackoffFactor * blacklistConf.backoffFactor; } else { // Was able to execute something before the last blacklist. Reset the exponent. cumulativeBackoffFactor = 1.0f; } long delayTime = (long) (duration * cumulativeBackoffFactor); if (delayTime > blacklistConf.maxDelay) { delayTime = blacklistConf.maxDelay; } if (LOG.isInfoEnabled()) { LOG.info("Disabling instance {} for {} milli-seconds. commFailure={}", toShortString(), delayTime, commFailure); } expireTimeMillis = currentTime + delayTime; numSuccessfulTasksAtLastBlacklist = numSuccessfulTasks; }
try { LOG.info("Start delete log thread."); Long currentClock = clock.getTime(); fs = FileSystem.get(conf); FileStatus[] allHistoryLog = fs.listStatus(historyLog);
private static long maybeLogSummary(Clock clock, long lastSummaryLogTime, LlapStatusServiceDriver statusServiceDriver, boolean watchMode, long watchTimeout, State launchingState) { long currentTime = clock.getTime(); if (lastSummaryLogTime < currentTime - LOG_SUMMARY_INTERVAL) { String diagString = null; if (launchingState == null && statusServiceDriver.appStatusBuilder.getState() == State.COMPLETE && watchMode) { // First known state was COMPLETED. Wait for the app launch to start. diagString = "Awaiting LLAP launch"; // Clear completed instances in this case. Don't want to provide information from the previous run. statusServiceDriver.appStatusBuilder.clearCompletedLlapInstances(); } else { diagString = constructDiagnostics(statusServiceDriver.appStatusBuilder); } if (lastSummaryLogTime == -1) { if (watchMode) { CONSOLE_LOGGER.info("\nLLAPSTATUS WatchMode with timeout={} s", watchTimeout/1000); } else { CONSOLE_LOGGER.info("\nLLAPSTATUS"); } CONSOLE_LOGGER.info("--------------------------------------------------------------------------------"); } CONSOLE_LOGGER.info(diagString); CONSOLE_LOGGER.info("--------------------------------------------------------------------------------"); lastSummaryLogTime = currentTime; } return lastSummaryLogTime; }
private void updateFallOffStats( String requestId) { long now = clock.getTime(); FragmentCompletion fragmentCompletion = completingFragmentMap.remove(requestId);
private ApplicationReport getAppReport(String appName, ServiceClient serviceClient, long timeoutMs) throws LlapStatusCliException { Clock clock = SystemClock.getInstance(); long startTime = clock.getTime(); long timeoutTime = timeoutMs < 0 ? Long.MAX_VALUE : (startTime + timeoutMs); ApplicationReport appReport = null; long remainingTime = Math.min(timeoutTime - clock.getTime(), 500L); if (remainingTime > 0) { Thread.sleep(remainingTime);
LOG.info("preempting {} for task at priority {} with potentialHosts={}", taskInfo, forPriority, potentialHosts == null ? "" : Arrays.toString(potentialHosts)); taskInfo.setPreemptedInfo(clock.getTime()); if (preemptedTaskList == null) { preemptedTaskList = new LinkedList<>();
@Override public void run() { while (!Thread.currentThread().isInterrupted()) { long currentTime = clock.getTime(); Set<Entry<XLearningContainerId, LastTime>> entrySet = runningContainers.entrySet(); for (Entry<XLearningContainerId, LastTime> entry : entrySet) {
LastTime lastTime = runningContainers.get(containerId); if (lastTime != null) { lastTime.setLastTime(clock.getTime());
requestedHostsDebugStr); long schedulerAttemptTime = clock.getTime();
@Override public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks, Priority priority, Object containerSignature, Object clientCookie) { TezTaskAttemptID id = getTaskAttemptId(task); TaskInfo taskInfo = new TaskInfo(localityDelayConf, clock, task, clientCookie, priority, capability, hosts, racks, clock.getTime(), id); LOG.info("Received allocateRequest. task={}, priority={}, capability={}, hosts={}", task, priority, capability, Arrays.toString(hosts)); writeLock.lock(); try { if (!dagRunning && metrics != null && id != null) { metrics.setDagId(id.getTaskID().getVertexID().getDAGId().toString()); } dagRunning = true; dagStats.registerTaskRequest(hosts, racks); } finally { writeLock.unlock(); } addPendingTask(taskInfo); trySchedulingPendingTasks(); }
@Override public void allocateTask(Object task, Resource capability, ContainerId containerId, Priority priority, Object containerSignature, Object clientCookie) { // Container affinity can be implemented as Host affinity for LLAP. Not required until // 1:1 edges are used in Hive. TezTaskAttemptID id = getTaskAttemptId(task); TaskInfo taskInfo = new TaskInfo(localityDelayConf, clock, task, clientCookie, priority, capability, null, null, clock.getTime(), id); LOG.info("Received allocateRequest. task={}, priority={}, capability={}, containerId={}", task, priority, capability, containerId); writeLock.lock(); try { if (!dagRunning && metrics != null && id != null) { metrics.setDagId(id.getTaskID().getVertexID().getDAGId().toString()); } dagRunning = true; dagStats.registerTaskRequest(null, null); } finally { writeLock.unlock(); } addPendingTask(taskInfo); trySchedulingPendingTasks(); }
private HiveHookEventProto getPostHookEvent(HookContext hookContext, boolean success) { QueryPlan plan = hookContext.getQueryPlan(); LOG.info("Received post-hook notification for: " + plan.getQueryId()); HiveHookEventProto.Builder builder = HiveHookEventProto.newBuilder(); builder.setEventType(EventType.QUERY_COMPLETED.name()); builder.setTimestamp(clock.getTime()); builder.setHiveQueryId(plan.getQueryId()); builder.setUser(getUser(hookContext)); builder.setRequestUser(getRequestUser(hookContext)); if (hookContext.getOperationId() != null) { builder.setOperationId(hookContext.getOperationId()); } addMapEntry(builder, OtherInfoType.STATUS, Boolean.toString(success)); JSONObject perfObj = new JSONObject(); for (String key : hookContext.getPerfLogger().getEndTimes().keySet()) { perfObj.put(key, hookContext.getPerfLogger().getDuration(key)); } addMapEntry(builder, OtherInfoType.PERF, perfObj.toString()); return builder.build(); }
dagStats.registerTaskAllocated(taskInfo.requestedHosts, taskInfo.requestedRacks, nodeInfo.getHost()); taskInfo.setAssignmentInfo(nodeInfo, container.getId(), clock.getTime()); registerRunningTask(taskInfo); nodeInfo.registerTaskScheduled();