public AttemptId nextAttempt() { return new AttemptId(externalId, attempt + 1); } }
public ExternalId getExternalId() { return attemptId.getExternalId(); }
protected Foreman( final SabotContext context, final Executor executor, final CompletionListener listener, final ExternalId externalId, final QueryObserver observer, final UserSession session, final UserRequest request, final OptionProvider config, final ReAttemptHandler attemptHandler, final CoordToExecTunnelCreator tunnelCreator, Cache<Long, PreparedPlan> plans, final ResourceAllocator queryResourceManager) { this.attemptId = AttemptId.of(externalId); this.executor = executor; this.context = context; this.listener = listener; this.session = session; this.request = request; this.config = config; this.observer = observer; this.attemptHandler = attemptHandler; this.tunnelCreator = tunnelCreator; this.plans = plans; this.queryResourceManager = queryResourceManager; }
@Test public void testCTAS() { OptionManager options = Mockito.mock(OptionManager.class); ReAttemptHandler attemptHandler = new ExternalAttemptHandler(options); AttemptId id = new AttemptId(); final UserException userException = UserException.memoryError(null).build(NoOutputLogger.INSTANCE); assertEquals(AttemptReason.NONE, attemptHandler.isRecoverable(new ReAttemptContext(id, userException, false, true))); }
protected AttemptManager newAttemptManager(SabotContext context, AttemptId attemptId, UserRequest queryRequest, AttemptObserver observer, UserSession session, OptionProvider options, CoordToExecTunnelCreator tunnelCreator, Cache<Long, PreparedPlan> plans, Predicate<DatasetConfig> datasetValidityChecker) { final QueryContext queryContext = new QueryContext(session, context, attemptId.toQueryId(), queryRequest.getPriority(), queryRequest.getMaxAllocation(), datasetValidityChecker); return new AttemptManager(context, attemptId, queryRequest, observer, options, tunnelCreator, plans, queryContext, queryResourceManager); }
public static String toString(AttemptId attemptId) { return QueryIdHelper.getQueryId(attemptId.toQueryId()); }
private boolean recoverFromFailure(AttemptReason reason, Predicate<DatasetConfig> datasetValidityChecker) { // request a new attemptId attemptId = attemptId.nextAttempt(); logger.info("{}: Starting new attempt because of {}", attemptId, reason); synchronized (this) { if (canceled) { return false; // no need to run a new attempt } newAttempt(reason, datasetValidityChecker); } return true; }
protected CommandCreator newCommandCreator(QueryContext queryContext, AttemptObserver observer, Pointer<QueryId> prepareId) { return new CommandCreator(this.sabotContext, queryContext, tunnelCreator, queryRequest, observer, plans, prepareId, attemptId.getAttemptNum(), queryResourceManager); }
@Override public Iterator<Object> getIterator(SabotContext sContext, OperatorContext context) { final CoordinationProtos.NodeEndpoint endpoint = sContext.getEndpoint(); final Iterable<TaskPool.ThreadInfo> threadInfos = sContext.getWorkStatsProvider().get().getSlicingThreads(); return (Iterator<Object>) (Object) StreamSupport.stream(threadInfos.spliterator(), false) .map((info) -> new SlicingThreadInfo( endpoint.getAddress(), endpoint.getFabricPort(), info )).iterator(); } },
@Override public Object next() { ThreadInfo currentThread = threadInfoIterator.next(); final NodeEndpoint endpoint = dbContext.getEndpoint(); final long id = currentThread.getThreadId(); return new ThreadSummary(endpoint.getAddress(), endpoint.getFabricPort(), currentThread.getThreadName(), currentThread.getThreadId(), currentThread.isInNative(), currentThread.isSuspended(), currentThread.getThreadState().name(), stats.getCpuTrailingAverage(id, 1), stats.getUserTrailingAverage(id, 1), VM.availableProcessors(), getStackTrace(currentThread)); }
@Override public void run() { SafeExit safeExit; try { safeExit = provider.get(); } catch (Exception ex){ // ignore since this means no instance wasn't running on this node. return; } safeExit.waitToExit(); } });
public FragmentIterator(SabotContext dbContext, OperatorContext c) { iter = dbContext.getWorkStatsProvider().get().getRunningFragments(); }
public QueryIterator(SabotContext dbContext, OperatorContext c) { iter = ImmutableList.copyOf(dbContext.getRunningQueryProvider().get().getRunningQueries()).iterator(); }
/** * Generate set of assigned fragments based on predefined PlanningSet * versus doing parallelization in place * QueryContext has to be not null from construction of Parallelizer * @param options * @param planningSet * @param reader * @param rootFragment * @return * @throws ExecutionSetupException */ public List<PlanFragment> getFragments( OptionList options, PlanningSet planningSet, PhysicalPlanReader reader, Fragment rootFragment) throws ExecutionSetupException { Preconditions.checkNotNull(queryContext); final Stopwatch stopwatch = Stopwatch.createStarted(); observer.planAssignmentTime(stopwatch.elapsed(TimeUnit.MILLISECONDS)); List<PlanFragment> fragments = generateWorkUnit(options, reader, rootFragment, planningSet); stopwatch.stop(); observer.planGenerationTime(stopwatch.elapsed(TimeUnit.MILLISECONDS)); observer.plansDistributionComplete(new QueryWorkUnit(fragments)); return fragments; }
@Override public FragmentInfo apply(final FragmentExecutor fragmentExecutor) { final FragmentStatus status = fragmentExecutor.getStatus(); final ExecProtos.FragmentHandle handle = fragmentExecutor.getHandle(); final MinorFragmentProfile profile = status == null ? null : status.getProfile(); Long memoryUsed = profile == null ? 0 : profile.getMemoryUsed(); Long rowsProcessed = profile == null ? 0 : getRowsProcessed(profile); Timestamp startTime = profile == null ? new Timestamp(0) : new Timestamp(profile.getStartTime()); return new FragmentInfo(dbContext.get().getEndpoint().getAddress(), QueryIdHelper.getQueryId(handle.getQueryId()), handle.getMajorFragmentId(), handle.getMinorFragmentId(), memoryUsed, rowsProcessed, startTime, fragmentExecutor.getBlockingStatus(), fragmentExecutor.getTaskDescriptor()); }
/** * Convert an external id to an attempt id, assuming this is its first attempt * @param externalId * @return */ public static AttemptId of(final ExternalId externalId) { return new AttemptId(externalId, 0); }
private static String toTableName(String schema1, String schema2, final AttemptId id) { return PathUtils.constructFullPath(Arrays.asList(schema1, schema2, QueryIdHelper.getQueryId(id.toQueryId()))); } }
/** * Convert a query id to an attemp id * @param queryId * @return */ public static AttemptId of(final QueryId queryId) { ExternalId externalId = ExternalIdHelper.toExternal(queryId); int attempt = (int) (queryId.getPart2() & MASK); return new AttemptId(externalId, attempt); }
public static AttemptId fromString(final String attemptIdStr) { final UUID uuid = UUID.fromString(attemptIdStr); final long part1 = uuid.getMostSignificantBits(); final long part2 = uuid.getLeastSignificantBits(); final ExternalId externalId = ExternalId.newBuilder() .setPart1(part1) .setPart2(part2 & ExternalIdHelper.MASK) .build(); final int attempt = (int) part2 & 0xFF; return new AttemptId(externalId, attempt); } }