@Override public long getDelay(TimeUnit unit) { return unit.convert(expireTimeMillis - clock.getTime(), TimeUnit.MILLISECONDS); }
@Override public void readFields(DataInput dataInput) throws IOException { this.containerId = ConverterUtils.toContainerId(Text.readString(dataInput)); }
/** * Compute the expired date partition, using the underlying clock in UTC time. */ private static void computeExpiredDatePtn(long ttl) { // Use UTC date to ensure reader date is same on all timezones. LocalDate expiredDate = LocalDateTime.ofEpochSecond((clock.getTime() - ttl) / 1000, 0, ZoneOffset.UTC).toLocalDate(); expiredDatePtn = "date=" + DateTimeFormatter.ISO_LOCAL_DATE.format(expiredDate); }
public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(path); localResource.setResource(ConverterUtils.getYarnUrlFromPath(path)); localResource.setSize(fileStatus.getLen()); localResource.setTimestamp(fileStatus.getModificationTime()); localResource.setType(type); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
public Resource getCapability() { // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(memoryMb); capability.setVirtualCores(vCores); DoYUtil.callSetDiskIfExists(capability, disks); return capability; }
@Override protected void preHead(HTML<_> html) { super.preHead(html); setTitle(join($(APP_TYPE) + " Application ", $(APP_ID))); }
@Override @Nullable public ApplicationId getClusterId(CommandLine commandLine) { if (commandLine.hasOption(applicationId.getOpt())) { return ConverterUtils.toApplicationId(commandLine.getOptionValue(applicationId.getOpt())); } else if (isYarnPropertiesFileMode(commandLine)) { return yarnApplicationIdFromYarnProperties; } else { return null; } }
private LlapDaemonJvmMetrics(String displayName, String sessionId, final Configuration conf) { this.name = displayName; this.sessionId = sessionId; Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, null, ResourceCalculatorProcessTree.class); this.processTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("" + daemonPid, clazz, conf); if (processTree != null) { this.processTree.setConf(conf); } this.registry = new MetricsRegistry("LlapDaemonJvmRegistry"); this.registry.tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME).tag(SessionId, sessionId); }
@Override public void setup() { bind(HsWebServices.class); bind(JAXBContextResolver.class); bind(GenericExceptionHandler.class); bind(AppContext.class).toInstance(history); bind(HistoryContext.class).toInstance(history); route("/", HsController.class); route(pajoin("/job", APP_ID), HsController.class, "job"); route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER, CONTAINER_LOG_TYPE), HsController.class, "logs"); } }
static EventLogger getInstance(HiveConf conf) { if (instance == null) { synchronized (EventLogger.class) { if (instance == null) { instance = new EventLogger(conf, SystemClock.getInstance()); ShutdownHookManager.addShutdownHook(instance::shutdown); } } } return instance; } }
@Override public void serviceStart() throws Exception { LOG.info("Using ShufflePort: " + localShufflePort.get()); AuxiliaryServiceHelper.setServiceDataIntoEnv( TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID, ByteBuffer.allocate(4).putInt(localShufflePort.get()), localEnv); super.serviceStart(); }
/** * @param file Path. * @param fs File system. * @param type Local resource type. * @throws Exception If failed. */ public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type) throws Exception { LocalResource resource = Records.newRecord(LocalResource.class); file = fs.makeQualified(file); FileStatus stat = fs.getFileStatus(file); resource.setResource(ConverterUtils.getYarnUrlFromPath(file)); resource.setSize(stat.getLen()); resource.setTimestamp(stat.getModificationTime()); resource.setType(type); resource.setVisibility(LocalResourceVisibility.APPLICATION); return resource; }
/** * Creates a YARN resource for the remote object at the given location. * * @param remoteRsrcPath remote location of the resource * @param resourceSize size of the resource * @param resourceModificationTime last modification time of the resource * * @return YARN resource */ private static LocalResource registerLocalResource( Path remoteRsrcPath, long resourceSize, long resourceModificationTime) { LocalResource localResource = Records.newRecord(LocalResource.class); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(resourceSize); localResource.setTimestamp(resourceModificationTime); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
@Override public long getDelay(TimeUnit unit) { return unit.convert(localityDelayTimeout - clock.getTime(), TimeUnit.MILLISECONDS); }
@Override protected void preHead(Page.HTML<_> html) { super.preHead(html); String jobID = $(APP_ID); set(TITLE, jobID.isEmpty() ? "Bad request: missing job ID" : join($(APP_TYPE) + " Application ", $(APP_ID))); }
@Override public synchronized long getTime() { if (time != -1) { return time; } return actualClock.getTime(); }
public void savedmodel() { setTitle(join($(APP_TYPE) + " Application ", $(APP_ID))); set(SAVE_MODEL_STATUS, String.valueOf(app.context.getSavingModelStatus())); set(SAVE_MODEL_TOTAL, String.valueOf(app.context.getSavingModelTotalNum())); if (!app.context.getStartSavingStatus()) { app.context.startSavingModelStatus(true); set(SAVE_MODEL, String.valueOf(app.context.getStartSavingStatus())); } else { set(SAVE_MODEL, String.valueOf(app.context.getStartSavingStatus())); } render(InfoPage.class); } }
/** * Returns the current time, using the underlying clock in UTC time. */ public LocalDateTime getNow() { // Use UTC date to ensure reader date is same on all timezones. return LocalDateTime.ofEpochSecond(clock.getTime() / 1000, 0, ZoneOffset.UTC); }
@Override public void index() { setTitle(join($(APP_TYPE) + " Application ", $(APP_ID))); if (app.context.getLastSavingStatus() && app.context.getStartSavingStatus() && app.context.getSavingModelStatus() == app.context.getSavingModelTotalNum()) { app.context.startSavingModelStatus(false); } set(SAVE_MODEL, String.valueOf(app.context.getStartSavingStatus())); set(SAVE_MODEL_STATUS, String.valueOf(app.context.getSavingModelStatus())); set(LAST_SAVE_STATUS, String.valueOf(app.context.getLastSavingStatus())); set(SAVE_MODEL_TOTAL, String.valueOf(app.context.getSavingModelTotalNum())); render(InfoPage.class); }