try { boolean uploadedLogsInThisCycle = false; try (LogWriter writer = new LogWriter()){ try { writer.initialize(this.conf, this.remoteNodeTmpLogFileForApp, this.userUgi); writer.writeApplicationACLs(appAcls); writer.writeApplicationOwner(this.userUgi.getShortUserName()); } catch (IOException e1) { logAggregationSucceedInThisCycle = false;
try (LogWriter logWriter = new LogWriter()) { logWriter.initialize(conf, remoteAppLogFile, ugi); logWriter.append(logKey, logValue);
try { writer = new LogWriter(this.conf, this.remoteNodeTmpLogFileForApp, this.userUgi); writer.writeApplicationACLs(appAcls); writer.writeApplicationOwner(this.userUgi.getShortUserName()); writer.close(); writer = null; writer.close();
try (LogWriter logWriter = new LogWriter()) { logWriter.initialize(conf, remoteAppLogFile, ugi); logWriter.append(logKey, logValue);
try { writer = new LogWriter(this.conf, this.remoteNodeTmpLogFileForApp, this.userUgi); writer.writeApplicationACLs(appAcls); writer.writeApplicationOwner(this.userUgi.getShortUserName()); writer.close(); writer = null; writer.close();
try (LogWriter logWriter = new LogWriter()) { logWriter.initialize(conf, remoteAppLogFile, ugi); LogKey logKey = new LogKey(testContainerId1); logWriter.append(logKey, logValue);
try (LogWriter logWriter = new LogWriter()) { logWriter.initialize(conf, remoteAppLogFile, ugi); spyLogValue).secureOpenFile(errorFile); logWriter.append(logKey, spyLogValue);
writeSrcFile(srcFilePath, "stdout", numChars); try (LogWriter logWriter = new LogWriter()) { logWriter.initialize(conf, remoteAppLogFile, ugi); LogKey logKey = new LogKey(testContainerId); LogValue logValue = spyLogValue).secureOpenFile(errorFile); logWriter.append(logKey, spyLogValue);
YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE), null, conf); writeVersion();
this.uploadedFileMeta, appFinished, containerFinished, userFolder); try { writer.append(logKey, logValue); } catch (Exception e) { LOG.error("Couldn't upload logs for " + containerId
this.uploadedFileMeta, appFinished); try { writer.append(logKey, logValue); } catch (Exception e) { LOG.error("Couldn't upload logs for " + containerId
this.uploadedFileMeta, appFinished); try { writer.append(logKey, logValue); } catch (Exception e) { LOG.error("Couldn't upload logs for " + containerId
try (LogWriter logWriter = new LogWriter()) { logWriter.initialize(new Configuration(), remoteAppLogFile, ugi); logWriter.append(logKey, logValue);
public LogWriter(final Configuration conf, final Path remoteAppLogFile, UserGroupInformation userUgi) throws IOException { try { this.fsDataOStream = userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() { @Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } }); } catch (InterruptedException e) { throw new IOException(e); } // Keys are not sorted: null arg // 256KB minBlockSize : Expected log size for each container too this.writer = new TFile.Writer(this.fsDataOStream, 256 * 1024, conf.get( YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE, YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE), null, conf); //Write the version string writeVersion(); }
public LogWriter(final Configuration conf, final Path remoteAppLogFile, UserGroupInformation userUgi) throws IOException { try { this.fsDataOStream = userUgi.doAs(new PrivilegedExceptionAction<FSDataOutputStream>() { @Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } }); } catch (InterruptedException e) { throw new IOException(e); } // Keys are not sorted: null arg // 256KB minBlockSize : Expected log size for each container too this.writer = new TFile.Writer(this.fsDataOStream, 256 * 1024, conf.get( YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE, YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE), null, conf); //Write the version string writeVersion(); }
private void writeLog(Configuration configuration, String user) throws Exception { ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1); ContainerId containerId = ContainerIdPBImpl.newContainerId(appAttemptId, 1); String path = "target/logs/" + user + "/logs/application_0_0001/localhost_1234"; File f = new File(path); if (!f.getParentFile().exists()) { assertTrue(f.getParentFile().mkdirs()); } List<String> rootLogDirs = Arrays.asList("target/logs/logs"); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); try (AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter()) { writer.initialize(configuration, new Path(path), ugi); writer.writeApplicationOwner(ugi.getUserName()); Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>(); appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); writer.writeApplicationACLs(appAcls); writer.append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"), new AggregatedLogFormat.LogValue(rootLogDirs, containerId, UserGroupInformation.getCurrentUser().getShortUserName(), UserGroupInformation.getCurrentUser(). getShortUserName())); } }
private void writeLog(Configuration configuration, String user) throws Exception { ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1); ContainerId containerId = ContainerIdPBImpl.newContainerId(appAttemptId, 1); String path = "target/logs/" + user + "/logs/application_0_0001/localhost_1234"; File f = new File(path); if (!f.getParentFile().exists()) { assertTrue(f.getParentFile().mkdirs()); } List<String> rootLogDirs = Arrays.asList("target/logs/logs"); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter( configuration, new Path(path), ugi); writer.writeApplicationOwner(ugi.getUserName()); Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>(); appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName()); writer.writeApplicationACLs(appAcls); writer.append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"), new AggregatedLogFormat.LogValue(rootLogDirs, containerId,UserGroupInformation.getCurrentUser().getShortUserName())); writer.close(); }