Refine search
reader = new BufferedReader(new InputStreamReader(xlearningProcess.getInputStream())); List<OutputInfo> outputs = Arrays.asList(amClient.getOutputLocation()); JobConf jobConf = new JobConf(conf); jobConf.setOutputKeyClass(Text.class); jobConf.setOutputValueClass(Text.class); jobConf.setBoolean("mapred.output.compress", true); jobConf.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec"); jobConf); outputFormat.checkOutputSpecs(dfs, jobConf); JobID jobID = new JobID(new SimpleDateFormat("yyyyMMddHHmm").format(new Date()), 0); TaskAttemptID taId = new TaskAttemptID(new TaskID(jobID, true, 0), 0); jobConf.set("mapred.tip.id", taId.getTaskID().toString()); jobConf.set("mapred.task.id", taId.toString()); jobConf.set("mapred.job.id", jobID.toString()); amClient.reportMapedTaskID(containerId, taId.toString()); RecordWriter writer = outputFormat.getRecordWriter(dfs, jobConf, "part-r", Reporter.NULL);
private int detectCurrentInstance(Configuration conf) { TaskID taskID = HadoopCfgUtils.getTaskID(conf); if (taskID == null) { log.warn(String.format("Cannot determine task id - redirecting writes in a random fashion")); return NO_TASK_ID; } return taskID.getId(); }
public static TaskID getTaskID(Configuration cfg) { // first try with the attempt since some Hadoop versions mix the two String taskAttemptId = HadoopCfgUtils.getTaskAttemptId(cfg); if (StringUtils.hasText(taskAttemptId)) { try { return TaskAttemptID.forName(taskAttemptId).getTaskID(); } catch (IllegalArgumentException ex) { // the task attempt is invalid (Tez in particular uses the wrong string - see #346) // try to fallback to task id return parseTaskIdFromTaskAttemptId(taskAttemptId); } } String taskIdProp = HadoopCfgUtils.getTaskId(cfg); // double-check task id bug in Hadoop 2.5.x if (StringUtils.hasText(taskIdProp) && !taskIdProp.contains("attempt")) { return TaskID.forName(taskIdProp); } return null; }
/** * Localize the given JobConf to be specific for this task. */ public void localizeConfiguration(JobConf conf) throws IOException { conf.set(JobContext.TASK_ID, taskId.getTaskID().toString()); conf.set(JobContext.TASK_ATTEMPT_ID, taskId.toString()); conf.setBoolean(JobContext.TASK_ISMAP, isMapTask()); conf.setInt(JobContext.TASK_PARTITION, partition); conf.set(JobContext.ID, taskId.getJobID().toString()); }
@Test public <K, V> void testDuplicateCopySucceeded() throws Exception { JobConf job = new JobConf(); job.setNumMapTasks(2); FileSystem mockFileSystem = mock(FileSystem.class); Class<? extends org.apache.hadoop.mapred.Reducer> combinerClass = job.getCombinerClass(); @SuppressWarnings("unchecked") // needed for mock with generic CombineOutputCollector<K, V> mockCombineOutputCollector = TaskAttemptID succeedAttempt1ID = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test", 0), TaskType.MAP, 0), 0); TaskAttemptID succeedAttempt2ID = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test", 0), TaskType.MAP, 0), 1); TaskAttemptID succeedAttempt3ID = new TaskAttemptID( new org.apache.hadoop.mapred.TaskID( new JobID("test", 0), TaskType.MAP, 1), 0);
TaskAttemptID attemptidObj = TaskAttemptID.forName(attemptid); (attemptidObj == null) ? TaskID.forName(request.getParameter("tipid")) : attemptidObj.getTaskID(); JobID jobidObj = tipidObj.getJobID(); String jobid = jobidObj.toString(); JobID jobIdObj = JobID.forName(jobid); String historyFile = tracker.getJobHistory().getHistoryFilePath(jobIdObj); if (historyFile == null) { if (attemptid == null) { counters = tracker.getTipCounters(tipidObj); attemptid = tipidObj.toString(); // for page title etc for (String groupName : counters.getGroupNames()) { Counters.Group group = counters.getGroup(groupName); String displayGroupName = group.getDisplayName();
final TaskAttemptID attemptidObj = TaskAttemptID.forName(attemptid); (attemptidObj == null) ? TaskID.forName(request.getParameter("tipid")) : attemptidObj.getTaskID(); if (tipidObj == null) { out.print("<b>tipid sent is not valid.</b><br>\n"); final JobID jobidObj = tipidObj.getJobID(); String jobid = jobidObj.toString(); "&tipid=" + tipidObj.toString(); response.sendRedirect(response.encodeRedirectURL(historyUrl)); return; "&tipid=" + tipidObj.toString()); "&tipid=" + tipidObj.toString()); taskLogUrl = TaskLogServlet.getTaskLogUrl(taskTracker.getHost(), String.valueOf(taskTracker.getHttpPort()), status.getTaskID().toString()); + ((status.getCounters() != null) ? status.getCounters().size() : 0) + "</a></td>"); out.print("<td>");
public static void setMRProperties( ProcessorContext context, Configuration config, boolean isMapperOutput ) { TaskAttemptID taskAttemptId = org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl .createMockTaskAttemptID( context.getApplicationId().getClusterTimestamp(), context.getTaskVertexIndex(), context.getApplicationId().getId(), context.getTaskIndex(), context.getTaskAttemptNumber(), isMapperOutput ); config.set( JobContext.TASK_ATTEMPT_ID, taskAttemptId.toString() ); config.set( JobContext.TASK_ID, taskAttemptId.getTaskID().toString() ); config.setBoolean( JobContext.TASK_ISMAP, isMapperOutput ); config.setInt( JobContext.TASK_PARTITION, taskAttemptId.getTaskID().getId() ); } }
private void validateJob(RunningJob job, MiniMRCluster mr) throws IOException { assertEquals(JobStatus.SUCCEEDED, job.getJobState()); long uses = job.getCounters().findCounter("jvm", "use").getValue(); assertTrue("maps = " + numMappers + ", jvms = " + uses, numMappers < uses); JobID jobId = job.getID(); for (int i = 0; i < numMappers; i++) { TaskAttemptID attemptId = new TaskAttemptID(new TaskID(jobId, TaskType.MAP, i), 0); TaskInProgress tip = mr.getJobTrackerRunner().getJobTracker().getTip( attemptId.getTaskID()); TaskStatus ts = mr.getJobTrackerRunner().getJobTracker().getTaskStatus( attemptId); validateAttempt(tip, attemptId, ts, i == taskWithCleanup); if (i == taskWithCleanup) { // validate second attempt of the task attemptId = new TaskAttemptID(new TaskID(jobId, TaskType.MAP, i), 1); ts = mr.getJobTrackerRunner().getJobTracker().getTaskStatus(attemptId); validateAttempt(tip, attemptId, ts, false); } } }
@Nonnull public static String getJobIdFromTaskId(@Nonnull String taskidStr) { if (!taskidStr.startsWith("task_")) {// workaround for Tez taskidStr = taskidStr.replace("task", "task_"); taskidStr = taskidStr.substring(0, taskidStr.lastIndexOf('_')); } TaskID taskId = TaskID.forName(taskidStr); JobID jobId = taskId.getJobID(); return jobId.toString(); }
/** * test deprecated methods of TaskID * @throws IOException */ @SuppressWarnings("deprecation") @Test (timeout=5000) public void testDepricatedMethods() throws IOException { JobID jid = new JobID(); TaskID test = new TaskID(jid, true, 1); assertEquals(test.getTaskType(), TaskType.MAP); test = new TaskID(jid, false, 1); assertEquals(test.getTaskType(), TaskType.REDUCE); test = new TaskID("001", 1, false, 1); assertEquals(test.getTaskType(), TaskType.REDUCE); test = new TaskID("001", 1, true, 1); assertEquals(test.getTaskType(), TaskType.MAP); ByteArrayOutputStream out = new ByteArrayOutputStream(); test.write(new DataOutputStream(out)); TaskID ti = TaskID.read(new DataInputStream(new ByteArrayInputStream(out .toByteArray()))); assertEquals(ti.toString(), test.toString()); assertEquals("task_001_0001_m_000002", TaskID.getTaskIDsPattern("001", 1, true, 2)); assertEquals("task_003_0001_m_000004", TaskID.getTaskIDsPattern("003", 1, TaskType.MAP, 4)); assertEquals("003_0001_m_000004", TaskID.getTaskIDsPatternWOPrefix("003", 1, TaskType.MAP, 4).toString()); }
/** * test TaskAttemptID */ @SuppressWarnings( "deprecation" ) @Test (timeout=5000) public void testTaskAttemptID (){ TaskAttemptID task = new TaskAttemptID("001",2,true,3,4); assertEquals("attempt_001_0002_m_000003_4", TaskAttemptID.getTaskAttemptIDsPattern("001", 2, true, 3, 4)); assertEquals("task_001_0002_m_000003", task.getTaskID().toString()); assertEquals("attempt_001_0001_r_000002_3",TaskAttemptID.getTaskAttemptIDsPattern("001", 1, TaskType.REDUCE, 2, 3)); assertEquals("001_0001_m_000001_2", TaskAttemptID.getTaskAttemptIDsPatternWOPrefix("001",1, TaskType.MAP, 1, 2).toString()); }
/** Create a local reduce input file name. * @param mapTaskId a map task id * @param reduceTaskId a reduce task id * @param size the size of the file */ public Path getInputFileForWrite(TaskID mapId, TaskAttemptID reduceTaskId, long size) throws IOException { // TODO *oom* should use a format here return lDirAlloc.getLocalPathForWrite(TaskTracker.getIntermediateOutputDir( jobId.toString(), reduceTaskId.toString()) + "/map_" + mapId.getId() + ".out", size, conf); }
public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { System.err.println(taskid); this.reporter = reporter; if (TaskAttemptID.forName(taskid).getTaskID().getId() == taskWithCleanup) { if (taskid.endsWith("_0")) { System.exit(-1); } } }
private Path getAttemptJobXml(JobConf conf, JobID jobId, TaskType taskType) throws IOException, LoginException { String taskid = new TaskAttemptID(new TaskID(jobId, taskType, 0), 0).toString(); return new LocalDirAllocator(MRConfig.LOCAL_DIR).getLocalPathToRead( TaskTracker.getTaskConfFile(UserGroupInformation.getCurrentUser() .getUserName(), jobId.toString(), taskid, false), conf); }
public static TaskID read(DataInput in) throws IOException { TaskID tipId = new TaskID(); tipId.readFields(in); return tipId; }