static synchronized String getOutputName(TaskAttemptContext context) { return context.getConfiguration().get("mapreduce.output.basename", "part") + "-" + NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId()); }
private void futureGet(Context context) throws IOException { try { future.get(); } catch (Exception e) { throw new IOException("Failed to build cube in mapper " + context.getTaskAttemptID().getTaskID().getId(), e); } } }
@Override public void run(Context ctx) throws IOException, InterruptedException { System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId()); latch.get("combineAwaitLatch").await(); combineExecCnt.incrementAndGet(); System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId()); } }
@Override public void run(Context ctx) throws IOException, InterruptedException { System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId()); latch.get("mapAwaitLatch").await(); mapExecCnt.incrementAndGet(); System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId()); } }
@Override public void run(Context ctx) throws IOException, InterruptedException { System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId()); latch.get("reduceAwaitLatch").await(); reduceExecCnt.incrementAndGet(); System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId()); } }
@Override public RecordReader<LongWritable, LongWritable> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { int taskId = context.getTaskAttemptID().getTaskID().getId(); int numMapTasks = context.getConfiguration().getInt(NUM_MAPS_KEY, NUM_MAPS); int numIterations = context.getConfiguration().getInt(NUM_IMPORT_ROUNDS_KEY, NUM_IMPORT_ROUNDS); int iteration = context.getConfiguration().getInt(ROUND_NUM_KEY, 0); taskId = taskId + iteration * numMapTasks; numMapTasks = numMapTasks * numIterations; long chainId = Math.abs(new Random().nextLong()); chainId = chainId - (chainId % numMapTasks) + taskId; // ensure that chainId is unique per task and across iterations LongWritable[] keys = new LongWritable[] {new LongWritable(chainId)}; return new FixedRecordReader<>(keys, keys); } }
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); cubeConfig = cube.getConfig(); baseCuboidId = cube.getCuboidScheduler().getBaseCuboidId(); baseCuboidRowCountInMappers = Lists.newLinkedList(); output = conf.get(BatchConstants.CFG_OUTPUT_PATH); samplingPercentage = Integer .parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT)); taskId = context.getTaskAttemptID().getTaskID().getId(); cuboidHLLMap = Maps.newHashMap(); }
cubeDesc = cube.getDescriptor(); taskId = context.getTaskAttemptID().getTaskID().getId();
byte valBytes[] = new byte[valLength]; int taskId = context.getTaskAttemptID().getTaskID().getId(); assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!"; Random random = new Random();
/** * Write random values to the writer assuming a table created using * {@link #FAMILIES} as column family descriptors */ private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, Cell> writer, TaskAttemptContext context, Set<byte[]> families, int numRows) throws IOException, InterruptedException { byte keyBytes[] = new byte[Bytes.SIZEOF_INT]; int valLength = 10; byte valBytes[] = new byte[valLength]; int taskId = context.getTaskAttemptID().getTaskID().getId(); assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!"; final byte [] qualifier = Bytes.toBytes("data"); Random random = new Random(); for (int i = 0; i < numRows; i++) { Bytes.putInt(keyBytes, 0, i); random.nextBytes(valBytes); ImmutableBytesWritable key = new ImmutableBytesWritable(keyBytes); for (byte[] family : families) { Cell kv = new KeyValue(keyBytes, family, qualifier, valBytes); writer.write(key, kv); } } }
byte valBytes[] = new byte[valLength]; int taskId = context.getTaskAttemptID().getTaskID().getId(); assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!";
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); mos = new MultipleOutputs(context); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); CubeDesc cubeDesc = cube.getDescriptor(); List<TblColRef> uhcColumns = cubeDesc.getAllUHCColumns(); int taskId = context.getTaskAttemptID().getTaskID().getId(); col = uhcColumns.get(taskId); logger.info("column name: " + col.getIdentity()); if (cube.getDescriptor().getShardByColumns().contains(col)) { //for ShardByColumns builder = DictionaryGenerator.newDictionaryBuilder(col.getType()); builder.init(null, 0, null); } else { //for GlobalDictionaryColumns String hdfsDir = conf.get(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR); DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype()); String builderClass = cubeDesc.getDictionaryBuilderClass(col); builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass); builder.init(dictionaryInfo, 0, hdfsDir); } }
private String getOutFileName(TaskAttemptContext context, String prefix) { TaskID taskId = context.getTaskAttemptID().getTaskID(); int partition = taskId.getId(); NumberFormat nf = NumberFormat.getInstance(); nf.setMinimumIntegerDigits(5); nf.setGroupingUsed(false); StringBuilder result = new StringBuilder(); result.append(prefix); result.append("-"); result.append(nf.format(partition)); return result.toString(); }
private void futureGet(Context context) throws IOException { try { future.get(); } catch (Exception e) { throw new IOException("Failed to build cube in mapper " + context.getTaskAttemptID().getTaskID().getId(), e); } } }
private Random createRandom(Context context) { long taskId = 0; if (context.getTaskAttemptID() != null) { // MRUnit returns null LOGGER.debug("context.getTaskAttemptID().getId(): {}", context.getTaskAttemptID().getId()); LOGGER.debug("context.getTaskAttemptID().getTaskID().getId(): {}", context.getTaskAttemptID().getTaskID().getId()); taskId = context.getTaskAttemptID().getTaskID().getId(); // taskId = 0, 1, ..., N } // create a good random seed, yet ensure deterministic PRNG sequence for easy reproducability return new Random(421439783L * (taskId + 1)); }
@Override protected void cleanup(Context context) throws IOException, InterruptedException { context.write(new IntWritable(context.getTaskAttemptID().getTaskID() .getId()), new VectorWritable(new DenseVector(mYtY.getData()))); } }
public static TezTaskID fromMRTaskId(org.apache.hadoop.mapreduce.TaskID taskid) { return TezTaskID.getInstance( TezVertexID.getInstance(fromMRJobId(taskid.getJobID()), (taskid.getTaskType() == TaskType.MAP ? 0 : 1) ), taskid.getId()); }
public static TaskId toYarn(org.apache.hadoop.mapreduce.TaskID id) { TaskId taskId = recordFactory.newRecordInstance(TaskId.class); taskId.setId(id.getId()); taskId.setTaskType(toYarn(id.getTaskType())); taskId.setJobId(toYarn(id.getJobID())); return taskId; }
public static TaskId toYarn(org.apache.hadoop.mapreduce.TaskID id) { TaskId taskId = recordFactory.newRecordInstance(TaskId.class); taskId.setId(id.getId()); taskId.setTaskType(toYarn(id.getTaskType())); taskId.setJobId(toYarn(id.getJobID())); return taskId; }