@Override protected void reduce(YearTemperature key, Iterable<Text> vals0, Context context) throws IOException, InterruptedException { X.println("___ : " + context.getTaskAttemptID() + " --> " + key); Set<UUID> ids = new HashSet<>(); for (Text val : vals0) assertTrue(ids.add(UUID.fromString(val.toString()))); for (Text val : vals0) assertTrue(ids.remove(UUID.fromString(val.toString()))); assertTrue(ids.isEmpty()); assertTrue(key.year > lastYear); lastYear = key.year; for (Text val : vals0) assertTrue(HadoopGroupingTestState.values().remove(UUID.fromString(val.toString()))); } }
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); cubeConfig = cube.getConfig(); baseCuboidId = cube.getCuboidScheduler().getBaseCuboidId(); baseCuboidRowCountInMappers = Lists.newLinkedList(); output = conf.get(BatchConstants.CFG_OUTPUT_PATH); samplingPercentage = Integer .parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT)); taskId = context.getTaskAttemptID().getTaskID().getId(); cuboidHLLMap = Maps.newHashMap(); }
@Override public void run(Context ctx) throws IOException, InterruptedException { System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId()); latch.get("reduceAwaitLatch").await(); reduceExecCnt.incrementAndGet(); System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId()); } }
public static void addReducerContext(Reducer<?,?,?,?>.Context context) { TaskID taskID = context.getTaskAttemptID().getTaskID(); contextMap.put(taskID, context); }
public static void addReducerContext(@SuppressWarnings("rawtypes") Reducer.Context context) { TaskID taskID = context.getTaskAttemptID().getTaskID(); if(contextMap.get(taskID) == null) { contextMap.put(taskID, context); } }
public static void addReducerContext(Reducer<?,?,?,?>.Context context) { TaskID taskID = context.getTaskAttemptID().getTaskID(); contextMap.put(taskID, context); }
public void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { String id = context.getTaskAttemptID().toString(); // Reducer 0 does not output anything if (!id.endsWith("0_0")) { for (Text val: values) { context.write(key, val); } } } }
public static void addReducerContext(Reducer<?,?,?,?>.Context context) { TaskID taskID = context.getTaskAttemptID().getTaskID(); contextMap.put(taskID, context); }
private void verifyPartitionAssignment(Context context) { if ("true".equals(System.getProperty("verifyPartitionAssignment", "true"))) { String partitionStr = context.getConfiguration().get("mapred.task.partition"); if (partitionStr == null) { partitionStr = context.getConfiguration().get("mapreduce.task.partition"); } int partition = Integer.parseInt(partitionStr); int taskId = context.getTaskAttemptID().getTaskID().getId(); Preconditions.checkArgument(partition == taskId, "mapred.task.partition: " + partition + " not equal to reducer taskId: " + taskId); } }
private void verifyPartitionAssignment(Context context) { if ("true".equals(System.getProperty("verifyPartitionAssignment", "true"))) { String partitionStr = context.getConfiguration().get("mapred.task.partition"); if (partitionStr == null) { partitionStr = context.getConfiguration().get("mapreduce.task.partition"); } int partition = Integer.parseInt(partitionStr); int taskId = context.getTaskAttemptID().getTaskID().getId(); Preconditions.checkArgument(partition == taskId, "mapred.task.partition: " + partition + " not equal to reducer taskId: " + taskId); } }
public static void addReducerContext(Reducer<?,?,?,?>.Context context) { TaskID taskID = context.getTaskAttemptID().getTaskID(); contextMap.put(taskID, context); }
public void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { String id = context.getTaskAttemptID().toString(); // Reducer 0 does not output anything if (!id.endsWith("0_0")) { for (Text val: values) { context.write(key, val); } } } }
@Override protected void cleanup(Context context) throws IOException, InterruptedException { final String output = context.getConfiguration().getStrings(TIMEINDEX_LOCATION_PROP)[0]; final Path indexOut = new Path(output + "/" + context.getTaskAttemptID()); System.out.println("Writing time index to: " + indexOut); System.out.println("Timemap contains: " + this.timeMap.size()); CountTweetsInTimeperiod.writeTimeIndex(this.timeMap, indexOut); } }
@Override protected void cleanup(Context context) throws IOException, InterruptedException { final String output = context.getConfiguration().getStrings(TIMEINDEX_LOCATION_PROP)[0]; final Path indexOut = new Path(output + "/" + context.getTaskAttemptID()); System.out.println("Writing time index to: " + indexOut); System.out.println("Timemap contains: " + this.timeMap.size()); CountTweetsInTimeperiod.writeTimeIndex(this.timeMap, indexOut); } }
public static void addReducerContext(@SuppressWarnings("rawtypes") Reducer.Context context) { TaskID taskID = context.getTaskAttemptID().getTaskID(); if(contextMap.get(taskID) == null) { contextMap.put(taskID, context); } }
outputFS, segmentTemplate, context.getTaskAttemptID(), config.DATA_SEGMENT_PUSHER ),
cubeDesc = cube.getDescriptor(); taskId = context.getTaskAttemptID().getTaskID().getId();
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); mos = new MultipleOutputs(context); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); CubeDesc cubeDesc = cube.getDescriptor(); List<TblColRef> uhcColumns = cubeDesc.getAllUHCColumns(); int taskId = context.getTaskAttemptID().getTaskID().getId(); col = uhcColumns.get(taskId); logger.info("column name: " + col.getIdentity()); if (cube.getDescriptor().getShardByColumns().contains(col)) { //for ShardByColumns builder = DictionaryGenerator.newDictionaryBuilder(col.getType()); builder.init(null, 0, null); } else { //for GlobalDictionaryColumns String hdfsDir = conf.get(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR); DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype()); String builderClass = cubeDesc.getDictionaryBuilderClass(col); builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass); builder.init(dictionaryInfo, 0, hdfsDir); } }
@Override public void run(Context ctx) throws IOException, InterruptedException { System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId()); latch.get("combineAwaitLatch").await(); combineExecCnt.incrementAndGet(); System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId()); } }
/** {@inheritDoc} */ @Override protected void setup(Context ctx) throws IOException, InterruptedException { X.println("___ Reducer: " + ctx.getTaskAttemptID()); String taskId = ctx.getTaskAttemptID().toString(); String workDir = FileSystem.getLocal(ctx.getConfiguration()).getWorkingDirectory().toString(); assertNull(taskWorkDirs.put(workDir, taskId)); }