org.apache.hadoop.mapreduce.Mapper$Context.getNumReduceTasks java code examples

Common ways to obtain Mapper$Context

private void myMethod () {
     Mapper$Context m =WrappedMapper wrappedMapper;new Context(wrappedMapper)
Mockito mockito;mockito.mock(Mapper.Context.class)
Smart code suggestions by Codota
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}

@Override
protected void setup(Context context) throws IOException, InterruptedException {
 // For class path debugging
 LOG.info("classpath: " + System.getProperty("java.class.path"));
 ClassLoader loader = AvroKeyMapper.class.getClassLoader();
 LOG.info("com.linkedin.events.fixed_16: " + loader.getResource("com/linkedin/events/fixed_16.class"));
 LOG.info("org.apache.avro.Schema: " + loader.getResource("org/apache/avro/Schema.class"));
 // We need to use RelaxedAvroSerialization to skip the illegal field names
 // during schema parsing.
 //keySchema = AvroJob.getMapOutputKeySchema(context.getConfiguration());
 keySchema = RelaxedAvroSerialization.getKeyWriterSchema(context.getConfiguration());
 outValue = new AvroValue<GenericRecord>();
 outKey = new AvroKey<GenericRecord>();
 outKey.datum(new GenericData.Record(keySchema));
 if (context.getNumReduceTasks() == 0)
  mapOnly = true;
}

@Override
protected void setup(
  Mapper<Rectangle, Iterable<S>, IntWritable, OGCJTSShape>.Context context)
    throws IOException, InterruptedException {
 super.setup(context);
 columnBoundaries = SpatialSite.getReduceSpace(context.getConfiguration());
 if (columnBoundaries == null)
  key.set(new Random().nextInt(context.getNumReduceTasks()));
}

@Override
protected void setup(Context context)
  throws IOException, InterruptedException {
 // TODO Auto-generated method stub
 super.setup(context);
 Configuration conf = context.getConfiguration();
 this.imageWidth = conf.getInt("width", 1000);
 this.imageHeight = conf.getInt("height", 1000);
 this.inputMBR = (Rectangle) OperationsParams.getShape(conf, InputMBR);
 this.outputValue = new IntWritable(0);
 this.plotter = Plotter.getPlotter(conf);
 this.smooth = plotter.isSmooth();
 this.numReducers = Math.max(1, context.getNumReduceTasks());
 this.random = new Random();
}

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}

@Override
protected void setup(Context context) throws IOException {
 // check if the map task attempt progress is 0
 assertEquals("Invalid progress in map setup", 
        0.0f, context.getProgress(), 0f);
 
 // define the progress boundaries
 if (context.getNumReduceTasks() == 0) {
  progressRange = 1f;
 } else {
  progressRange = 0.667f;
 }
}

@Override
public void cleanup(Context context)
throws IOException, InterruptedException {
 final int nReds = context.getNumReduceTasks();
 if (nReds > 0) {
  final SleepSplit split = (SleepSplit) context.getInputSplit();
  int id = split.getId();
  final int nMaps = split.getNumMaps();
  //This is a hack to pass the sleep duration via Gridmix key
  //TODO: We need to come up with better solution for this.
  
  final GridmixKey key = new GridmixKey(GridmixKey.REDUCE_SPEC, 0, 0L);
  for (int i = id, idx = 0; i < nReds; i += nMaps) {
   key.setPartition(i);
   key.setReduceOutputBytes(split.getReduceDurations(idx++));
   id += nReds;
   context.write(key, NullWritable.get());
  }
 }
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   KeyValueWritableComparable[] startKeyWraps =
     new KeyValueWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   KeyValueWritableComparable[] startKeyWraps =
     new KeyValueWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

  public void generateKey() {
    int numReduceTasks = context.getNumReduceTasks();
    set( RandomTools.getInt(numReduceTasks) );
  }
}

if (context.getNumReduceTasks() > 0 
  && context.getCounter(TaskCounter.SPILLED_RECORDS).getValue() == 0) {
 LOG.info("Boosting the map phase progress.");

final int nReduces = ctxt.getNumReduceTasks();
if (nReduces > 0) {

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}

Best Java code snippets using org.apache.hadoop.mapreduce.Mapper$Context.getNumReduceTasks (Showing top 16 results out of 315)