Codota Logo
Mapper$Context.getNumReduceTasks
Code IndexAdd Codota to your IDE (free)

How to use
getNumReduceTasks
method
in
org.apache.hadoop.mapreduce.Mapper$Context

Best Java code snippets using org.apache.hadoop.mapreduce.Mapper$Context.getNumReduceTasks (Showing top 16 results out of 315)

  • Common ways to obtain Mapper$Context
private void myMethod () {
Mapper$Context m =
  • Codota IconWrappedMapper wrappedMapper;new Context(wrappedMapper)
  • Codota IconMockito mockito;mockito.mock(Mapper.Context.class)
  • Smart code suggestions by Codota
}
origin: apache/hbase

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: apache/incubator-gobblin

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}
origin: confluentinc/camus

@Override
protected void setup(Context context) throws IOException, InterruptedException {
 // For class path debugging
 LOG.info("classpath: " + System.getProperty("java.class.path"));
 ClassLoader loader = AvroKeyMapper.class.getClassLoader();
 LOG.info("com.linkedin.events.fixed_16: " + loader.getResource("com/linkedin/events/fixed_16.class"));
 LOG.info("org.apache.avro.Schema: " + loader.getResource("org/apache/avro/Schema.class"));
 // We need to use RelaxedAvroSerialization to skip the illegal field names
 // during schema parsing.
 //keySchema = AvroJob.getMapOutputKeySchema(context.getConfiguration());
 keySchema = RelaxedAvroSerialization.getKeyWriterSchema(context.getConfiguration());
 outValue = new AvroValue<GenericRecord>();
 outKey = new AvroKey<GenericRecord>();
 outKey.datum(new GenericData.Record(keySchema));
 if (context.getNumReduceTasks() == 0)
  mapOnly = true;
}
origin: aseldawy/spatialhadoop2

@Override
protected void setup(
  Mapper<Rectangle, Iterable<S>, IntWritable, OGCJTSShape>.Context context)
    throws IOException, InterruptedException {
 super.setup(context);
 columnBoundaries = SpatialSite.getReduceSpace(context.getConfiguration());
 if (columnBoundaries == null)
  key.set(new Random().nextInt(context.getNumReduceTasks()));
}

origin: aseldawy/spatialhadoop2

@Override
protected void setup(Context context)
  throws IOException, InterruptedException {
 // TODO Auto-generated method stub
 super.setup(context);
 Configuration conf = context.getConfiguration();
 this.imageWidth = conf.getInt("width", 1000);
 this.imageHeight = conf.getInt("height", 1000);
 this.inputMBR = (Rectangle) OperationsParams.getShape(conf, InputMBR);
 this.outputValue = new IntWritable(0);
 this.plotter = Plotter.getPlotter(conf);
 this.smooth = plotter.isSmooth();
 this.numReducers = Math.max(1, context.getNumReduceTasks());
 this.random = new Random();
}
origin: com.linkedin.gobblin/gobblin-compaction

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}
origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

@Override
protected void setup(Context context) throws IOException {
 // check if the map task attempt progress is 0
 assertEquals("Invalid progress in map setup", 
        0.0f, context.getProgress(), 0f);
 
 // define the progress boundaries
 if (context.getNumReduceTasks() == 0) {
  progressRange = 1f;
 } else {
  progressRange = 0.667f;
 }
}

origin: ch.cern.hadoop/hadoop-gridmix

@Override
public void cleanup(Context context)
throws IOException, InterruptedException {
 final int nReds = context.getNumReduceTasks();
 if (nReds > 0) {
  final SleepSplit split = (SleepSplit) context.getInputSplit();
  int id = split.getId();
  final int nMaps = split.getNumMaps();
  //This is a hack to pass the sleep duration via Gridmix key
  //TODO: We need to come up with better solution for this.
  
  final GridmixKey key = new GridmixKey(GridmixKey.REDUCE_SPEC, 0, 0L);
  for (int i = id, idx = 0; i < nReds; i += nMaps) {
   key.setPartition(i);
   key.setReduceOutputBytes(split.getReduceDurations(idx++));
   id += nReds;
   context.write(key, NullWritable.get());
  }
 }
}
origin: com.aliyun.hbase/alihbase-mapreduce

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   KeyValueWritableComparable[] startKeyWraps =
     new KeyValueWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: org.apache.hbase/hbase-mapreduce

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   KeyValueWritableComparable[] startKeyWraps =
     new KeyValueWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: com.aliyun.hbase/alihbase-mapreduce

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: org.apache.hbase/hbase-mapreduce

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: io.github.repir/RepIRTools

  public void generateKey() {
    int numReduceTasks = context.getNumReduceTasks();
    set( RandomTools.getInt(numReduceTasks) );
  }
}
origin: ch.cern.hadoop/hadoop-gridmix

if (context.getNumReduceTasks() > 0 
  && context.getCounter(TaskCounter.SPILLED_RECORDS).getValue() == 0) {
 LOG.info("Boosting the map phase progress.");
origin: ch.cern.hadoop/hadoop-gridmix

final int nReduces = ctxt.getNumReduceTasks();
if (nReduces > 0) {
origin: org.apache.gobblin/gobblin-compaction

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}
org.apache.hadoop.mapreduceMapper$ContextgetNumReduceTasks

Popular methods of Mapper$Context

  • write
  • getConfiguration
  • getCounter
  • getInputSplit
  • progress
  • setStatus
  • getTaskAttemptID
  • nextKeyValue
  • getCurrentValue
  • getCurrentKey
  • getJobID
  • getInputFormatClass
  • getJobID,
  • getInputFormatClass,
  • getLocalCacheFiles,
  • getOutputCommitter,
  • getCredentials,
  • getLocalCacheArchives,
  • getStatus,
  • getCacheArchives,
  • getCacheFiles

Popular in Java

  • Making http post requests using okhttp
  • getExternalFilesDir (Context)
  • setScale (BigDecimal)
    Returns a BigDecimal whose scale is the specified value, and whose value is numerically equal to thi
  • scheduleAtFixedRate (ScheduledExecutorService)
    Creates and executes a periodic action that becomes enabled first after the given initial delay, and
  • Kernel (java.awt.image)
  • URL (java.net)
    A Uniform Resource Locator that identifies the location of an Internet resource as specified by RFC
  • Timestamp (java.sql)
    A Java representation of the SQL TIMESTAMP type. It provides the capability of representing the SQL
  • TreeMap (java.util)
    A Red-Black tree based NavigableMap implementation. The map is sorted according to the Comparable of
  • StringUtils (org.apache.commons.lang)
    Operations on java.lang.String that arenull safe. * IsEmpty/IsBlank - checks if a String contains
  • Scheduler (org.quartz)
    This is the main interface of a Quartz Scheduler. A Scheduler maintains a registery of org.quartz
Codota Logo
  • Products

    Search for Java codeSearch for JavaScript codeEnterprise
  • IDE Plugins

    IntelliJ IDEAWebStormAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogCodota Academy Plugin user guide Terms of usePrivacy policyJava Code IndexJavascript Code Index
Get Codota for your IDE now