/** * Use this before submitting a TableReduce job. It will * appropriately set up the JobConf. * * @param table The output table. * @param reducer The reducer class to use. * @param job The current job to adjust. * @throws IOException When determining the region count fails. */ public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer, Job job) throws IOException { initTableReducerJob(table, reducer, job, null); }
/** * Use this before submitting a TableReduce job. It will * appropriately set up the JobConf. * * @param table The output table. * @param reducer The reducer class to use. * @param job The current job to adjust. * @param partitioner Partitioner to use. Pass <code>null</code> to use * default partitioner. * @throws IOException When determining the region count fails. */ public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer, Job job, Class partitioner) throws IOException { initTableReducerJob(table, reducer, job, partitioner, null, null, null); }
Class partitioner, String quorumAddress, String serverClass, String serverImpl) throws IOException { initTableReducerJob(table, reducer, job, partitioner, quorumAddress, serverClass, serverImpl, true);
/** * Job configuration. */ public static Job configureJob(Configuration conf, String [] args) throws IOException { Path inputPath = new Path(args[0]); String tableName = args[1]; Job job = new Job(conf, NAME + "_" + tableName); job.setJarByClass(Uploader.class); FileInputFormat.setInputPaths(job, inputPath); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(Uploader.class); // No reducers. Just write straight to table. Call initTableReducerJob // because it sets up the TableOutputFormat. TableMapReduceUtil.initTableReducerJob(tableName, null, job); job.setNumReduceTasks(0); return job; }
TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster, null, null);
TableMapReduceUtil.initTableReducerJob(tableName.getNameAsString(), null, job); job.setNumReduceTasks(0);
TableMapReduceUtil.initTableReducerJob(tableName.getNameAsString(), null, job);
TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress, null, null);
TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job, null, null, null, null); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
MultithreadedTableMapper.setMapperClass(job, ProcessContentsMapper.class); MultithreadedTableMapper.setNumberOfThreads(job, NUMBER_OF_THREADS); TableMapReduceUtil.initTableReducerJob( table.getName().getNameAsString(), IdentityTableReducer.class, job);
ProcessContentsMapper.class, ImmutableBytesWritable.class, Put.class, job); TableMapReduceUtil.initTableReducerJob( table.getName().getNameAsString(), IdentityTableReducer.class, job);
public static void setupHBaseJob(Job job, String sourceTable, String cf) throws IOException { Scan scan = new Scan(); if(cf != null) { scan.addFamily(Bytes.toBytes(cf)); } scan.setCaching(500); // 1 is the default in Scan, which will be bad for MapReduce jobs scan.setCacheBlocks(false); // don't set to true for MR jobs // set other scan attrs TableMapReduceUtil.initTableMapperJob( sourceTable, // input table scan, // Scan instance to control CF and attribute selection PrunerMapper.class, // mapper class null, // mapper output key null, // mapper output value job); TableMapReduceUtil.initTableReducerJob( sourceTable, // output table null, // reducer class job); }
TableMapReduceUtil.initTableMapperJob(input, scan, ParseMapper.class, ImmutableBytesWritable.class, Put.class, job); TableMapReduceUtil.initTableReducerJob(output, IdentityTableReducer.class, job); /*[*/job.setNumReduceTasks(0);/*]*/
TableMapReduceUtil.initTableMapperJob(input, scan, ParseMapper.class, // co ParseJson-3-SetMap Setup map phase details using the utility method. ImmutableBytesWritable.class, Put.class, job); TableMapReduceUtil.initTableReducerJob(output, // co ParseJson-4-SetReduce Configure an identity reducer to store the parsed data. IdentityTableReducer.class, job);
/** * Use this before submitting a TableReduce job. It will * appropriately set up the JobConf. * * @param table The output table. * @param reducer The reducer class to use. * @param job The current job to adjust. * @throws IOException When determining the region count fails. */ public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer, Job job) throws IOException { initTableReducerJob(table, reducer, job, null); }
/** * Use this before submitting a TableReduce job. It will * appropriately set up the JobConf. * * @param table The output table. * @param reducer The reducer class to use. * @param job The current job to adjust. * @throws IOException When determining the region count fails. */ public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer, Job job) throws IOException { initTableReducerJob(table, reducer, job, null); }
/** * Use this before submitting a TableReduce job. It will * appropriately set up the JobConf. * * @param table The output table. * @param reducer The reducer class to use. * @param job The current job to adjust. * @throws IOException When determining the region count fails. */ public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer, Job job) throws IOException { initTableReducerJob(table, reducer, job, null); }
/** * Use this before submitting a TableReduce job. It will * appropriately set up the JobConf. * * @param table The output table. * @param reducer The reducer class to use. * @param job The current job to adjust. * @throws IOException When determining the region count fails. */ public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer, Job job) throws IOException { initTableReducerJob(table, reducer, job, null); }
public static void setOutputHBaseTable(Job job, String outTable) throws IOException { boolean addDependencyJar = job.getConfiguration().getBoolean(HadoopVariantStorageEngine.MAPREDUCE_ADD_DEPENDENCY_JARS, true); LOGGER.info("Use table {} as output", outTable); TableMapReduceUtil.initTableReducerJob( outTable, // output table null, // reducer class job, null, null, null, null, addDependencyJar); }
public int run(String[] args) throws Exception { // setup the Job Job job = Job.getInstance(getConf(), getClass().getSimpleName()); job.setJarByClass(getClass()); FileOutputFormat.setOutputPath(job, new Path(args[0])); // instantiate and initialize the scan object Scan scan = new Scan(); scan.setMaxVersions(); scan.addFamily(COLUMN_FAMILY_PRICE); // initialize the mapper and reducer TableMapReduceUtil.initTableMapperJob(TABLE_NAME, scan, StockMapper.class, Text.class, LongWritable.class, job); TableMapReduceUtil.initTableReducerJob(TABLE_NAME, StockReducer.class, job); // launch the job and block waiting return job.waitForCompletion(true) ? 0 : 1; }