Utilities.getInputSummary(context, currWork, null).getLength();
Utilities.getInputSummary(context, currWork, null).getLength();
return Utilities.getInputSummary(context, mapWork, null); } finally { if (fs.exists(testTablePath)) {
inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null);
inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null);
ContentSummary inputSummary = Utilities.getInputSummary(pctx.getContext(), task.getWork().getMapWork(), null); long inputSize = inputSummary.getLength(); if (!indexHandler.checkQuerySize(inputSize, pctx.getConf())) {
for (ExecDriver mrtask : mrtasks) { try { ContentSummary inputSummary = Utilities.getInputSummary (ctx, mrtask.getWork().getMapWork(), p); int numReducers = getNumberOfReducers(mrtask.getWork(), conf);
for (ExecDriver mrtask : mrtasks) { try { ContentSummary inputSummary = Utilities.getInputSummary (ctx, mrtask.getWork().getMapWork(), p); int numReducers = getNumberOfReducers(mrtask.getWork(), conf);
} else { if (inputSummary == null) { inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null);
} else { if (inputSummary == null) { inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null);
Utilities.getInputSummary(context, currWork, null).getLength();
/** * Estimate the number of reducers needed for this job, based on job input, * and configuration parameters. * * @return the number of reducers. */ private int estimateNumberOfReducers() throws IOException { long bytesPerReducer = conf.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER); int maxReducers = conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS); if(inputSummary == null) { // compute the summary and stash it away inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work, null); } long totalInputFileSize = inputSummary.getLength(); LOG.info("BytesPerReducer=" + bytesPerReducer + " maxReducers=" + maxReducers + " totalInputFileSize=" + totalInputFileSize); int reducers = (int) ((totalInputFileSize + bytesPerReducer - 1) / bytesPerReducer); reducers = Math.max(1, reducers); reducers = Math.min(maxReducers, reducers); return reducers; }
inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work, null);
inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null);
for (ExecDriver mrtask: mrtasks) { try { ContentSummary inputSummary = Utilities.getInputSummary (ctx, (MapredWork)mrtask.getWork(), p); int numReducers = getNumberOfReducers(mrtask.getWork(), conf);
ContentSummary inputSummary = Utilities.getInputSummary(pctx.getContext(), task.getWork().getMapWork(), null); long inputSize = inputSummary.getLength(); if (!indexHandler.checkQuerySize(inputSize, pctx.getConf())) {
} else { if (inputSummary == null) { inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work.getMapWork(), null);
for (ExecDriver mrtask : mrtasks) { try { ContentSummary inputSummary = Utilities.getInputSummary (ctx, mrtask.getWork().getMapWork(), p); int numReducers = getNumberOfReducers(mrtask.getWork(), conf);