@Override protected void closeRecordProcessor() { recordHandler.close(); } }
@Override protected boolean processingDone() { return recordHandler.getDone(); }
@Override protected void processNextRecord(Tuple2<BytesWritable, BytesWritable> inputRecord) throws IOException { recordHandler.processRow(inputRecord._1(), inputRecord._2()); }
@SuppressWarnings("unchecked") @Override public Iterator<Tuple2<HiveKey, BytesWritable>> call(Iterator<Tuple2<BytesWritable, BytesWritable>> it) throws Exception { initJobConf(); SparkRecordHandler mapRecordHandler; // need different record handler for MergeFileWork if (MergeFileMapper.class.getName().equals(jobConf.get(Utilities.MAPRED_MAPPER_CLASS))) { mapRecordHandler = new SparkMergeFileRecordHandler(); } else { mapRecordHandler = new SparkMapRecordHandler(); } HiveMapFunctionResultList result = new HiveMapFunctionResultList(it, mapRecordHandler); mapRecordHandler.init(jobConf, result, sparkReporter); return result; }
/** * Logger processed row number and used memory info. */ protected void logMemoryInfo() { rowNumber++; if (rowNumber == nextLogThreshold) { long usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed(); LOG.info("processing " + rowNumber + " rows: used memory = " + usedMemory); nextLogThreshold = getNextLogThreshold(rowNumber); } }
public void close() { memoryAndRowLogExecutor.shutdown(); memoryAndRowLogFuture.cancel(false); try { if (!memoryAndRowLogExecutor.awaitTermination(5, TimeUnit.SECONDS)) { memoryAndRowLogExecutor.shutdownNow(); } } catch (InterruptedException e) { memoryAndRowLogExecutor.shutdownNow(); Thread.currentThread().interrupt(); } if (LOG.isInfoEnabled()) { logMemoryInfo(); } }
@SuppressWarnings("unchecked") @Override public Iterator<Tuple2<HiveKey, BytesWritable>> call(Iterator<Tuple2<BytesWritable, BytesWritable>> it) throws Exception { initJobConf(); SparkRecordHandler mapRecordHandler; // need different record handler for MergeFileWork if (MergeFileMapper.class.getName().equals(jobConf.get(Utilities.MAPRED_MAPPER_CLASS))) { mapRecordHandler = new SparkMergeFileRecordHandler(); } else { mapRecordHandler = new SparkMapRecordHandler(); } HiveMapFunctionResultList result = new HiveMapFunctionResultList(it, mapRecordHandler); mapRecordHandler.init(jobConf, result, sparkReporter); return result; }
/** * Log processed row number and used memory info. */ protected void logMemoryInfo() { rowNumber++; if (rowNumber == nextLogThreshold) { long usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed(); LOG.info("processing " + rowNumber + " rows: used memory = " + usedMemory); nextLogThreshold = getNextLogThreshold(rowNumber); } }
@SuppressWarnings("unchecked") @Override public <K, V> void init(JobConf job, OutputCollector<K, V> output, Reporter reporter) throws Exception { super.init(job, output, reporter);
@Override protected void closeRecordProcessor() { recordHandler.close(); } }
@Override protected void processNextRecord(Tuple2<BytesWritable, BytesWritable> inputRecord) throws IOException { recordHandler.processRow(inputRecord._1(), inputRecord._2()); }
@Override protected boolean processingDone() { return recordHandler.getDone(); }
@SuppressWarnings("unchecked") @Override public <K, V> void init(JobConf job, OutputCollector<K, V> output, Reporter reporter) throws Exception { super.init(job, output, reporter);
@Override public void close() { super.close(); LOG.info("Closing Merge Operator " + mergeOp.getName()); try { mergeOp.closeOp(abort); } catch (HiveException e) { throw new RuntimeException(e); } }
@Override protected void processNextRecord(Tuple2<BytesWritable, BytesWritable> inputRecord) throws IOException { recordHandler.processRow(inputRecord._1(), inputRecord._2()); }
@Override protected boolean processingDone() { return recordHandler.getDone(); }
public void init(JobConf job, OutputCollector output, Reporter reporter) throws Exception { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_INIT_OPERATORS); super.init(job, output, reporter);
@Override public void close() { super.close();
public void init(JobConf job, OutputCollector output, Reporter reporter) throws Exception { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_INIT_OPERATORS); super.init(job, output, reporter);
@Override public void close() { super.close();