/** * Create the heart beat object thread set it to daemon priority and start the thread. When the count in * {@link #threadsNeedingHeartBeat} is positive, the heart beat will be issued on the progress object ever 5 seconds. */ public HeartBeater(Progressable progress, long waitTimeMs) { setDaemon(true); this.progress = progress; this.waitTimeMs = waitTimeMs; LOG.info("Heart beat reporting class is " + progress.getClass().getName()); start(); }
@Override public void close(TaskAttemptContext ctx) throws IOException, InterruptedException { LOG.info("Written " + rows + " total rows into " + permPool.entrySet().size() + " partitions."); FileSystem fs = FileSystem.get(ctx.getConfiguration()); try { if (ctx != null) { heartBeater.setProgress(ctx); } outputFormat.close(); for (Map.Entry<Integer, Path> entry : permPool.entrySet()) { Path localFile = tempPool.get(entry.getKey()); if (FileSystem.getLocal(ctx.getConfiguration()).exists(localFile)) { LOG.info("Commiting local file " + localFile + " to final destination " + entry.getValue()); // Hadoop - completeLocalOutput() fs.completeLocalOutput(entry.getValue(), tempPool.get(entry.getKey())); } else { LOG.info("Strange. Local file " + localFile + " is empty. No data to upload to " + entry.getValue()); } } } finally { // in any case, destroy the HeartBeater heartBeater.cancelHeartBeat(); } }
heartBeater = new HeartBeater(context, waitTimeHeartBeater); heartBeater.needHeartBeat(); conf = context.getConfiguration(); this.context = context;