@Override public void close(TaskAttemptContext ctx) throws IOException, InterruptedException { LOG.info("Written " + rows + " total rows into " + permPool.entrySet().size() + " partitions."); FileSystem fs = FileSystem.get(ctx.getConfiguration()); try { if (ctx != null) { heartBeater.setProgress(ctx); } outputFormat.close(); for (Map.Entry<Integer, Path> entry : permPool.entrySet()) { Path localFile = tempPool.get(entry.getKey()); if (FileSystem.getLocal(ctx.getConfiguration()).exists(localFile)) { LOG.info("Commiting local file " + localFile + " to final destination " + entry.getValue()); // Hadoop - completeLocalOutput() fs.completeLocalOutput(entry.getValue(), tempPool.get(entry.getKey())); } else { LOG.info("Strange. Local file " + localFile + " is empty. No data to upload to " + entry.getValue()); } } } finally { // in any case, destroy the HeartBeater heartBeater.cancelHeartBeat(); } }