public LabeledGroup(Map<String, Object> pool, int index) { super(pool, index, GroupType.LABELED); drillbitLabelExpr = (String) pool.get(DRILLBIT_LABEL); if (drillbitLabelExpr == null) { Log.warn("Labeled pool is missing the drillbit label expression (" + DRILLBIT_LABEL + "), will treat pool as basic."); } }
private Path createDownloadDir(String workDir, String fileUrl) { Path downloadDirPath = Paths.get(workDir, DOWNLOAD_DIR); File downloadDirFile = downloadDirPath.toFile(); try { Log.info(String.format("Creating download dir %s", downloadDirFile.toPath().toString())); FileUtils.forceMkdir(downloadDirFile); } catch (IOException e) { throw new RuntimeException(String .format("Unable to create download location for archive: %s at %s", fileUrl, downloadDirPath.toString())); } Log.info(String.format("Created download dir %s", downloadDirFile.toPath().toString())); return downloadDirPath; }
public static void warn(String msg) { if (!initialized()) return; __log.warn(msg,null,null); }
public static void warn(Throwable th) { if (!initialized()) return; __log.warn(EXCEPTION,th); unwind(th); }
public static void debug(Throwable th) { if (!isDebugEnabled()) return; __log.debug(EXCEPTION,th); unwind(th); }
public static void debug(String msg,Object arg) { if (!initialized()) return; __log.debug(msg,arg,null); }
public static void info(String msg) { if (!initialized()) return; __log.info(msg,null,null); }
public static boolean isDebugEnabled() { if (!initialized()) return false; return __log.isDebugEnabled(); }
/** Obtain a named Logger. * Obtain a named Logger or the default Logger if null is passed. */ public static Logger getLogger(String name) { if (!initialized()) return null; if (name==null) return __log; return __log.getLogger(name); }
private void downloadFile(String fileUrl, Path destination) { if (destination.toFile().exists()) { Log.info(String.format("Skipping download for %s at %s because destination already exists", fileUrl, destination.toString())); return; } try { URL archiveUrl = new URL(fileUrl); ReadableByteChannel rbc = Channels.newChannel(archiveUrl.openStream()); FileOutputStream fos = new FileOutputStream(String.valueOf(destination)); Log.info(String.format("Downloading %s at %s", fileUrl, destination.toString())); fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); Log.info(String.format("Download complete for %s at %s", fileUrl, destination.toString())); } catch (IOException e) { e.printStackTrace(); } } }
public static void warn(String msg,Object arg) { if (!initialized()) return; __log.warn(msg,arg,null); }
public static void warn(String msg, Throwable th) { if (!initialized()) return; __log.warn(msg,th); unwind(th); }
public static void debug(String msg) { if (!initialized()) return; __log.debug(msg,null,null); }
public static void info(String msg,Object arg0, Object arg1) { if (!initialized()) return; __log.info(msg,arg0,arg1); }
private void moveTmpPathToOutputPath() throws IOException { Retryer<Void> retryer = RetryerFactory.newInstance(this.retrierConfig); LOG.info(String.format("Moving %s to %s", this.dataset.outputTmpPath(), this.dataset.outputPath())); this.fs.delete(this.dataset.outputPath(), true); if (this.isRetryEnabled) { try { retryer.call(() -> { if (fs.exists(this.dataset.outputPath())) { throw new IOException("Path " + this.dataset.outputPath() + " exists however it should not. Will wait more."); } return null; }); } catch (Exception e) { throw new IOException(e); } } WriterUtils.mkdirsWithRecursivePermissionWithRetry(MRCompactorJobRunner.this.fs, this.dataset.outputPath().getParent(), this.perm, this.retrierConfig); Log.info("Moving from fs: ("+MRCompactorJobRunner.this.tmpFs.getUri()+") path: "+ this.dataset.outputTmpPath() + " to "+ "fs: ("+ FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()).getUri()+") output path: " + this.dataset.outputPath()); HadoopUtils.movePath (MRCompactorJobRunner.this.tmpFs, this.dataset.outputTmpPath(), FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()), this.dataset.outputPath(), false, this.fs.getConf()) ; }
public static void warn(String msg,Object arg0, Object arg1) { if (!initialized()) return; __log.warn(msg,arg0,arg1); }
public static void debug(String msg,Object arg0, Object arg1) { if (!initialized()) return; __log.debug(msg,arg0,arg1); }
public static void info(String msg,Object arg) { if (!initialized()) return; __log.info(msg,arg,null); }
Log.info("Tmp fs uri:"+this.tmpFsUri); if (this.tmpFsUri != null) { this.tmpFs = FileSystem.get(new URI(this.tmpFsUri), new Configuration());