private Path createDownloadDir(String workDir, String fileUrl) { Path downloadDirPath = Paths.get(workDir, DOWNLOAD_DIR); File downloadDirFile = downloadDirPath.toFile(); try { Log.info(String.format("Creating download dir %s", downloadDirFile.toPath().toString())); FileUtils.forceMkdir(downloadDirFile); } catch (IOException e) { throw new RuntimeException(String .format("Unable to create download location for archive: %s at %s", fileUrl, downloadDirPath.toString())); } Log.info(String.format("Created download dir %s", downloadDirFile.toPath().toString())); return downloadDirPath; }
private void downloadFile(String fileUrl, Path destination) { if (destination.toFile().exists()) { Log.info(String.format("Skipping download for %s at %s because destination already exists", fileUrl, destination.toString())); return; } try { URL archiveUrl = new URL(fileUrl); ReadableByteChannel rbc = Channels.newChannel(archiveUrl.openStream()); FileOutputStream fos = new FileOutputStream(String.valueOf(destination)); Log.info(String.format("Downloading %s at %s", fileUrl, destination.toString())); fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); Log.info(String.format("Download complete for %s at %s", fileUrl, destination.toString())); } catch (IOException e) { e.printStackTrace(); } } }
private void moveTmpPathToOutputPath() throws IOException { Retryer<Void> retryer = RetryerFactory.newInstance(this.retrierConfig); LOG.info(String.format("Moving %s to %s", this.dataset.outputTmpPath(), this.dataset.outputPath())); this.fs.delete(this.dataset.outputPath(), true); if (this.isRetryEnabled) { try { retryer.call(() -> { if (fs.exists(this.dataset.outputPath())) { throw new IOException("Path " + this.dataset.outputPath() + " exists however it should not. Will wait more."); } return null; }); } catch (Exception e) { throw new IOException(e); } } WriterUtils.mkdirsWithRecursivePermissionWithRetry(MRCompactorJobRunner.this.fs, this.dataset.outputPath().getParent(), this.perm, this.retrierConfig); Log.info("Moving from fs: ("+MRCompactorJobRunner.this.tmpFs.getUri()+") path: "+ this.dataset.outputTmpPath() + " to "+ "fs: ("+ FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()).getUri()+") output path: " + this.dataset.outputPath()); HadoopUtils.movePath (MRCompactorJobRunner.this.tmpFs, this.dataset.outputTmpPath(), FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()), this.dataset.outputPath(), false, this.fs.getConf()) ; }
Log.info("Tmp fs uri:"+this.tmpFsUri); if (this.tmpFsUri != null) { this.tmpFs = FileSystem.get(new URI(this.tmpFsUri), new Configuration());
Log.info("Created tmpPath " + tmpPath + " with permissions " + perm + " and umask " + getUmask(conf));
private Path createDownloadDir(String workDir, String fileUrl) { Path downloadDirPath = Paths.get(workDir, DOWNLOAD_DIR); File downloadDirFile = downloadDirPath.toFile(); try { Log.info(String.format("Creating download dir %s", downloadDirFile.toPath().toString())); FileUtils.forceMkdir(downloadDirFile); } catch (IOException e) { throw new RuntimeException(String .format("Unable to create download location for archive: %s at %s", fileUrl, downloadDirPath.toString())); } Log.info(String.format("Created download dir %s", downloadDirFile.toPath().toString())); return downloadDirPath; }
protected void doStart() throws Exception { Log.info("Starting Bitronix Transaction Manager"); TransactionManagerServices.getTransactionManager(); }
private void undeploy(String filename) throws Exception { ContextHandler context=(ContextHandler)_currentDeployments.get(filename); Log.info("Undeploy "+filename+" -> "+context); if (context==null) return; context.stop(); _contexts.removeHandler(context); _currentDeployments.remove(filename); }
protected void doStop() throws Exception { Log.info("Shutting down Bitronix Transaction Manager"); TransactionManagerServices.getTransactionManager().shutdown(); }
protected void executeViewGeneration(TupleMRBuilder builder) throws IOException, InterruptedException, ClassNotFoundException, TablespaceGeneratorException, TupleMRException { try { Job generationJob = builder.createJob(); long start = System.currentTimeMillis(); generationJob.waitForCompletion(true); if (!generationJob.isSuccessful()) { throw new TablespaceGeneratorException("Error executing generation Job"); } long end = System.currentTimeMillis(); Log.info("Tablespace store generated in " + (end - start) + " ms."); } finally { builder.cleanUpInstanceFiles(); } }
static void linkUpFirstComponents(Configuration conf, String path, FileSystem fsTarget, String info) { int indexOfEnd = path.indexOf('/', 1); if (Shell.WINDOWS) { indexOfEnd = path.indexOf('/', indexOfEnd + 1); } String firstComponent = path.substring(0, indexOfEnd); URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri(); ConfigUtil.addLink(conf, firstComponent, linkTarget); Log.info("Added link for " + info + " " + firstComponent + "->" + linkTarget); } }
static void linkUpFirstComponents(Configuration conf, String path, FileSystem fsTarget, String info) { int indexOfEnd = path.indexOf('/', 1); if (Shell.WINDOWS) { indexOfEnd = path.indexOf('/', indexOfEnd + 1); } String firstComponent = path.substring(0, indexOfEnd); URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri(); ConfigUtil.addLink(conf, firstComponent, linkTarget); Log.info("Added link for " + info + " " + firstComponent + "->" + linkTarget); } }
private void deploy(String filename) throws Exception { ContextHandler context=createContext(filename); Log.info("Deploy "+filename+" -> "+ context); _contexts.addHandler(context); _currentDeployments.put(filename,context); if (_contexts.isStarted()) context.start(); }
static void linkUpFirstComponents(Configuration conf, String path, FileContext fsTarget, String info) { int indexOfEnd = path.indexOf('/', 1); if (Shell.WINDOWS) { indexOfEnd = path.indexOf('/', indexOfEnd + 1); } String firstComponent = path.substring(0, indexOfEnd); URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri(); ConfigUtil.addLink(conf, firstComponent, linkTarget); Log.info("Added link for " + info + " " + firstComponent + "->" + linkTarget); }
static void linkUpFirstComponents(Configuration conf, String path, FileContext fsTarget, String info) { int indexOfEnd = path.indexOf('/', 1); if (Shell.WINDOWS) { indexOfEnd = path.indexOf('/', indexOfEnd + 1); } String firstComponent = path.substring(0, indexOfEnd); URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri(); ConfigUtil.addLink(conf, firstComponent, linkTarget); Log.info("Added link for " + info + " " + firstComponent + "->" + linkTarget); }
public void dumpUrl() { Connector[] connectors = getServer().getConnectors(); for (int i=0;i<connectors.length;i++) { String connectorName = connectors[i].getName(); String displayName = getDisplayName(); if (displayName == null) displayName = "WebApp@"+connectors.hashCode(); Log.info(displayName + " at http://" + connectorName + getContextPath()); } }
static void setUpHomeDir(Configuration conf, FileSystem fsTarget) { String homeDir = fsTarget.getHomeDirectory().toUri().getPath(); int indexOf2ndSlash = homeDir.indexOf('/', 1); if (indexOf2ndSlash >0) { linkUpFirstComponents(conf, homeDir, fsTarget, "home dir"); } else { // home dir is at root. Just link the home dir itse URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri(); ConfigUtil.addLink(conf, homeDir, linkTarget); Log.info("Added link for home dir " + homeDir + "->" + linkTarget); } // Now set the root of the home dir for viewfs String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath(); ConfigUtil.setHomeDirConf(conf, homeDirRoot); Log.info("Home dir base for viewfs" + homeDirRoot); }
static void setUpHomeDir(Configuration conf, FileSystem fsTarget) { String homeDir = fsTarget.getHomeDirectory().toUri().getPath(); int indexOf2ndSlash = homeDir.indexOf('/', 1); if (indexOf2ndSlash >0) { linkUpFirstComponents(conf, homeDir, fsTarget, "home dir"); } else { // home dir is at root. Just link the home dir itse URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri(); ConfigUtil.addLink(conf, homeDir, linkTarget); Log.info("Added link for home dir " + homeDir + "->" + linkTarget); } // Now set the root of the home dir for viewfs String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath(); ConfigUtil.setHomeDirConf(conf, homeDirRoot); Log.info("Home dir base for viewfs" + homeDirRoot); }
public NodeHeartbeatResponse nodeHeartbeat(ApplicationAttemptId attemptId, long containerId, ContainerState containerState) throws Exception { ContainerStatus containerStatus = BuilderUtils.newContainerStatus( BuilderUtils.newContainerId(attemptId, containerId), containerState, "Success", 0); ArrayList<ContainerStatus> containerStatusList = new ArrayList<ContainerStatus>(1); containerStatusList.add(containerStatus); Log.info("ContainerStatus: " + containerStatus); return nodeHeartbeat(containerStatusList, true, ++responseId); }
static void setUpHomeDir(Configuration conf, FileContext fsTarget) { String homeDir = fsTarget.getHomeDirectory().toUri().getPath(); int indexOf2ndSlash = homeDir.indexOf('/', 1); if (indexOf2ndSlash >0) { linkUpFirstComponents(conf, homeDir, fsTarget, "home dir"); } else { // home dir is at root. Just link the home dir itse URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri(); ConfigUtil.addLink(conf, homeDir, linkTarget); Log.info("Added link for home dir " + homeDir + "->" + linkTarget); } // Now set the root of the home dir for viewfs String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath(); ConfigUtil.setHomeDirConf(conf, homeDirRoot); Log.info("Home dir base for viewfs" + homeDirRoot); }