@Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); setConf(conf); String host = uri.getHost(); if (host == null) { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } homeDirPrefix = conf.get( DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT); this.dfs = new DFSClient(uri, conf, statistics); this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority()); this.workingDir = getHomeDirectory(); }
@Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); setConf(conf); String host = uri.getHost(); if (host == null) { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } homeDirPrefix = conf.get( DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT); this.dfs = new DFSClient(uri, conf, statistics); this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority()); this.workingDir = getHomeDirectory(); }
public void initialize(URI uri, Configuration conf) throws IOException { setConf(conf); String host = uri.getHost(); if (host == null) { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } InetSocketAddress namenode = NameNode.getAddress(uri.getAuthority()); this.dfs = new DFSClient(namenode, conf, statistics); this.uri = NameNode.getUri(namenode); this.workingDir = getHomeDirectory(); }
@Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); setConf(conf); String host = uri.getHost(); if (host == null) { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } homeDirPrefix = conf.get( HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT); this.dfs = new DFSClient(uri, conf, statistics); this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority()); this.workingDir = getHomeDirectory(); storageStatistics = (DFSOpsCountStatistics) GlobalStorageStatistics.INSTANCE .put(DFSOpsCountStatistics.NAME, new StorageStatisticsProvider() { @Override public StorageStatistics provide() { return new DFSOpsCountStatistics(); } }); }
public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); setConf(conf); String host = uri.getHost(); if (host == null) { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } InetSocketAddress namenode = NameNode.getAddress(uri.getAuthority()); this.dfs = new DFSClient(namenode, conf, statistics); this.uri = NameNode.getUri(namenode); this.workingDir = getHomeDirectory(); pathValidator = new PathValidator(conf); }
properties.store(new FileWriter(tempFile), "DataCleaner Spark runner properties"); final URI uri = copyFileToHdfs(tempFile, _fileSystem.getHomeDirectory().toUri().resolve("temp/" + tempFile.getName()).toString()); sparkLauncher.addAppArgs(uri.toString());
cluster.waitActive(); DistributedFileSystem hdfs = cluster.getFileSystem(); String userDir = hdfs.getHomeDirectory().toUri().getPath().toString(); Path file = new Path(userDir, "file"); Path dir = new Path(userDir, "/dir");