public AvroFsHelper(State state) { this(state, HadoopUtils.newConfiguration()); }
/*** * Get source {@link FileSystem} * @return Source {@link FileSystem} * @throws IOException Issue in fetching {@link FileSystem} */ private static FileSystem getSourceFs() throws IOException { return FileSystem.get(HadoopUtils.newConfiguration()); }
public AbstractAvroToOrcConverter() { try { this.fs = FileSystem.get(HadoopUtils.newConfiguration()); } catch (IOException e) { throw new RuntimeException(e); } }
public static Configuration getConfFromProperties(Properties properties) { Configuration conf = newConfiguration(); for (String propName : properties.stringPropertyNames()) { conf.set(propName, properties.getProperty(propName)); } return conf; }
private FileSystem getFileSystem() throws IOException { final Configuration conf = HadoopUtils.newConfiguration(); final FileSystem fs = this.clusterConfig.hasPath(ConfigurationKeys.FS_URI_KEY) ? FileSystem .get(URI.create(this.clusterConfig.getString(ConfigurationKeys.FS_URI_KEY)), conf) : FileSystem.get(conf); return fs; } }
public HadoopFsEndPointDataset(HadoopFsEndPoint endPoint){ this.endPoint = endPoint; Configuration conf = HadoopUtils.newConfiguration(); try { FileSystem fs = FileSystem.get(this.endPoint.getFsURI(), conf); qualifiedDatasetRoot = fs.makeQualified(this.endPoint.getDatasetPath()); } catch (IOException e1) { // ignored qualifiedDatasetRoot = this.endPoint.getDatasetPath(); } }
private void calculateDatasetURN() { EndPoint e = this.copyRoute.getCopyTo(); if (e instanceof HadoopFsEndPoint) { HadoopFsEndPoint copyTo = (HadoopFsEndPoint) e; Configuration conf = HadoopUtils.newConfiguration(); try { FileSystem copyToFs = FileSystem.get(copyTo.getFsURI(), conf); this.datasetURN = copyToFs.makeQualified(copyTo.getDatasetPath()).toString(); } catch (IOException e1) { // ignored } } else { this.datasetURN = e.toString(); } }
/** * A helper utility for data/filesystem availability checking * @param path The path to be checked. * @return If the filesystem/path exists or not. */ public boolean isPathAvailable(Path path) { try { Configuration conf = HadoopUtils.newConfiguration(); FileSystem fs = FileSystem.get(this.getFsURI(), conf); if (fs.exists(path)) { return true; } else { log.warn("The data path [" + path + "] is not available on FileSystem: " + this.getFsURI()); return false; } } catch (IOException ioe) { log.warn("Errors occurred while checking path [" + path + "] existence " + this.getFsURI(), ioe); return false; } }
/** * Provides Hadoop configuration given state. * It also supports decrypting values on "encryptedPath". * Note that this encryptedPath path will be removed from full path of each config key and leaving only child path on the key(s). * If there's same config path as child path, the one stripped will have higher priority. * * e.g: * - encryptedPath: writer.fs.encrypted * before: writer.fs.encrypted.secret * after: secret * * Common use case for these encryptedPath: * When there's have encrypted credential in job property but you'd like Filesystem to get decrypted value. * * @param srcConfig source config. * @param encryptedPath Optional. If provided, config that is on this path will be decrypted. @see ConfigUtils.resolveEncrypted * Note that config on encryptedPath will be included in the end result even it's not part of includeOnlyPath * @return Hadoop Configuration. */ public static Configuration getConfFromState(State state, Optional<String> encryptedPath) { Config config = ConfigFactory.parseProperties(state.getProperties()); if (encryptedPath.isPresent()) { config = ConfigUtils.resolveEncrypted(config, encryptedPath); } Configuration conf = newConfiguration(); for (Entry<String, ConfigValue> entry : config.entrySet()) { conf.set(entry.getKey(), entry.getValue().unwrapped().toString()); } return conf; }
private static FileSystem getWriterFsUsingToken(State state, URI uri) throws IOException { try { String user = state.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME); Optional<Token<?>> token = ProxiedFileSystemUtils .getTokenFromSeqFile(user, new Path(state.getProp(ConfigurationKeys.FS_PROXY_AS_USER_TOKEN_FILE))); if (!token.isPresent()) { throw new IOException("No token found for user " + user); } return ProxiedFileSystemCache.fromToken().userNameToken(token.get()) .userNameToProxyAs(state.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME)).fsURI(uri) .conf(HadoopUtils.newConfiguration()).build(); } catch (ExecutionException e) { throw new IOException(e); } }
@Test public void testStateToConfiguration() throws IOException { Map<String, String> vals = Maps.newHashMap(); vals.put("test_key1", "test_val1"); vals.put("test_key2", "test_val2"); Configuration expected = HadoopUtils.newConfiguration(); State state = new State(); for (Map.Entry<String, String> entry : vals.entrySet()) { state.setProp(entry.getKey(), entry.getValue()); expected.set(entry.getKey(), entry.getValue()); } Assert.assertEquals(HadoopUtils.getConfFromState(state), expected); Assert.assertEquals(HadoopUtils.getConfFromState(state, Optional.<String>absent()), expected); Assert.assertEquals(HadoopUtils.getConfFromState(state, Optional.of("dummy")), expected); }
public HiveConvertPublisher(State state) throws IOException { super(state); this.avroSchemaManager = new AvroSchemaManager(FileSystem.get(HadoopUtils.newConfiguration()), state); this.metricContext = Instrumented.getMetricContext(state, HiveConvertPublisher.class); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, EventConstants.CONVERSION_NAMESPACE).build();
@Override public FileAwareInputStream readRecord(@Deprecated FileAwareInputStream reuse) throws DataRecordException, IOException { if (!this.recordRead) { Configuration conf = this.state == null ? HadoopUtils.newConfiguration() : HadoopUtils.getConfFromState(this.state); FileSystem fsFromFile = this.file.getOrigin().getPath().getFileSystem(conf); this.recordRead = true; FileAwareInputStream.FileAwareInputStreamBuilder builder = FileAwareInputStream.builder().file(this.file); if (this.file.getFileStatus().isDirectory()) { return builder.inputStream(EmptyInputStream.instance).build(); } FSDataInputStream dataInputStream = fsFromFile.open(this.file.getFileStatus().getPath()); if (this.state != null && DistcpFileSplitter.isSplitWorkUnit(this.state)) { Optional<DistcpFileSplitter.Split> split = DistcpFileSplitter.getSplit(this.state); builder.split(split); if (split.isPresent()) { dataInputStream.seek(split.get().getLowPosition()); } } builder.inputStream(MeteredInputStream.builder().in(dataInputStream).build()); return builder.build(); } return null; }
private static FileSystem getWriterFsUsingKeytab(State state, URI uri) throws IOException { FileSystem fs = FileSystem.newInstance(uri, new Configuration()); try { Preconditions.checkArgument(state.contains(ConfigurationKeys.FS_PROXY_AS_USER_NAME), "Missing required property " + ConfigurationKeys.FS_PROXY_AS_USER_NAME); Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS), "Missing required property " + ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS); Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION), "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION); String user = state.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME); String superUser = state.getProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS); Path keytabLocation = new Path(state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION)); return ProxiedFileSystemCache.fromKeytab().userNameToProxyAs(user).fsURI(uri) .superUserKeytabLocation(keytabLocation).superUserName(superUser).conf(HadoopUtils.newConfiguration()) .referenceFS(fs).build(); } catch (ExecutionException e) { throw new IOException(e); } }
Configuration conf = HadoopUtils.newConfiguration(); FileSystem copyFromFs = FileSystem.get(copyFrom.getFsURI(), conf); FileSystem copyToFs = FileSystem.get(copyTo.getFsURI(), conf);
this.dedicatedTaskDriverCluster = ConfigUtils.getBoolean(config, GobblinClusterConfigurationKeys.DEDICATED_TASK_DRIVER_CLUSTER_ENABLED, false); Configuration conf = HadoopUtils.newConfiguration(); this.fs = buildFileSystem(config, conf); this.appWorkPath = initAppWorkDir(config, appWorkDirOptional);
public AbstractAvroToOrcConverter() { try { this.fs = FileSystem.get(HadoopUtils.newConfiguration()); } catch (IOException e) { throw new RuntimeException(e); } }
public static Configuration getConfFromProperties(Properties properties) { Configuration conf = newConfiguration(); for (String propName : properties.stringPropertyNames()) { conf.set(propName, properties.getProperty(propName)); } return conf; }
private FileSystem getFileSystem() throws IOException { final Configuration conf = HadoopUtils.newConfiguration(); final FileSystem fs = this.clusterConfig.hasPath(ConfigurationKeys.FS_URI_KEY) ? FileSystem .get(URI.create(this.clusterConfig.getString(ConfigurationKeys.FS_URI_KEY)), conf) : FileSystem.get(conf); return fs; } }
public HadoopFsEndPointDataset(HadoopFsEndPoint endPoint){ this.endPoint = endPoint; Configuration conf = HadoopUtils.newConfiguration(); try { FileSystem fs = FileSystem.get(this.endPoint.getFsURI(), conf); qualifiedDatasetRoot = fs.makeQualified(this.endPoint.getDatasetPath()); } catch (IOException e1) { // ignored qualifiedDatasetRoot = this.endPoint.getDatasetPath(); } }