@Override public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); State state = HadoopUtils.getStateFromConf(conf); this.fsHelper = new SftpFsHelper(state); try { this.fsHelper.connect(); } catch (FileBasedHelperException e) { throw new IOException(e); } }
try (Closer closer = Closer.create()) { this.isSpeculativeEnabled = isSpeculativeExecutionEnabled(HadoopUtils.getStateFromConf(context.getConfiguration()).getProperties()); this.fs = FileSystem.get(context.getConfiguration()); this.taskStateStore = this.jobMetrics = Optional.of(JobMetrics.get(this.jobState)); this.jobMetrics.get() .startMetricReportingWithFileSuffix(HadoopUtils.getStateFromConf(configuration), context.getTaskAttemptID().toString());
@Override public synchronized void initialize(URI uri, Configuration conf) throws IOException { if (this.client == null) { super.initialize(uri, conf); State state = HadoopUtils.getStateFromConf(conf); Credential credential = new GoogleCommon.CredentialBuilder(state.getProp(SOURCE_CONN_PRIVATE_KEY), state.getPropAsList(API_SCOPES)) .fileSystemUri(state.getProp(PRIVATE_KEY_FILESYSTEM_URI)) .proxyUrl(state.getProp(SOURCE_CONN_USE_PROXY_URL)) .port(state.getProp(SOURCE_CONN_USE_PROXY_PORT)) .serviceAccountId(state.getProp(SOURCE_CONN_USERNAME)) .build(); this.client = new Drive.Builder(credential.getTransport(), GoogleCommon.getJsonFactory(), credential) .setApplicationName(Preconditions.checkNotNull(state.getProp(APPLICATION_NAME), "ApplicationName is required")) .build(); this.pageSize = state.getPropAsInt(PAGE_SIZE, DEFAULT_PAGE_SIZE); } }
@Override public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); State state = HadoopUtils.getStateFromConf(conf); this.fsHelper = new SftpFsHelper(state); try { this.fsHelper.connect(); } catch (FileBasedHelperException e) { throw new IOException(e); } }
try (Closer closer = Closer.create()) { this.isSpeculativeEnabled = isSpeculativeExecutionEnabled(HadoopUtils.getStateFromConf(context.getConfiguration()).getProperties()); this.fs = FileSystem.get(context.getConfiguration()); this.taskStateStore = this.jobMetrics = Optional.of(JobMetrics.get(this.jobState)); this.jobMetrics.get() .startMetricReportingWithFileSuffix(HadoopUtils.getStateFromConf(configuration), context.getTaskAttemptID().toString());
@Override public synchronized void initialize(URI uri, Configuration conf) throws IOException { if (this.client == null) { super.initialize(uri, conf); State state = HadoopUtils.getStateFromConf(conf); Credential credential = new GoogleCommon.CredentialBuilder(state.getProp(SOURCE_CONN_PRIVATE_KEY), state.getPropAsList(API_SCOPES)) .fileSystemUri(state.getProp(PRIVATE_KEY_FILESYSTEM_URI)) .proxyUrl(state.getProp(SOURCE_CONN_USE_PROXY_URL)) .port(state.getProp(SOURCE_CONN_USE_PROXY_PORT)) .serviceAccountId(state.getProp(SOURCE_CONN_USERNAME)) .build(); this.client = new Drive.Builder(credential.getTransport(), GoogleCommon.getJsonFactory(), credential) .setApplicationName(Preconditions.checkNotNull(state.getProp(APPLICATION_NAME), "ApplicationName is required")) .build(); this.pageSize = state.getPropAsInt(PAGE_SIZE, DEFAULT_PAGE_SIZE); } }