private void runDistcp(Path from, Path to) throws Exception { info("sourcePath: " + from + ", destinationPath: " + to); EmbeddedGobblin embeddedGobblin = new EmbeddedGobblinDistcp(from, to).mrMode(); // Used for global throttling" embeddedGobblin.distributeJar("lib/*"); for (Map.Entry<String, String> entry : this.props.entrySet()) { if (entry.getKey() != null && (entry.getKey()).startsWith(ATTR_PREFIX)) { String key = (entry.getKey()).substring(ATTR_PREFIX.length()); embeddedGobblin.setConfiguration(key, entry.getValue()); } } JobExecutionResult result = embeddedGobblin.run(); if (!result.isSuccessful()) { throw new RuntimeException("Distcp job failed!", result.getErrorCause()); } }
/** * Set the timeout for launching the Gobblin job. */ public EmbeddedGobblin setLaunchTimeout(long timeout, TimeUnit timeUnit) { this.launchTimeout = new FullTimeout(timeout, timeUnit); return this; }
/** * Specify that the input jar should be added to workers' classpath on distributed mode. */ public EmbeddedGobblin distributeJar(String jarPath) { return distributeJarWithPriority(jarPath, 0); }
/** * Run the Gobblin job. This call will block until the job is done. * @return a {@link JobExecutionResult} containing the result of the execution. */ @NotOnCli public JobExecutionResult run() throws InterruptedException, TimeoutException, ExecutionException { JobExecutionDriver jobDriver = runAsync(); return jobDriver.get(this.jobTimeout.getTimeout(), this.jobTimeout.getTimeUnit()); }
/** * Enable state store. */ public EmbeddedGobblin useStateStore(String rootDir) { this.setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, "true"); this.setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, rootDir); return this; }
@Override @NotOnCli public EmbeddedGobblin setTemplate(String templateURI) throws URISyntaxException, SpecNotFoundException, JobTemplate.TemplateException { return super.setTemplate(templateURI); } }
private void stopGobblinInstanceDriver() { try { driver.stopAsync(); driver.awaitTerminated(EmbeddedGobblin.this.shutdownTimeout.getTimeout(), EmbeddedGobblin.this.shutdownTimeout .getTimeUnit()); } catch (TimeoutException te) { log.error("Failed to shutdown Gobblin instance driver."); } } });
@Override public void onFailure(Throwable t) { stopGobblinInstanceDriver(); }
/** * Run in simulate mode. Will log everythin it would copy, but not actually copy anything. */ public EmbeddedGobblinDistcp simulate() { this.setConfiguration(CopySource.SIMULATE, Boolean.toString(true)); return this; }
@Override public void run() { // Add jars needed at runtime to the sys config so MR job launcher will add them to distributed cache. EmbeddedGobblin.this.sysConfigOverrides.put(ConfigurationKeys.JOB_JAR_FILES_KEY, Joiner.on(",").join(getPrioritizedDistributedJars())); } };
/** * Use a {@link gobblin.runtime.api.GobblinInstancePlugin} identified by name. */ public EmbeddedGobblin usePlugin(String pluginAlias) throws ClassNotFoundException, IllegalAccessException, InstantiationException { return usePlugin(GobblinInstancePluginUtils.instantiatePluginByAlias(pluginAlias)); }
/** * Set the timeout for the Gobblin job execution from ISO-style period. */ public EmbeddedGobblin setJobTimeout(String timeout) { return setJobTimeout(Period.parse(timeout).getSeconds(), TimeUnit.SECONDS); }
/** * Set the timeout for launching the Gobblin job from ISO-style period. */ public EmbeddedGobblin setLaunchTimeout(String timeout) { return setLaunchTimeout(Period.parse(timeout).getSeconds(), TimeUnit.SECONDS); }
/** * Set the timeout for shutting down the Gobblin instance driver after the job is done from ISO-style period. */ public EmbeddedGobblin setShutdownTimeout(String timeout) { return setShutdownTimeout(Period.parse(timeout).getSeconds(), TimeUnit.SECONDS); }
/** * Set the timeout for shutting down the Gobblin instance driver after the job is done. */ public EmbeddedGobblin setShutdownTimeout(long timeout, TimeUnit timeUnit) { this.shutdownTimeout = new FullTimeout(timeout, timeUnit); return this; }
/** * Specify that the input jar should be added to workers' classpath on distributed mode. Jars with lower priority value * will appear first in the classpath. Default priority is 0. */ public EmbeddedGobblin distributeJarByClassWithPriority(Class<?> klazz, int priority) { return distributeJarWithPriority(ClassUtil.findContainingJar(klazz), priority); }
@Override public void onSuccess(@Nullable JobExecutionResult result) { stopGobblinInstanceDriver(); }
/** * If {@link #delete()} is used, specifies that newly empty parent directories should also be deleted. */ @CliObjectOption(description = "If deleting files on target, also delete newly empty parent directories.") public EmbeddedGobblinDistcp deleteEmptyParentDirectories() { this.setConfiguration(RecursiveCopyableDataset.DELETE_EMPTY_DIRECTORIES_KEY, Boolean.toString(true)); return this; }
/** * Set the timeout for the Gobblin job execution. */ public EmbeddedGobblin setJobTimeout(long timeout, TimeUnit timeUnit) { this.jobTimeout = new FullTimeout(timeout, timeUnit); return this; }