How to use

EmbeddedGobblin

in

gobblin.runtime.embedded

private void runDistcp(Path from, Path to) throws Exception {
  info("sourcePath: " + from + ", destinationPath: " + to);
  EmbeddedGobblin embeddedGobblin = new EmbeddedGobblinDistcp(from, to).mrMode();
  // Used for global throttling"
  embeddedGobblin.distributeJar("lib/*");
  for (Map.Entry<String, String> entry : this.props.entrySet()) {
    if (entry.getKey() != null && (entry.getKey()).startsWith(ATTR_PREFIX)) {
      String key = (entry.getKey()).substring(ATTR_PREFIX.length());
      embeddedGobblin.setConfiguration(key, entry.getValue());
    }
  }
  JobExecutionResult result =  embeddedGobblin.run();
  if (!result.isSuccessful()) {
    throw new RuntimeException("Distcp job failed!", result.getErrorCause());
  }
}

 @Override
 @NotOnCli
 public EmbeddedGobblin setTemplate(String templateURI)
   throws URISyntaxException, SpecNotFoundException, JobTemplate.TemplateException {
  return super.setTemplate(templateURI);
 }
}

/**
 * Enable state store.
 */
public EmbeddedGobblin useStateStore(String rootDir) {
 this.setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, "true");
 this.setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, rootDir);
 return this;
}

@CliObjectSupport(argumentNames = {"jobName"})
public EmbeddedGobblin(String name) {
 HadoopUtils.addGobblinSite();
 this.specBuilder = new JobSpec.Builder(name);
 this.userConfigMap = Maps.newHashMap();
 this.builtConfigMap = Maps.newHashMap();
 this.sysConfigOverrides = Maps.newHashMap();
 this.defaultSysConfig = getDefaultSysConfig();
 this.distributedJars = Maps.newHashMap();
 loadCoreGobblinJarsToDistributedJars();
 this.distributeJarsFunction = new Runnable() {
  @Override
  public void run() {
   // NOOP
  }
 };
}

 embeddedGobblin.run();
} catch (InterruptedException | TimeoutException | ExecutionException exc) {
 throw new RuntimeException("Failed to run Gobblin job.", exc);

/**
 * Specify that the input jar should be added to workers' classpath on distributed mode.
 */
public EmbeddedGobblin distributeJar(String jarPath) {
 return distributeJarWithPriority(jarPath, 0);
}

distributeJarByClassWithPriority(State.class, 0);
distributeJarByClassWithPriority(ConstructState.class, 0);
distributeJarByClassWithPriority(InstrumentedExtractorBase.class, 0);
distributeJarByClassWithPriority(MetricContext.class, 0);
distributeJarByClassWithPriority(GobblinMetrics.class, 0);
distributeJarByClassWithPriority(FsStateStore.class, 0);
distributeJarByClassWithPriority(Task.class, 0);
distributeJarByClassWithPriority(PathUtils.class, 0);
distributeJarByClassWithPriority(ReadableInstant.class, 0);
distributeJarByClassWithPriority(Escaper.class, -10); // Escaper was added in guava 15, so we use it to identify correct jar
distributeJarByClassWithPriority(MetricFilter.class, 0);
distributeJarByClassWithPriority(DataTemplate.class, 0);
distributeJarByClassWithPriority(ClassUtils.class, 0);
distributeJarByClassWithPriority(SchemaBuilder.class, 0);
distributeJarByClassWithPriority(RetryListener.class, 0);

 @Override
 public void run() {
  // Add jars needed at runtime to the sys config so MR job launcher will add them to distributed cache.
  EmbeddedGobblin.this.sysConfigOverrides.put(ConfigurationKeys.JOB_JAR_FILES_KEY,
    Joiner.on(",").join(getPrioritizedDistributedJars()));
 }
};

new SimpleGobblinInstanceEnvironment("EmbeddedGobblinInstance", this.useLog, getSysConfig());

/**
 * Set a {@link JobTemplate} to use.
 */
public EmbeddedGobblin setTemplate(String templateURI) throws URISyntaxException, SpecNotFoundException,
                           JobTemplate.TemplateException {
 return setTemplate(new PackagedTemplatesJobCatalogDecorator().getTemplate(new URI(templateURI)));
}

/**
 * Manually set a key-value pair in the job configuration. Input is of the form <key>:<value>
 */
public EmbeddedGobblin setConfiguration(String keyValue) {
 List<String> split = KEY_VALUE_SPLITTER.splitToList(keyValue);
 if (split.size() != 2) {
  throw new RuntimeException("Cannot parse " + keyValue + ". Expected <key>:<value>.");
 }
 return setConfiguration(split.get(0), split.get(1));
}

/**
 * Specify that the input jar should be added to workers' classpath on distributed mode. Jars with lower priority value
 * will appear first in the classpath. Default priority is 0.
 */
public EmbeddedGobblin distributeJarByClassWithPriority(Class<?> klazz, int priority) {
 return distributeJarWithPriority(ClassUtil.findContainingJar(klazz), priority);
}