@Override public boolean apply(WorkUnitState input) { return null == input || !WorkingState.SUCCESSFUL.equals(input.getWorkingState()); } };
@SuppressWarnings("unchecked") public T withProps(State props) { this.props = new State(props.getProperties()); return (T) this; }
public CachedInstanceKey(State state) { this.numOfEncryptionKeys = state.getPropAsInt(ConfigurationKeys.NUMBER_OF_ENCRYPT_KEYS, ConfigurationKeys.DEFAULT_NUMBER_OF_MASTER_PASSWORDS); this.useStrongEncryptor = shouldUseStrongEncryptor(state); this.fsURI = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC); this.masterPasswordFile = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC); }
/** * TODO: Figure out what this means for checkpointing. * Get final state for this object. By default this returns an empty {@link org.apache.gobblin.configuration.State}, but * concrete subclasses can add information that will be added to the task state. * @return Empty {@link org.apache.gobblin.configuration.State}. */ @Override public State getFinalState() { return new State(); }
@Override public String datasetURN() { return state.getProp(SERIALIZE_COMPACTION_FILE_PATH_NAME); } };
private static SourceState getCombinedState(SourceState state, State tableSpecificState) { if (tableSpecificState == null) { return state; } SourceState combinedState = new SourceState(state, state.getPreviousDatasetStatesByUrns(), state.getPreviousWorkUnitStates()); combinedState.addAll(tableSpecificState); return combinedState; }
/** * Return true if the current publisher can be skipped. * * <p> * For a publisher that can be skipped, it should not have any effect on state persistence. It will be skipped when * a job is cancelled, and all finished tasks are configured to be committed. * </p> */ public boolean canBeSkipped() { return this.state.getPropAsBoolean(ConfigurationKeys.DATA_PUBLISHER_CAN_BE_SKIPPED, ConfigurationKeys.DEFAULT_DATA_PUBLISHER_CAN_BE_SKIPPED); }
private static SourceState getTestState(String dbName) { SourceState testState = new SourceState(); testState.setProp("hive.dataset.database", dbName); testState.setProp("hive.dataset.table.pattern", "*"); testState.setProp(ConfigurationKeys.JOB_ID_KEY, "testJobId"); return testState; }
public RestApiConnector(State state) { this.state = state; this.authTokenTimeout = state.getPropAsInt(ConfigurationKeys.SOURCE_CONN_TIMEOUT, ConfigurationKeys.DEFAULT_CONN_TIMEOUT); }
@Override public void close() throws IOException { workUnitState.setActualHighWatermark(currentWatermark); }
@Override public LongWatermark apply(WorkUnitState wus) { return wus.getActualHighWatermark(LongWatermark.class); } });
/** * If creating a {@link WorkUnitState} for use by a task, use {@link #WorkUnitState(WorkUnit, State, SharedResourcesBroker)} * instead. */ public WorkUnitState(WorkUnit workUnit, State jobState) { this(workUnit, jobState, buildTaskBroker(null, jobState, workUnit)); }
public State getPreviousTableState() { WorkUnitState workUnitState = (WorkUnitState) this.state; return workUnitState.getPreviousTableState(); } }
/** * Get final state for this object. By default this returns an empty {@link org.apache.gobblin.configuration.State}, but * concrete subclasses can add information that will be added to the task state. * @return Empty {@link org.apache.gobblin.configuration.State}. */ @Override public State getFinalState() { return new State(); }
/** * Creates a unique {@link String} representing this branch. */ private static String getForkMetricsId(State state, int index) { return state.getProp(ConfigurationKeys.FORK_BRANCH_NAME_KEY + "." + index, ConfigurationKeys.DEFAULT_FORK_BRANCH_NAME + index); }
@Override public boolean apply(WorkUnitState workUnitState) { return workUnitState.getWorkingState().equals(WorkingState.SUCCESSFUL); } }
/** * Get final state for this object. By default this returns an empty {@link org.apache.gobblin.configuration.State}, but * concrete subclasses can add information that will be added to the task state. * @return Empty {@link org.apache.gobblin.configuration.State}. */ @Override public State getFinalState() { return new State(); }
public CleanableHivePartitionDataset(Partition partition, FileSystem fs, State state) { super(partition); this.fs = fs; this.state = new State(state); }