@Override public Set<String> getPropertyNames() { Set<String> set = Sets.newHashSet(super.getPropertyNames()); set.addAll(this.workUnit.getPropertyNames()); set.addAll(this.jobState.getPropertyNames()); return set; }
/** * Put all configuration properties in a given {@link State} object into a given * {@link Configuration} object. * * @param state the given {@link State} object * @param configuration the given {@link Configuration} object */ public static void putStateIntoConfiguration(State state, Configuration configuration) { for (String key : state.getPropertyNames()) { configuration.set(key, state.getProp(key)); } }
private static Map<String, String> getParameters(State props) { Map<String, String> parameters = Maps.newHashMap(); if (props.contains(RUNTIME_PROPS)) { String runtimePropsString = props.getProp(RUNTIME_PROPS); for (String propValue : LIST_SPLITTER_COMMA.splitToList(runtimePropsString)) { List<String> tokens = LIST_SPLITTER_COLON.splitToList(propValue); Preconditions.checkState(tokens.size() == 2, propValue + " is not a valid Hive table/partition property"); parameters.put(tokens.get(0), tokens.get(1)); } } for (String propKey : props.getPropertyNames()) { if (!propKey.equals(RUNTIME_PROPS)) { parameters.put(propKey, props.getProp(propKey)); } } return parameters; }
/** * Set serde parameters for a table/partition. * * <p> * When using {@link gobblin.hive.metastore.HiveMetaStoreBasedRegister}, since it internally use * {@link org.apache.hadoop.hive.metastore.api.Table} and {@link org.apache.hadoop.hive.metastore.api.Partition} * which distinguishes between table/partition parameters, storage descriptor parameters, and serde parameters, * one may need to distinguish them when constructing a {@link HiveRegistrationUnit} by using * {@link #setProps(State)}, {@link #setStorageProps(State)} and * {@link #setSerDeProps(State)}. When using query-based Hive registration, they do not need to be * distinguished since all parameters will be passed via TBLPROPERTIES. * </p> */ public void setSerDeProps(State serdeProps) { for (String propKey : serdeProps.getPropertyNames()) { setSerDeProp(propKey, serdeProps.getProp(propKey)); } }
/** * Set table/partition parameters. * * <p> * When using {@link gobblin.hive.metastore.HiveMetaStoreBasedRegister}, since it internally use * {@link org.apache.hadoop.hive.metastore.api.Table} and {@link org.apache.hadoop.hive.metastore.api.Partition} * which distinguishes between table/partition parameters, storage descriptor parameters, and serde parameters, * one may need to distinguish them when constructing a {@link HiveRegistrationUnit} by using * {@link #setProps(State)}, {@link #setStorageProps(State)} and * {@link #setSerDeProps(State)}. When using query-based Hive registration, they do not need to be * distinguished since all parameters will be passed via TBLPROPERTIES. * </p> */ public void setProps(State props) { for (String propKey : props.getPropertyNames()) { setProp(propKey, props.getProp(propKey)); } }
/** * Set storage parameters for a table/partition. * * <p> * When using {@link gobblin.hive.metastore.HiveMetaStoreBasedRegister}, since it internally use * {@link org.apache.hadoop.hive.metastore.api.Table} and {@link org.apache.hadoop.hive.metastore.api.Partition} * which distinguishes between table/partition parameters, storage descriptor parameters, and serde parameters, * one may need to distinguish them when constructing a {@link HiveRegistrationUnit} by using * {@link #setProps(State)}, {@link #setStorageProps(State)} and * {@link #setSerDeProps(State)}. When using query-based Hive registration, they do not need to be * distinguished since all parameters will be passed via TBLPROPERTIES. * </p> */ public void setStorageProps(State storageProps) { for (String propKey : storageProps.getPropertyNames()) { setStorageProp(propKey, storageProps.getProp(propKey)); } }
/** * Adds all properties from {@link gobblin.configuration.State} to this {@link gobblin.configuration.WorkUnitState}. * * <p> * A property with name "property" will be added to this object with the key * "{@link #FINAL_CONSTRUCT_STATE_PREFIX}[.<infix>].property" * </p> * * @param infix Optional infix used for the name of the property in the {@link gobblin.configuration.WorkUnitState}. * @param finalConstructState {@link gobblin.configuration.State} for which all properties should be added to this * object. */ public void addFinalConstructState(String infix, State finalConstructState) { for (String property : finalConstructState.getPropertyNames()) { if (Strings.isNullOrEmpty(infix)) { setProp(FINAL_CONSTRUCT_STATE_PREFIX + property, finalConstructState.getProp(property)); } else { setProp(FINAL_CONSTRUCT_STATE_PREFIX + infix + "." + property, finalConstructState.getProp(property)); } } }