private String getMetadataFileNameForBranch(WorkUnitState state, int branchId) { // Note: This doesn't follow the pattern elsewhere in Gobblin where we have branch specific config // parameters! Leaving this way for backwards compatibility. String filePrefix = state.getProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_FILE); return ForkOperatorUtils.getPropertyNameForBranch(filePrefix, this.numBranches, branchId); }
/** * Given a {@link FsPermission} objects, set a key, value pair in the given {@link State} for the writer to * use when creating files. This method should be used in conjunction with {@link #deserializeWriterFilePermissions(State, int, int)}. */ public static void serializeWriterFilePermissions(State state, int numBranches, int branchId, FsPermission fsPermissions) { serializeFsPermissions(state, ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PERMISSIONS, numBranches, branchId), fsPermissions); }
/** * Given a {@link String} in octal notation, set a key, value pair in the given {@link State} for the writer to * use when creating files. This method should be used in conjunction with {@link #deserializeWriterFilePermissions(State, int, int)}. */ public static void setWriterFileOctalPermissions(State state, int numBranches, int branchId, String octalPermissions) { state.setProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PERMISSIONS, numBranches, branchId), octalPermissions); }
/** * Given a {@link FsPermission} objects, set a key, value pair in the given {@link State} for the writer to * use when creating files. This method should be used in conjunction with {@link #deserializeWriterDirPermissions(State, int, int)}. */ public static void serializeWriterDirPermissions(State state, int numBranches, int branchId, FsPermission fsPermissions) { serializeFsPermissions(state, ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DIR_PERMISSIONS, numBranches, branchId), fsPermissions); }
@Override public Converter<Class<String>, Class<String>, String, String> init(WorkUnitState workUnit) { this.pattern = Pattern.compile(Strings.nullToEmpty(workUnit.getProp( ForkOperatorUtils.getPropertyNameForBranch(workUnit, ConfigurationKeys.CONVERTER_STRING_FILTER_PATTERN)))); this.matcher = Optional.absent(); return this; }
private static String getWriterPartitionSuffix(State state, int numBranches, int branchId) { String propName = ForkOperatorUtils.getPropertyNameForBranch(WRITER_PARTITION_SUFFIX, numBranches, branchId); return state.getProp(propName, StringUtils.EMPTY); }
/** * Given a {@link String} in octal notation, set a key, value pair in the given {@link State} for the writer to * use when creating directories. This method should be used in conjunction with {@link #deserializeWriterDirPermissions(State, int, int)}. */ public static void setWriterDirOctalPermissions(State state, int numBranches, int branchId, String octalPermissions) { state.setProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DIR_PERMISSIONS, numBranches, branchId), octalPermissions); }
protected String getPartitionPath(State properties) { if (this.partition.isPresent()) { boolean includePartitionerFieldNames = properties.getPropAsBoolean(ForkOperatorUtils .getPropertyNameForBranch(WRITER_INCLUDE_PARTITION_IN_FILE_NAMES, this.branches, this.branch), false); boolean removePathSeparators = properties.getPropAsBoolean(ForkOperatorUtils .getPropertyNameForBranch(WRITER_REPLACE_PATH_SEPARATORS_IN_PARTITIONS, this.branches, this.branch), false); return AvroUtils.serializeAsPath(this.partition.get(), includePartitionerFieldNames, removePathSeparators).toString(); } else { return null; } }
@Override public Converter<Class<String>, Class<String>, String, String> init(WorkUnitState workUnit) { String stringSplitterDelimiterKey = ForkOperatorUtils.getPropertyNameForBranch( workUnit, ConfigurationKeys.CONVERTER_STRING_SPLITTER_DELIMITER); Preconditions.checkArgument(workUnit.contains(stringSplitterDelimiterKey), "Cannot use " + this.getClass().getName() + " with out specifying " + ConfigurationKeys.CONVERTER_STRING_SPLITTER_DELIMITER); this.splitter = Splitter.on(workUnit.getProp(stringSplitterDelimiterKey)).omitEmptyStrings(); return this; }
private Optional<DateTimeFormatter> getTimestampToPathFormatter(State state, int numBranches, int branchId) { String propName = ForkOperatorUtils.getPropertyNameForBranch(WRITER_PARTITION_PATTERN, numBranches, branchId); if (state.contains(propName)) { return Optional.of(DateTimeFormat.forPattern(state.getProp(propName)).withZone(this.timeZone)); } return Optional.absent(); }
@Override public Converter<Schema, Schema, GenericRecord, Object> init(WorkUnitState workUnit) { String fieldPathKey = ForkOperatorUtils.getPropertyNameForBranch(workUnit, ConfigurationKeys.CONVERTER_AVRO_EXTRACTOR_FIELD_PATH); Preconditions.checkArgument(workUnit.contains(fieldPathKey), "The converter " + this.getClass().getName() + " cannot be used without setting the property " + ConfigurationKeys.CONVERTER_AVRO_EXTRACTOR_FIELD_PATH); this.fieldLocation = workUnit.getProp(fieldPathKey); return this; }
@Override public Converter<String, String, Object, RawJsonDocument> init(WorkUnitState workUnit) { String keyFieldPath = ForkOperatorUtils.getPropertyNameForBranch(workUnit, KEY_FIELD_CONFIG); if (!workUnit.contains(keyFieldPath)) { log.warn("No configuration for which field to use as the key. Using the default {}", this.keyField); } else { this.keyField = workUnit.getProp(keyFieldPath); log.info("Using the field {} from config for writing converter", this.keyField); } return this; }
@Override public Converter<String, String, String, List<String>> init(WorkUnitState workUnit) { String stringSplitterDelimiterKey = ForkOperatorUtils.getPropertyNameForBranch(workUnit, ConfigurationKeys.CONVERTER_STRING_SPLITTER_DELIMITER); Preconditions.checkArgument(workUnit.contains(stringSplitterDelimiterKey), "Cannot use " + this.getClass().getName() + " with out specifying " + ConfigurationKeys.CONVERTER_STRING_SPLITTER_DELIMITER); this.splitter = Splitter.on(workUnit.getProp(stringSplitterDelimiterKey)); this.shouldTrimResults = workUnit.getPropAsBoolean(ConfigurationKeys.CONVERTER_STRING_SPLITTER_SHOULD_TRIM_RESULTS, ConfigurationKeys.DEFAULT_CONVERTER_STRING_SPLITTER_SHOULD_TRIM_RESULTS); return this; }
private static DatePartitionType getGranularity(State state, int numBranches, int branchId) { String propName = ForkOperatorUtils.getPropertyNameForBranch(WRITER_PARTITION_GRANULARITY, numBranches, branchId); String granularityValue = state.getProp(propName, DEFAULT_WRITER_PARTITION_GRANULARITY.toString()); Optional<DatePartitionType> granularity = Enums.getIfPresent(DatePartitionType.class, granularityValue.toUpperCase()); Preconditions.checkState(granularity.isPresent(), granularityValue + " is not a valid writer partition granularity"); return granularity.get(); }
@Test public void testGetPropertyNameForBranch() { Assert.assertEquals(ForkOperatorUtils.getPropertyNameForBranch(PROPERTY_FOO, -1), PROPERTY_FOO); Assert.assertEquals(ForkOperatorUtils.getPropertyNameForBranch(PROPERTY_FOO, 0), PROPERTY_FOO + ".0"); Assert.assertEquals(ForkOperatorUtils.getPropertyNameForBranch(PROPERTY_FOO, 1), PROPERTY_FOO + ".1"); Assert.assertEquals(ForkOperatorUtils.getPropertyNameForBranch(PROPERTY_FOO, 0, 0), PROPERTY_FOO); Assert.assertEquals(ForkOperatorUtils.getPropertyNameForBranch(PROPERTY_FOO, 1, 0), PROPERTY_FOO); Assert.assertEquals(ForkOperatorUtils.getPropertyNameForBranch(PROPERTY_FOO, 2, 0), PROPERTY_FOO + ".0"); Assert.assertEquals(ForkOperatorUtils.getPropertyNameForBranch(PROPERTY_FOO, 2, 1), PROPERTY_FOO + ".1"); }
private boolean shouldPublishWriterMetadataForBranch(int branchId) { String keyName = ForkOperatorUtils .getPropertyNameForBranch(ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_KEY, this.numBranches, branchId); return this.getState().getPropAsBoolean(keyName, false); }
/** * Deserializes a {@link FsPermission}s object that should be used when a {@link DataWriter} is creating directories. */ public static FsPermission deserializeWriterDirPermissions(State state, int numBranches, int branchId) { return new FsPermission(state.getPropAsShortWithRadix( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DIR_PERMISSIONS, numBranches, branchId), FsPermission.getDefault().toShort(), ConfigurationKeys.PERMISSION_PARSING_RADIX)); }
/** * Deserializes a {@link FsPermission}s object that should be used when a {@link DataWriter} is writing a file. */ public static FsPermission deserializeWriterFilePermissions(State state, int numBranches, int branchId) { return new FsPermission(state.getPropAsShortWithRadix( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PERMISSIONS, numBranches, branchId), FsPermission.getDefault().toShort(), ConfigurationKeys.PERMISSION_PARSING_RADIX)); }
private JdbcWriterCommands createJdbcWriterCommands(Connection conn) { String destKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY, this.branches, this.branchId); String destType = Preconditions.checkNotNull(this.state.getProp(destKey), destKey + " is required for underlying JDBC product name"); Destination dest = Destination.of(DestinationType.valueOf(destType.toUpperCase()), this.state); return this.jdbcWriterCommandsFactory.newInstance(dest, conn); }
/** * Get the writer {@link Destination.DestinationType}. * * @param branches number of forked branches * @param index branch index * @return writer {@link Destination.DestinationType} */ public Destination.DestinationType getDestinationType(int branches, int index) { return Destination.DestinationType.valueOf(this.taskState.getProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY, branches, index), Destination.DestinationType.HDFS.name())); }