/** * Conf for the token writer. */ public static class Conf extends PluginConfig { @Description("The fileset to delete files from.") private String filesetName; @Description("Delete files that match this regex.") private String deleteRegex; } }
/** * Returns description for the plugin. */ private String getPluginDescription(Class<?> cls) { Description annotation = cls.getAnnotation(Description.class); return annotation == null ? "" : annotation.value(); }
/** * Config for window plugin. */ public static class Conf extends PluginConfig { @Description("Width of the window in seconds. Must be a multiple of the pipeline's batch interval. " + "Each window generated will contain all the events from this many seconds.") long width; @Description("Sliding interval of the window in seconds. Must be a multiple of the pipeline's batch interval. " + "Every this number of seconds, a new window will be generated.") long slideInterval; } }
/** * Returns description for the plugin. */ private String getPluginDescription(Class<?> cls) { Description annotation = cls.getAnnotation(Description.class); return annotation == null ? "" : annotation.value(); }
/** * ValidatorConfig whose list of validators and script can be configured */ public static class ValidatorConfig extends PluginConfig { @Description("Comma-separated list of validator plugins that are used in script") String validators; @Description(SCRIPT_DESCRIPTION) String validationScript; @Description("Lookup tables to use during transform. Currently supports KeyValueTable.") @Nullable String lookup; } }
/** * Adds a class that extends from <code>AbstractDirective</code> to usage registry. * * @param classz representing an AbstractDirective</code> implementation. */ public void addUsage(Class<? extends Executor> classz) { Name name = classz.getAnnotation(Name.class); Description description = classz.getAnnotation(Description.class); Usage usage = classz.getAnnotation(Usage.class); if (usage == null || name == null || description == null) { return; } usages.put(name.value(), new UsageEntry(name.value(), usage.value(), description.value())); usageList.add(new UsageEntry(name.value(), usage.value(), description.value())); }
/** * Base aggregator config, since almost all aggregators should support setting the number of partitions. */ public class AggregatorConfig extends PluginConfig { @Nullable @Description("Number of partitions to use when aggregating. If not specified, the execution framework " + "will decide how many to use.") protected Integer numPartitions; }
/** * Creates a {@link PluginPropertyField} based on the given field. */ private PluginPropertyField createPluginProperty(Field field, TypeToken<?> resolvingType) throws UnsupportedTypeException { TypeToken<?> fieldType = resolvingType.resolveType(field.getGenericType()); Class<?> rawType = fieldType.getRawType(); Name nameAnnotation = field.getAnnotation(Name.class); Description descAnnotation = field.getAnnotation(Description.class); String name = nameAnnotation == null ? field.getName() : nameAnnotation.value(); String description = descAnnotation == null ? "" : descAnnotation.value(); Macro macroAnnotation = field.getAnnotation(Macro.class); boolean macroSupported = macroAnnotation != null; if (rawType.isPrimitive()) { return new PluginPropertyField(name, description, rawType.getName(), true, macroSupported); } rawType = Primitives.unwrap(rawType); if (!rawType.isPrimitive() && !String.class.equals(rawType)) { throw new UnsupportedTypeException("Only primitive and String types are supported"); } boolean required = true; for (Annotation annotation : field.getAnnotations()) { if (annotation.annotationType().getName().endsWith(".Nullable")) { required = false; break; } } return new PluginPropertyField(name, description, rawType.getSimpleName().toLowerCase(), required, macroSupported); }
/** * Plugin Configuration */ public static class Conf extends AggregatorConfig { @Nullable @Description("Optional comma-separated list of fields to perform the distinct on. If none is given, each record " + "will be taken as is. Otherwise, only fields in this list will be considered.") private String fields; Iterable<String> getFields() { return fields == null ? Collections.emptyList() : Splitter.on(',').trimResults().split(fields); } }
/** * Creates a {@link PluginPropertyField} based on the given field. */ private PluginPropertyField createPluginProperty(Field field, TypeToken<?> resolvingType) throws UnsupportedTypeException { TypeToken<?> fieldType = resolvingType.resolveType(field.getGenericType()); Class<?> rawType = fieldType.getRawType(); Name nameAnnotation = field.getAnnotation(Name.class); Description descAnnotation = field.getAnnotation(Description.class); String name = nameAnnotation == null ? field.getName() : nameAnnotation.value(); String description = descAnnotation == null ? "" : descAnnotation.value(); Macro macroAnnotation = field.getAnnotation(Macro.class); boolean macroSupported = macroAnnotation != null; if (rawType.isPrimitive()) { return new PluginPropertyField(name, description, rawType.getName(), true, macroSupported); } rawType = Primitives.unwrap(rawType); if (!rawType.isPrimitive() && !String.class.equals(rawType)) { throw new UnsupportedTypeException("Only primitive and String types are supported"); } boolean required = true; for (Annotation annotation : field.getAnnotations()) { if (annotation.annotationType().getName().endsWith(".Nullable")) { required = false; break; } } return new PluginPropertyField(name, description, rawType.getSimpleName().toLowerCase(), required, macroSupported); }
/** * Conf for the token writer. */ public static class Conf extends PluginConfig { @Description("The fileset to delete files from.") private String sourceFileset; @Description("The path to the files to delete") private String destinationFileset; @Nullable @Description("Filter any files whose name matches this regex. Defaults to '^\\.', which will filter any files " + "that begin with a period.") private String filterRegex; // set defaults for properties in a no-argument constructor. public Conf() { filterRegex = "^\\."; } } }
public DirectiveInfo(Scope scope, Class<?> directive) throws IllegalAccessException, InstantiationException { this.scope = scope; this.directive = directive; Object object = directive.newInstance(); this.definition = ((Directive) object).define(); if (definition != null) { this.usage = definition.toString(); } else { this.usage = "No definition available for directive '" + directive + "'"; } this.name = directive.getAnnotation(Name.class).value(); Description desc = directive.getAnnotation(Description.class); if (desc == null) { this.description = "No description specified for directive class '" + directive.getSimpleName() + "'"; } else { this.description = desc.value(); } Deprecated annotation = directive.getAnnotation(Deprecated.class); if (annotation == null) { deprecated = false; } else { deprecated = true; } Categories category = directive.getAnnotation(Categories.class); if (category == null) { categories = new String[] { "default" }; } else { categories = category.categories(); } }
/** * Config for TimePartitionedFileSetAvroSink */ public static class TPFSAvroSinkConfig extends TPFSSinkConfig { @Nullable @Description("Used to specify the compression codec to be used for the final dataset.") private String compressionCodec; public TPFSAvroSinkConfig(String name, @Nullable String basePath, @Nullable String pathFormat, @Nullable String timeZone, @Nullable String compressionCodec) { super(name, basePath, pathFormat, timeZone); this.compressionCodec = compressionCodec; } } }
/** * Config for TimePartitionedFileSetParquetSink */ public static class TPFSParquetSinkConfig extends TPFSSinkConfig { @Nullable @Description("Used to specify the compression codec to be used for the final dataset.") private String compressionCodec; public TPFSParquetSinkConfig(String name, @Nullable String basePath, @Nullable String pathFormat, @Nullable String timeZone, @Nullable String compressionCodec) { super(name, basePath, pathFormat, timeZone); this.compressionCodec = compressionCodec; } } }
/** * The plugin config */ public static class Config extends PluginConfig { @Nullable @Description("The name of the error message field to use in the output schema. " + "If this not specified, the error message will be dropped.") private String messageField; @Nullable @Description("The name of the error code field to use in the output schema. " + "If this not specified, the error code will be dropped.") private String codeField; @Nullable @Description("The name of the error stage field to use in the output schema. " + "If this not specified, the error stage will be dropped.") private String stageField; } }
/** * Configuration for the script transform. */ public static class Config extends PluginConfig { @Description("Python code defining how to transform one record into another. The script must implement a " + "function called 'transform', which takes as input a dictionary representing the input record, an emitter " + "object, and a context object (which contains CDAP metrics and logger). The emitter object can be used to emit " + "one or more key-value pairs to the next stage. It can also be used to emit errors. " + "For example:\n" + "'def transform(record, emitter, context):\n" + " if record['count'] == 0:\n" + " emitter.emitError({\"errorCode\":31, \"errorMsg\":\"Count is zero.\", \"invalidRecord\": record}\n" + " return\n" + " record['count'] *= 1024\n" + " if(record['count'] < 0):\n" + " context.getMetrics().count(\"negative.count\", 1)\n" + " context.getLogger().debug(\"Received record with negative count\")\n" + " emitter.emit(record)'\n" + "will scale the 'count' field by 1024.") private final String script; @Description("The schema of the output object. If no schema is given, it is assumed that the output schema is " + "the same as the input schema.") @Nullable private final String schema; public Config(String script, String schema) { this.script = script; this.schema = schema; } }
/** * Config class for ProjectionTransform */ public static class ProjectionTransformConfig extends PluginConfig { @Description(DROP_DESC) @Nullable String drop; @Description(RENAME_DESC) @Nullable String rename; @Description(CONVERT_DESC) @Nullable String convert; @Description(KEEP_DESC) @Nullable String keep; public ProjectionTransformConfig(String drop, String rename, String convert, String keep) { this.drop = drop; this.rename = rename; this.convert = convert; this.keep = keep; } }
public static class P2Config extends PluginConfig { @Description("value to return when called") private int v; } }
@Description("JavaScript defining how to transform input record into zero or more records. " + "The script must implement a function " + "called 'transform', which takes as input a JSON object (representing the input record) " + @Description("The schema of output objects. If no schema is given, it is assumed that the output schema is " + "the same as the input schema.") @Nullable private final String schema; @Description("Lookup tables to use during transform. Currently supports KeyValueTable.") @Nullable private final String lookup;
@Description("Used to specify the compression codec to be used for the final dataset.") private String compressionCodec; @Description("Number of bytes in each compression chunk.") private Long compressionChunkSize; @Description("Number of bytes in each stripe.") private Long stripeSize; @Description("Number of rows between index entries (must be >= 1,000)") private Long indexStride; @Description("Whether to create inline indexes") private Boolean createIndex;