protected HeadersToBody(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.excludeFields = getConfigs().getStringList(config, CONF_EXCLUDE_FIELDS, ImmutableList.<String>of()); this.excludeFields.add(Fields.ATTACHMENT_BODY); this.mapper = new ObjectMapper(); }
protected HeadersToBody(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.excludeFields = getConfigs().getStringList(config, CONF_EXCLUDE_FIELDS, ImmutableList.<String>of()); this.excludeFields.add(Fields.ATTACHMENT_BODY); this.mapper = new ObjectMapper(); }
/** Deprecated; will be removed in the next release */ @Deprecated protected AbstractParser(Config config, Command parent, Command child, MorphlineContext context) { super(config, parent, child, context); List<String> mimeTypes = getConfigs().getStringList(config, SUPPORTED_MIME_TYPES, Collections.<String>emptyList()); for (String mimeType : mimeTypes) { addSupportedMimeType(mimeType); } this.numRecordsMeter = getMeter(Metrics.NUM_RECORDS); }
/** Deprecated; will be removed in the next release */ @Deprecated protected AbstractParser(Config config, Command parent, Command child, MorphlineContext context) { super(config, parent, child, context); List<String> mimeTypes = getConfigs().getStringList(config, SUPPORTED_MIME_TYPES, Collections.<String>emptyList()); for (String mimeType : mimeTypes) { addSupportedMimeType(mimeType); } this.numRecordsMeter = getMeter(Metrics.NUM_RECORDS); }
public FieldFilter(final CommandBuilder builder, final Config config, final Command parent, final Command child, final MorphlineContext context) { super(builder, config, parent, child, context); this.excludeFields = getConfigs().getStringList(config, CONF_EXCLUDE_FIELDS, DEFAULT_EXCLUDE_FIELDS); this.includeFields = getConfigs().getStringList(config, CONF_INCLUDE_FIELDS, DEFAULT_INCLUDE_FIELDS); if (excludeFields.isEmpty() && includeFields.isEmpty()) { throw new MorphlineCompilationException(String.format("Either %s or %s must be set", CONF_EXCLUDE_FIELDS, CONF_INCLUDE_FIELDS), config); } this.isRegex = getConfigs().getBoolean(config, CONF_IS_REGEX, DEFAULT_IS_REGEX); if (this.isRegex) { if (!excludeFields.isEmpty()) { excludePattern = Pattern.compile(String.format("(%s)", Joiner.on('|').join(excludeFields))); } else { excludePattern = Pattern.compile(".*"); } if (!includeFields.isEmpty()) { includePattern = Pattern.compile(String.format("(%s)", Joiner.on('|').join(includeFields))); } else { includePattern = Pattern.compile(".*"); } } else { includePattern = null; excludePattern = null; } }
public RemoveFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); List<String> includes = getConfigs().getStringList(config, "blacklist", Collections.singletonList("*")); List<String> excludes = getConfigs().getStringList(config, "whitelist", Collections.<String>emptyList()); int cacheCapacity = getConfigs().getInt(config, "cacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(includes, excludes, cacheCapacity); validateArguments(); }
protected AbstractParser(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); List<String> mimeTypes = getConfigs().getStringList(config, SUPPORTED_MIME_TYPES, Collections.<String>emptyList()); for (String mimeType : mimeTypes) { addSupportedMimeType(mimeType); } this.numRecordsMeter = getMeter(Metrics.NUM_RECORDS); }
public OpenHdfsFile(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.conf = new Configuration(); String defaultFileSystemUri = getConfigs().getString(config, "fs", null); if (defaultFileSystemUri != null) { FileSystem.setDefaultUri(conf, defaultFileSystemUri); // see Hadoop's GenericOptionsParser } for (String value : getConfigs().getStringList(config, "conf", Collections.<String>emptyList())) { conf.addResource(new Path(value)); // see Hadoop's GenericOptionsParser } validateArguments(); }
public RemoveFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); List<String> includes = getConfigs().getStringList(config, "blacklist", Collections.singletonList("*")); List<String> excludes = getConfigs().getStringList(config, "whitelist", Collections.<String>emptyList()); int cacheCapacity = getConfigs().getInt(config, "cacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(includes, excludes, cacheCapacity); validateArguments(); }
protected AbstractParser(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); List<String> mimeTypes = getConfigs().getStringList(config, SUPPORTED_MIME_TYPES, Collections.<String>emptyList()); for (String mimeType : mimeTypes) { addSupportedMimeType(mimeType); } this.numRecordsMeter = getMeter(Metrics.NUM_RECORDS); }
public LogCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.format = getConfigs().getString(config, "format"); List<String> argList = getConfigs().getStringList(config, "args", Collections.<String>emptyList()); this.expressions = new FieldExpression[argList.size()]; for (int i = 0; i < argList.size(); i++) { this.expressions[i] = new FieldExpression(argList.get(i), getConfig()); } validateArguments(); }
public LogCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.format = getConfigs().getString(config, "format"); List<String> argList = getConfigs().getStringList(config, "args", Collections.<String>emptyList()); this.expressions = new FieldExpression[argList.size()]; for (int i = 0; i < argList.size(); i++) { this.expressions[i] = new FieldExpression(argList.get(i), getConfig()); } validateArguments(); }
public ReplaceValues(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context, boolean isRemoving) { super(builder, config, parent, child, context); List<String> nameBlacklist = getConfigs().getStringList(config, "nameBlacklist", Collections.singletonList("*")); List<String> nameWhitelist = getConfigs().getStringList(config, "nameWhitelist", Collections.<String>emptyList()); List<String> valueBlacklist = getConfigs().getStringList(config, "valueBlacklist", Collections.singletonList("*")); List<String> valueWhitelist = getConfigs().getStringList(config, "valueWhitelist", Collections.<String>emptyList()); int nameCacheCapacity = getConfigs().getInt(config, "nameCacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(nameBlacklist, nameWhitelist, nameCacheCapacity); int valueCacheCapacity = getConfigs().getInt(config, "valueCacheCapacity", 0); this.valueMatcher = new PatternNameMatcher(valueBlacklist, valueWhitelist, valueCacheCapacity); this.replacement = isRemoving ? null : getConfigs().getString(config, "replacement"); validateArguments(); }
public ReplaceValues(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context, boolean isRemoving) { super(builder, config, parent, child, context); List<String> nameBlacklist = getConfigs().getStringList(config, "nameBlacklist", Collections.singletonList("*")); List<String> nameWhitelist = getConfigs().getStringList(config, "nameWhitelist", Collections.<String>emptyList()); List<String> valueBlacklist = getConfigs().getStringList(config, "valueBlacklist", Collections.singletonList("*")); List<String> valueWhitelist = getConfigs().getStringList(config, "valueWhitelist", Collections.<String>emptyList()); int nameCacheCapacity = getConfigs().getInt(config, "nameCacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(nameBlacklist, nameWhitelist, nameCacheCapacity); int valueCacheCapacity = getConfigs().getInt(config, "valueCacheCapacity", 0); this.valueMatcher = new PatternNameMatcher(valueBlacklist, valueWhitelist, valueCacheCapacity); this.replacement = isRemoving ? null : getConfigs().getString(config, "replacement"); validateArguments(); }
public DownloadHdfsFile(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws IOException { super(builder, config, parent, child, context); List<String> uris = getConfigs().getStringList(config, "inputFiles", Collections.<String>emptyList()); File dstRootDir = new File(getConfigs().getString(config, "outputDir", ".")); Configuration conf = new Configuration(); String defaultFileSystemUri = getConfigs().getString(config, "fs", null); if (defaultFileSystemUri != null) { FileSystem.setDefaultUri(conf, defaultFileSystemUri); // see Hadoop's GenericOptionsParser } for (String value : getConfigs().getStringList(config, "conf", Collections.<String>emptyList())) { conf.addResource(new Path(value)); // see Hadoop's GenericOptionsParser } validateArguments(); download(uris, conf, dstRootDir); }
public Pipe(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.id = getConfigs().getString(config, "id"); List<String> importCommandSpecs = getConfigs().getStringList(config, "importCommands", Arrays.asList("com.**", "org.**", "net.**")); context.importCommandBuilders(importCommandSpecs); getConfigs().getConfigList(config, "commands", null); List<Command> childCommands = buildCommandChain(config, "commands", child, false); if (childCommands.size() > 0) { this.realChild = childCommands.get(0); } else { this.realChild = child; } validateArguments(); }
public Pipe(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.id = getConfigs().getString(config, "id"); List<String> importCommandSpecs = getConfigs().getStringList(config, "importCommands", Arrays.asList("com.**", "org.**", "net.**")); context.importCommandBuilders(importCommandSpecs); getConfigs().getConfigList(config, "commands", null); List<Command> childCommands = buildCommandChain(config, "commands", child, false); if (childCommands.size() > 0) { this.realChild = childCommands.get(0); } else { this.realChild = child; } validateArguments(); }
public Split(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.fieldName = getConfigs().getString(config, "field"); this.separator = getConfigs().getString(config, "separator"); this.newFields = getConfigs().getStringList(config, "newFields"); this.dropUndeclaredField = getConfigs().getBoolean(config, "dropUndeclaredField", true); this.trimSegments = getConfigs().getBoolean(config, "trimSegments", true); LOG.debug("fieldName: {}", fieldName); validateArguments(); }
for (String clazz : getConfigs().getStringList(config, "extensionFunctions", Collections.<String>emptyList())) { Object function; try {
protected RefererParser(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.uriInputField = getConfigs().getString(config, URI_INPUT_FIELD); this.pageHostInputField = getConfigs().getString(config, PAGE_HOST_INPUT_FIELD); this.sourceOutputFiled = getConfigs().getString(config, SOURCE_OUTPUT_FIELD, DEFAULT_SOURCE_OUTPUT_FIELD); this.mediumOutputField = getConfigs().getString(config, MEDIUM_OUTPUT_FIELD, DEFAULT_MEDIUM_OUTPUT_FIELD); this.termOutputField = getConfigs().getString(config, TERM_OUTPUT_FIELD, DEFAULT_TERM_OUTPUT_FIELD); this.campaignOutputField = getConfigs() .getString(config, CAMPAIGN_OUTPUT_FIELD, DEFAULT_CAMPAIGN_OUTPUT_FIELD); this.contentOutputField = getConfigs() .getString(config, CONTENT_OUTPUT_FIELD, DEFAULT_CONTENT_OUTPUT_FIELD); this.internalDomainsInputField = getConfigs().getStringList(config, INTERNAL_DOMAINS_INPUT_FIELD, Collections.<String>emptyList()); try { parser = new Parser(); } catch (IOException e) { throw new RefererParserException("Resources file must be provided.", e); } validateArguments(); }