public SolrLocator(Config config, MorphlineContext context) { this(context); this.config = config; Configs configs = new Configs(); collectionName = configs.getString(config, "collection", null); zkHost = configs.getString(config, "zkHost", null); solrHomeDir = configs.getString(config, "solrHomeDir", null); solrUrl = configs.getString(config, "solrUrl", null); batchSize = configs.getInt(config, "batchSize", batchSize); zkClientSessionTimeout = configs.getInt(config, "zkClientSessionTimeout", zkClientSessionTimeout); zkClientConnectTimeout = configs.getInt(config, "zkClientConnectTimeout", zkClientConnectTimeout); LOG.trace("Constructed solrLocator: {}", this); configs.validateArguments(config); }
public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); String schemaString = getConfigs().getString(config, "readerSchemaString", null); if (schemaString != null) { this.readerSchema = new Parser().parse(schemaString); } else { String schemaFile = getConfigs().getString(config, "readerSchemaFile", null); if (schemaFile != null) { try { this.readerSchema = new Parser().parse(new File(schemaFile)); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e); } } else { this.readerSchema = null; } } if (getClass() == ReadAvroContainer.class) { resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>( getConfigs().getInt(config, "schemaCacheCapacity", 100)); validateArguments(); } else { resolverCache = null; } }
public RemoveFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); List<String> includes = getConfigs().getStringList(config, "blacklist", Collections.singletonList("*")); List<String> excludes = getConfigs().getStringList(config, "whitelist", Collections.<String>emptyList()); int cacheCapacity = getConfigs().getInt(config, "cacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(includes, excludes, cacheCapacity); validateArguments(); }
public RemoveFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); List<String> includes = getConfigs().getStringList(config, "blacklist", Collections.singletonList("*")); List<String> excludes = getConfigs().getStringList(config, "whitelist", Collections.<String>emptyList()); int cacheCapacity = getConfigs().getInt(config, "cacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(includes, excludes, cacheCapacity); validateArguments(); }
public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); String schemaString = getConfigs().getString(config, "readerSchemaString", null); if (schemaString != null) { this.readerSchema = new Parser().parse(schemaString); } else { String schemaFile = getConfigs().getString(config, "readerSchemaFile", null); if (schemaFile != null) { try { this.readerSchema = new Parser().parse(new File(schemaFile)); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e); } } else { this.readerSchema = null; } } if (getClass() == ReadAvroContainer.class) { resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>( getConfigs().getInt(config, "schemaCacheCapacity", 100)); validateArguments(); } else { resolverCache = null; } }
public SolrLocator(Config config, MorphlineContext context) { this(context); this.config = config; Configs configs = new Configs(); collectionName = configs.getString(config, "collection", null); zkHost = configs.getString(config, "zkHost", null); solrHomeDir = configs.getString(config, "solrHomeDir", null); solrUrl = configs.getString(config, "solrUrl", null); batchSize = configs.getInt(config, "batchSize", batchSize); LOG.trace("Constructed solrLocator: {}", this); configs.validateArguments(config); }
public SolrLocator(Config config, MorphlineContext context) { this(context); this.config = config; Configs configs = new Configs(); collectionName = configs.getString(config, "collection", null); zkHost = configs.getString(config, "zkHost", null); solrHomeDir = configs.getString(config, "solrHomeDir", null); solrUrl = configs.getString(config, "solrUrl", null); batchSize = configs.getInt(config, "batchSize", batchSize); LOG.trace("Constructed solrLocator: {}", this); configs.validateArguments(config); }
public ReplaceValues(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context, boolean isRemoving) { super(builder, config, parent, child, context); List<String> nameBlacklist = getConfigs().getStringList(config, "nameBlacklist", Collections.singletonList("*")); List<String> nameWhitelist = getConfigs().getStringList(config, "nameWhitelist", Collections.<String>emptyList()); List<String> valueBlacklist = getConfigs().getStringList(config, "valueBlacklist", Collections.singletonList("*")); List<String> valueWhitelist = getConfigs().getStringList(config, "valueWhitelist", Collections.<String>emptyList()); int nameCacheCapacity = getConfigs().getInt(config, "nameCacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(nameBlacklist, nameWhitelist, nameCacheCapacity); int valueCacheCapacity = getConfigs().getInt(config, "valueCacheCapacity", 0); this.valueMatcher = new PatternNameMatcher(valueBlacklist, valueWhitelist, valueCacheCapacity); this.replacement = isRemoving ? null : getConfigs().getString(config, "replacement"); validateArguments(); }
public ReplaceValues(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context, boolean isRemoving) { super(builder, config, parent, child, context); List<String> nameBlacklist = getConfigs().getStringList(config, "nameBlacklist", Collections.singletonList("*")); List<String> nameWhitelist = getConfigs().getStringList(config, "nameWhitelist", Collections.<String>emptyList()); List<String> valueBlacklist = getConfigs().getStringList(config, "valueBlacklist", Collections.singletonList("*")); List<String> valueWhitelist = getConfigs().getStringList(config, "valueWhitelist", Collections.<String>emptyList()); int nameCacheCapacity = getConfigs().getInt(config, "nameCacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(nameBlacklist, nameWhitelist, nameCacheCapacity); int valueCacheCapacity = getConfigs().getInt(config, "valueCacheCapacity", 0); this.valueMatcher = new PatternNameMatcher(valueBlacklist, valueWhitelist, valueCacheCapacity); this.replacement = isRemoving ? null : getConfigs().getString(config, "replacement"); validateArguments(); }
public RCFileColumn(Config columnConfig, Configuration conf) { this.conf = conf; Configs configs = new Configs(); this.inputField = configs.getInt(columnConfig, "inputField"); if (inputField < 0) { throw new MorphlineCompilationException( "Invalid column inputField specified: " + inputField, columnConfig); } this.outputField = configs.getString(columnConfig, "outputField"); String writableClassString = configs.getString(columnConfig, "writableClass"); if (writableClassString == null || writableClassString.isEmpty()) { throw new MorphlineCompilationException( "No writableClass specified for column " + outputField, columnConfig); } try { Class clazz = Class.forName(writableClassString); if (!Writable.class.isAssignableFrom(clazz)) { throw new MorphlineCompilationException("writableClass provided " + writableClassString + " for column " + outputField + " does not implement " + Writable.class.getName(), columnConfig); } this.writableClass = clazz; } catch (ClassNotFoundException e) { throw new MorphlineCompilationException("Could not load class " + writableClassString + " definition", columnConfig, e); } configs.validateArguments(columnConfig); }
public RCFileColumn(Config columnConfig, Configuration conf) { this.conf = conf; Configs configs = new Configs(); this.inputField = configs.getInt(columnConfig, "inputField"); if (inputField < 0) { throw new MorphlineCompilationException( "Invalid column inputField specified: " + inputField, columnConfig); } this.outputField = configs.getString(columnConfig, "outputField"); String writableClassString = configs.getString(columnConfig, "writableClass"); if (writableClassString == null || writableClassString.isEmpty()) { throw new MorphlineCompilationException( "No writableClass specified for column " + outputField, columnConfig); } try { Class clazz = Class.forName(writableClassString); if (!Writable.class.isAssignableFrom(clazz)) { throw new MorphlineCompilationException("writableClass provided " + writableClassString + " for column " + outputField + " does not implement " + Writable.class.getName(), columnConfig); } this.writableClass = clazz; } catch (ClassNotFoundException e) { throw new MorphlineCompilationException("Could not load class " + writableClassString + " definition", columnConfig, e); } configs.validateArguments(columnConfig); }
public RetryPolicyFactory parse(Config retryPolicyConfig) { final Config boundedExponentialBackoffRetryConfig = new Configs().getConfig(retryPolicyConfig, BOUNDED_EXPONENTIAL_BACKOFF_RETRY_NAME); final long baseSleepTime = new Configs() .getNanoseconds(boundedExponentialBackoffRetryConfig, "baseSleepTime", TimeUnit.MILLISECONDS.toNanos(125)); final long maxSleepTime = new Configs() .getNanoseconds(boundedExponentialBackoffRetryConfig, "maxSleepTime", TimeUnit.SECONDS.toNanos(10)); final int maxRetries = new Configs() .getInt(boundedExponentialBackoffRetryConfig, "maxRetries", 100); final long maxElapsedTime = new Configs() .getNanoseconds(boundedExponentialBackoffRetryConfig, "maxElapsedTime", TimeUnit.SECONDS.toNanos(3 * 60)); LOG.debug("Parsed retry policy BoundedExponentialBackoffRetry with " + "baseSleepTime:{}, maxSleepTime:{}, maxRetries:{}, maxElapsedTime:{}", new Object[]{baseSleepTime, maxSleepTime, maxRetries, maxElapsedTime}); if (maxRetries <= 0 || maxElapsedTime <= 0) { return null; } return new DefaultRetryPolicyFactory( new FlexibleBoundedExponentialBackoffRetry(baseSleepTime, maxSleepTime, maxRetries, maxElapsedTime)); }
public ExtractURIQueryParameters(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.parameterName = getConfigs().getString(config, "parameter"); this.inputFieldName = getConfigs().getString(config, "inputField"); this.outputFieldName = getConfigs().getString(config, "outputField"); this.failOnInvalidURI = getConfigs().getBoolean(config, "failOnInvalidURI", false); this.maxParameters = getConfigs().getInt(config, "maxParameters", Integer.MAX_VALUE); this.charset = getConfigs().getString(config, "charset", "UTF-8"); Charset.forName(charset); // fail fast if charset is unsupported validateArguments(); }
public ExtractURIQueryParameters(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.parameterName = getConfigs().getString(config, "parameter"); this.inputFieldName = getConfigs().getString(config, "inputField"); this.outputFieldName = getConfigs().getString(config, "outputField"); this.failOnInvalidURI = getConfigs().getBoolean(config, "failOnInvalidURI", false); this.maxParameters = getConfigs().getInt(config, "maxParameters", Integer.MAX_VALUE); this.charset = getConfigs().getString(config, "charset", "UTF-8"); Charset.forName(charset); // fail fast if charset is unsupported validateArguments(); }
this.inputFieldName = getConfigs().getString(config, "inputField"); String databaseFile = getConfigs().getString(config, "database", null); int cacheCapacity = getConfigs().getInt(config, "cacheCapacity", 1000); String nullReplacement = getConfigs().getString(config, "nullReplacement", "");
int limit = getConfigs().getInt(config, "limit", -1); if (limit > 0) { currentSplitter = currentSplitter.limit(limit);
int limit = getConfigs().getInt(config, "limit", -1); if (limit > 0) { currentSplitter = currentSplitter.limit(limit);
"Comment prefix must not have a length of more than one character: " + commentPrefix, config); this.maxCharactersPerRecord = getConfigs().getInt(config, "maxCharactersPerRecord", 1000 * 1000); this.ignoreTooLongRecords = new Validator<OnMaxCharactersPerRecord>().validateEnum( config,
"Comment prefix must not have a length of more than one character: " + commentPrefix, config); this.maxCharactersPerRecord = getConfigs().getInt(config, "maxCharactersPerRecord", 1000 * 1000); this.ignoreTooLongRecords = new Validator<OnMaxCharactersPerRecord>().validateEnum( config,
public CopyTestCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.name = getConfigs().getString(config, "name"); this.count = getConfigs().getInt(config, "count", 2); getCounter("counter1"); getHistogram("histogram1"); getMeter("meter1"); getTimer("timer1"); }