Refine search
String morphlineId = context.getString(MORPHLINE_ID_PARAM); if (morphlineFile == null || morphlineFile.trim().length() == 0) { throw new MorphlineCompilationException("Missing parameter: " + MORPHLINE_FILE_PARAM, null);
public ToAvro(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); String schemaFile = getConfigs().getString(config, "schemaFile", null); String schemaString = getConfigs().getString(config, "schemaString", null); this.schemaField = getConfigs().getString(config, "schemaField", null); throw new MorphlineCompilationException( "Either schemaFile or schemaString or schemaField must be defined", config); throw new MorphlineCompilationException( "Must define only one of schemaFile or schemaString or schemaField at the same time", config); this.fixedSchema = new Parser().parse(new File(schemaFile)); } catch (IOException e) { throw new MorphlineCompilationException( "Cannot parse external Avro schema file: " + schemaFile, config, e);
public DetectMimeType(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws IOException, MimeTypeException { super(builder, config, parent, child, context); this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true); this.includeMetaData = getConfigs().getBoolean(config, "includeMetaData", false); this.excludeParameters = getConfigs().getBoolean(config, "excludeParameters", true); List<InputStream> inputStreams = new ArrayList(); try { this.detector = new DefaultDetector(mimeTypes, loader); } else { throw new MorphlineCompilationException("Missing specification for MIME type mappings", config);
public ExtractAvroPaths(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); ListMultimap<String, String> stepMultiMap = ArrayListMultimap.create(); this.flatten = getConfigs().getBoolean(config, "flatten", true); Config paths = getConfigs().getConfig(config, "paths"); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(paths)) { String fieldName = entry.getKey(); String path = entry.getValue().toString().trim(); if (path.contains("//")) { throw new MorphlineCompilationException("No support for descendant axis available yet", config);
for (String dictionaryResource : configs.getStringList(config, "dictionaryResources", Collections.<String>emptyList())) { ClassLoader classLoader = getClass().getClassLoader(); URL resource = classLoader.getResource(dictionaryResource); for (String dictionaryFile : configs.getStringList(config, "dictionaryFiles", Collections.<String>emptyList())) { LOG.debug("Loading grok dictionary:{} source:filesystem", dictionaryFile); loadDictionaryFile(new File(dictionaryFile)); String dictionaryString = configs.getString(config, "dictionaryString", ""); throw new MorphlineCompilationException("Cannot compile grok dictionary", config, e);
public ReadAvro(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); String schemaString = getConfigs().getString(config, "writerSchemaString", null); if (schemaString != null) { this.writerSchema = new Parser().parse(schemaString); } else { String schemaFile = getConfigs().getString(config, "writerSchemaFile", null); if (schemaFile != null) { try { this.writerSchema = new Parser().parse(new File(schemaFile)); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse external Avro writer schema file: " + schemaFile, config, e); } } else { this.writerSchema = null; } } this.isJson = getConfigs().getBoolean(config, "isJson", false); validateArguments(); }
public ReadProtobuf(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); String protoClassName = getConfigs().getString(config, "protobufClass"); try { protobufClass = Class.forName(protoClassName); } catch (ClassNotFoundException e) { throw new MorphlineCompilationException("Class not found", config, e); } String outputClassName = getConfigs().getString(config, "outputClass"); outputClass = getInnerClass(protobufClass, outputClassName); if (outputClass == null) { throw new MorphlineCompilationException("Output class '" + outputClassName + "' does not exist in class '" + protoClassName + "'.", config); } try { parseMethod = outputClass.getMethod(PARSE_FROM, InputStream.class); } catch (NoSuchMethodException e) { throw new MorphlineCompilationException("Method '" + PARSE_FROM + "' does not exist in class '" + outputClassName + "'.", config, e); } catch (SecurityException e) { throw new MorphlineCompilationException("Method '" + PARSE_FROM + "' is probably not public in class '" + outputClassName + "'.", config, e); } validateArguments(); }
public ReadLine(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.charset = getConfigs().getCharset(config, "charset", null); this.ignoreFirstLine = getConfigs().getBoolean(config, "ignoreFirstLine", false); String cprefix = getConfigs().getString(config, "commentPrefix", ""); if (cprefix.length() > 1) { throw new MorphlineCompilationException("commentPrefix must be at most one character long: " + cprefix, config); } this.commentPrefix = (cprefix.length() > 0 ? cprefix : null); validateArguments(); }
public RCFileColumn(Config columnConfig, Configuration conf) { this.conf = conf; Configs configs = new Configs(); this.inputField = configs.getInt(columnConfig, "inputField"); if (inputField < 0) { throw new MorphlineCompilationException( "Invalid column inputField specified: " + inputField, columnConfig); } this.outputField = configs.getString(columnConfig, "outputField"); String writableClassString = configs.getString(columnConfig, "writableClass"); if (writableClassString == null || writableClassString.isEmpty()) { throw new MorphlineCompilationException( "No writableClass specified for column " + outputField, columnConfig); } try { Class clazz = Class.forName(writableClassString); if (!Writable.class.isAssignableFrom(clazz)) { throw new MorphlineCompilationException("writableClass provided " + writableClassString + " for column " + outputField + " does not implement " + Writable.class.getName(), columnConfig); } this.writableClass = clazz; } catch (ClassNotFoundException e) { throw new MorphlineCompilationException("Could not load class " + writableClassString + " definition", columnConfig, e); } configs.validateArguments(columnConfig); }
public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.inputFieldName = getConfigs().getString(config, "inputField"); this.outputFieldName = getConfigs().getString(config, "outputField"); String solrFieldType = getConfigs().getString(config, "solrFieldType"); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); FieldType fieldType = schema.getFieldTypeByName(solrFieldType); if (fieldType == null) { throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config); } this.analyzer = fieldType.getIndexAnalyzer(); Preconditions.checkNotNull(analyzer); try { // register CharTermAttribute for later (implicit) reuse this.token = analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class); } catch (IOException e) { throw new MorphlineCompilationException("Cannot create token stream", config, e); } Preconditions.checkNotNull(token); validateArguments(); }
List<? extends Config> morphlineConfigs = config.getConfigList("morphlines"); if (morphlineConfigs.size() == 0) { throw new MorphlineCompilationException( "Morphline file must contain at least one morphline: " + nameForErrorMsg, null); } else { for (Config candidate : morphlineConfigs) { if (morphlineId.equals(new Configs().getString(candidate, "id", null))) { morphlineConfig = candidate; break; throw new MorphlineCompilationException( "Morphline id '" + morphlineId + "' not found in morphline file: " + nameForErrorMsg, null);
public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); String schemaString = getConfigs().getString(config, "readerSchemaString", null); if (schemaString != null) { this.readerSchema = new Parser().parse(schemaString); } else { String schemaFile = getConfigs().getString(config, "readerSchemaFile", null); if (schemaFile != null) { try { this.readerSchema = new Parser().parse(new File(schemaFile)); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e); } } else { this.readerSchema = null; } } if (getClass() == ReadAvroContainer.class) { resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>( getConfigs().getInt(config, "schemaCacheCapacity", 100)); validateArguments(); } else { resolverCache = null; } }
TimeUnit defaultDurationUnit = getConfigs().getTimeUnit(config, "defaultDurationUnit", TimeUnit.MILLISECONDS); TimeUnit defaultRateUnit = getConfigs().getTimeUnit(config, "defaultRateUnit", TimeUnit.SECONDS); long frequency = getConfigs().getNanoseconds(config, "frequency", 10 * 1000L * 1000 * 1000); // 10 seconds if (LOG.isTraceEnabled()) { LOG.trace("availableLocales: {}", Joiner.on("\n").join(Locale.getAvailableLocales())); outputDir.mkdirs(); if (!outputDir.isDirectory()) { throw new MorphlineCompilationException("Directory not found: " + outputDir, config); throw new MorphlineCompilationException("Directory not writeable: " + outputDir, config);
public FieldFilter(final CommandBuilder builder, final Config config, final Command parent, final Command child, final MorphlineContext context) { super(builder, config, parent, child, context); this.excludeFields = getConfigs().getStringList(config, CONF_EXCLUDE_FIELDS, DEFAULT_EXCLUDE_FIELDS); this.includeFields = getConfigs().getStringList(config, CONF_INCLUDE_FIELDS, DEFAULT_INCLUDE_FIELDS); if (excludeFields.isEmpty() && includeFields.isEmpty()) { throw new MorphlineCompilationException(String.format("Either %s or %s must be set", CONF_EXCLUDE_FIELDS, CONF_INCLUDE_FIELDS), config); } this.isRegex = getConfigs().getBoolean(config, CONF_IS_REGEX, DEFAULT_IS_REGEX); if (this.isRegex) { if (!excludeFields.isEmpty()) { excludePattern = Pattern.compile(String.format("(%s)", Joiner.on('|').join(excludeFields))); } else { excludePattern = Pattern.compile(".*"); } if (!includeFields.isEmpty()) { includePattern = Pattern.compile(String.format("(%s)", Joiner.on('|').join(includeFields))); } else { includePattern = Pattern.compile(".*"); } } else { includePattern = null; excludePattern = null; } }
public Sample(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.probability = getConfigs().getDouble(config, "probability", 1.0); if (probability < 0.0) { throw new MorphlineCompilationException("Probability must not be negative: " + probability, config); } if (probability >= 1.0) { this.prng = null; } else { if (config.hasPath("seed")) { long seed = getConfigs().getLong(config, "seed"); this.prng = new Well19937c(seed); // non-secure & fast } else { Random rand = new SecureRandom(); int[] seed = new int[624]; for (int i = 0; i < seed.length; i++) { seed[i] = rand.nextInt(); } this.prng = new Well19937c(seed); // non-secure & fast } } validateArguments(); }
public void validateArguments(Config config) { Set<String> recognizedArgs = getRecognizedArguments(); for (String key : config.root().keySet()) { if (!recognizedArgs.contains(key)) { throw new MorphlineCompilationException("Unrecognized command argument: " + key + ", recognized arguments: " + recognizedArgs, config); } } }
public Head(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.limit = getConfigs().getLong(config, "limit", -1); if (limit < -1) { throw new MorphlineCompilationException("Illegal limit: " + limit, config); } validateArguments(); }
public Locale getLocale(Config config, String path) { addRecognizedArgument(path); String str = config.getString(path); String[] parts = Iterables.toArray(Splitter.on('_').split(str), String.class); if (parts.length == 1) { return new Locale(parts[0]); } else if (parts.length == 2) { return new Locale(parts[0], parts[1]); } else if (parts.length == 3) { return new Locale(parts[0], parts[1], parts[2]); } else { throw new MorphlineCompilationException("Illegal locale: " + str, config); } }
@Test public void testReadAvroWithMissingExternalSchema() throws Exception { try { morphline = createMorphline("test-morphlines/readAvroWithMissingExternalSchema"); fail(); } catch (MorphlineCompilationException e) { assertTrue(e.getMessage().startsWith( "You must specify an external Avro writer schema because this is required to read containerless Avro")); } }
Assert.fail("failed to detect non-existing input file"); } catch (MorphlineCompilationException e) { Assert.assertTrue(e.getCause() instanceof FileNotFoundException);