/** * Parses the given morphlineFile, then finds the morphline with the given morphlineId within, * then compiles the morphline and returns the corresponding morphline command. The returned * command will feed records into finalChild. */ public Command compile(File morphlineFile, String morphlineId, MorphlineContext morphlineContext, Command finalChild, Config... overrides) { Config config; try { config = parse(morphlineFile, overrides); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse morphline file: " + morphlineFile, null, e); } Config morphlineConfig = find(morphlineId, config, morphlineFile.getPath()); Command morphlineCommand = compile(morphlineConfig, morphlineContext, finalChild); return morphlineCommand; }
private Config parse(String file) throws IOException { SolrLocator locator = new SolrLocator(createMorphlineContext()); locator.setSolrHomeDir(testSolrHome + "/collection1"); File morphlineFile; if (new File(file).isAbsolute()) { morphlineFile = new File(file + ".conf"); } else { morphlineFile = new File(RESOURCES_DIR + "/" + file + ".conf"); } Config config = new Compiler().parse(morphlineFile, locator.toConfig("SOLR_LOCATOR")); config = config.getConfigList("morphlines").get(0); return config; }
/** * Parses the given morphlineFile, then finds the morphline with the given morphlineId within, * then compiles the morphline and returns the corresponding morphline command. The returned * command will feed records into finalChild. */ public Command compile(File morphlineFile, String morphlineId, MorphlineContext morphlineContext, Command finalChild, Config... overrides) { Config config; try { config = parse(morphlineFile, overrides); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse morphline file: " + morphlineFile, null, e); } Config morphlineConfig = find(morphlineId, config, morphlineFile.getPath()); Command morphlineCommand = compile(morphlineConfig, morphlineContext, finalChild); return morphlineCommand; }
@Override protected void setup(Context context) throws IOException, InterruptedException { File morphLineFile = new File(context.getConfiguration().get(MORPHLINE_FILE)); String morphLineId = context.getConfiguration().get(MORPHLINE_ID); RecordEmitter recordEmitter = new RecordEmitter(context); MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); morphline = new org.kitesdk.morphline.base.Compiler() .compile(morphLineFile, morphLineId, morphlineContext, recordEmitter); }
protected Command parse(String file, String collection) throws IOException { SolrLocator locator = new SolrLocator(createMorphlineContext()); locator.setCollectionName(collection); locator.setZkHost(cluster.getZkServer().getZkAddress()); //locator.setServerUrl(cloudJettys.get(0).url); // TODO: download IndexSchema from solrUrl not yet implemented //locator.setSolrHomeDir(SOLR_HOME_DIR.getPath()); Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), locator.toConfig("SOLR_LOCATOR")); config = config.getConfigList("morphlines").get(0); return createMorphline(config); }
@Test public void testCompileWithExplicitMorphlineId() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; morphline = new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline1", new MorphlineContext.Builder().build(), null); assertNotNull(morphline); }
@Test public void testCompileWithMissingMorphline() throws Exception { String file = "test-morphlines/compileWithMissingMorphline"; try { new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline1", new MorphlineContext.Builder().build(), null); fail(); } catch (MorphlineCompilationException e) { ; // expected } }
@Test public void testCompileWithUnknownMorphlineId() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; try { new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline2", new MorphlineContext.Builder().build(), null); fail(); } catch (MorphlineCompilationException e) { ; // expected } }
@Test public void testCompile() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; morphline = new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "", new MorphlineContext.Builder().build(), null); assertNotNull(morphline); new Fields(); new Metrics(); }