morphline = new Compiler().compile( new File(morphlineFile), morphlineId, morphlineContext, finalChild, override);
@Override protected void setup(Context context) throws IOException, InterruptedException { File morphLineFile = new File(context.getConfiguration().get(MORPHLINE_FILE)); String morphLineId = context.getConfiguration().get(MORPHLINE_ID); RecordEmitter recordEmitter = new RecordEmitter(context); MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); morphline = new org.kitesdk.morphline.base.Compiler() .compile(morphLineFile, morphLineId, morphlineContext, recordEmitter); }
protected Config parse(String file, Config... overrides) throws IOException { Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), overrides); config = config.getConfigList("morphlines").get(0); Preconditions.checkNotNull(config); return config; }
morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, null, override); morphlineFileAndId = morphlineFile + "@" + morphlineId;
private Command createMorphline(String file, Config... overrides) { return new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), null, createMorphlineContext(), null, overrides); }
private Command createMorphline(String file) { return new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), null, createMorphlineContext(), collector); }
morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, null, override); morphlineFileAndId = morphlineFile + "@" + morphlineId;
this.morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, collector, override);
this.morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, collector, override);
Files.write(morphlineFileContents, morphlineTmpFile, Charsets.UTF_8); collector = new Collector(); morphline = new Compiler().compile(morphlineTmpFile, morphlineId, morphlineContext, collector, override); } catch (Exception e) { throw new CrunchRuntimeException(e);
morphline = new Compiler().compile( new File(morphlineFile), morphlineId,
private Config parse(String file) throws IOException { SolrLocator locator = new SolrLocator(createMorphlineContext()); locator.setSolrHomeDir(testSolrHome + "/collection1"); File morphlineFile; if (new File(file).isAbsolute()) { morphlineFile = new File(file + ".conf"); } else { morphlineFile = new File(RESOURCES_DIR + "/" + file + ".conf"); } Config config = new Compiler().parse(morphlineFile, locator.toConfig("SOLR_LOCATOR")); config = config.getConfigList("morphlines").get(0); return config; }
morphline = new Compiler().compile( new File(morphlineFile), morphlineId, morphlineContext, finalChild, override);
@Test public void testCompileWithExplicitMorphlineId() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; morphline = new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline1", new MorphlineContext.Builder().build(), null); assertNotNull(morphline); }
@Test public void testCompileWithMissingMorphline() throws Exception { String file = "test-morphlines/compileWithMissingMorphline"; try { new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline1", new MorphlineContext.Builder().build(), null); fail(); } catch (MorphlineCompilationException e) { ; // expected } }
@Test public void testCompileWithUnknownMorphlineId() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; try { new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline2", new MorphlineContext.Builder().build(), null); fail(); } catch (MorphlineCompilationException e) { ; // expected } }
/** Usage: java ... <morphline.conf> <dataFile1> ... <dataFileN> */ public static void main(String[] args) throws IOException { // compile morphline.conf file on the fly File morphlineFile = new File(args[0]); String morphlineId = null; MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null); // process each input data file Notifications.notifyBeginTransaction(morphline); try { for (int i = 1; i < args.length; i++) { InputStream in = new BufferedInputStream(new FileInputStream(new File(args[i]))); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); Notifications.notifyStartSession(morphline); boolean success = morphline.process(record); if (!success) { System.out.println("Morphline failed to process record: " + record); } in.close(); } Notifications.notifyCommitTransaction(morphline); } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); morphlineContext.getExceptionHandler().handleException(e, null); } Notifications.notifyShutdown(morphline); } }
@Test public void testCompile() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; morphline = new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "", new MorphlineContext.Builder().build(), null); assertNotNull(morphline); new Fields(); new Metrics(); }
protected Command parse(String file, String collection) throws IOException { SolrLocator locator = new SolrLocator(createMorphlineContext()); locator.setCollectionName(collection); locator.setZkHost(cluster.getZkServer().getZkAddress()); //locator.setServerUrl(cloudJettys.get(0).url); // TODO: download IndexSchema from solrUrl not yet implemented //locator.setSolrHomeDir(SOLR_HOME_DIR.getPath()); Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), locator.toConfig("SOLR_LOCATOR")); config = config.getConfigList("morphlines").get(0); return createMorphline(config); }