morphline = new Compiler().compile( new File(morphlineFile), morphlineId, morphlineContext, finalChild, override);
@Override protected void setup(Context context) throws IOException, InterruptedException { File morphLineFile = new File(context.getConfiguration().get(MORPHLINE_FILE)); String morphLineId = context.getConfiguration().get(MORPHLINE_ID); RecordEmitter recordEmitter = new RecordEmitter(context); MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); morphline = new org.kitesdk.morphline.base.Compiler() .compile(morphLineFile, morphLineId, morphlineContext, recordEmitter); }
/** * Parses the given morphlineFile, then finds the morphline with the given morphlineId within, * then compiles the morphline and returns the corresponding morphline command. The returned * command will feed records into finalChild. */ public Command compile(File morphlineFile, String morphlineId, MorphlineContext morphlineContext, Command finalChild, Config... overrides) { Config config; try { config = parse(morphlineFile, overrides); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse morphline file: " + morphlineFile, null, e); } Config morphlineConfig = find(morphlineId, config, morphlineFile.getPath()); Command morphlineCommand = compile(morphlineConfig, morphlineContext, finalChild); return morphlineCommand; }
/** * Parses the given morphlineFile, then finds the morphline with the given morphlineId within, * then compiles the morphline and returns the corresponding morphline command. The returned * command will feed records into finalChild. */ public Command compile(File morphlineFile, String morphlineId, MorphlineContext morphlineContext, Command finalChild, Config... overrides) { Config config; try { config = parse(morphlineFile, overrides); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse morphline file: " + morphlineFile, null, e); } Config morphlineConfig = find(morphlineId, config, morphlineFile.getPath()); Command morphlineCommand = compile(morphlineConfig, morphlineContext, finalChild); return morphlineCommand; }
morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, null, override); morphlineFileAndId = morphlineFile + "@" + morphlineId;
private Command createMorphline(String file, Config... overrides) { return new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), null, createMorphlineContext(), null, overrides); }
morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, null, override); morphlineFileAndId = morphlineFile + "@" + morphlineId;
private Command createMorphline(String file) { return new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), null, createMorphlineContext(), collector); }
this.morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, collector, override);
this.morphline = new Compiler().compile(new File(morphlineFile), morphlineId, morphlineContext, collector, override);
morphline = new Compiler().compile( new File(morphlineFile), morphlineId,
Files.write(morphlineFileContents, morphlineTmpFile, Charsets.UTF_8); collector = new Collector(); morphline = new Compiler().compile(morphlineTmpFile, morphlineId, morphlineContext, collector, override); } catch (Exception e) { throw new CrunchRuntimeException(e);
morphline = new Compiler().compile( new File(morphlineFile), morphlineId, morphlineContext, finalChild, override);
@Test public void testCompileWithExplicitMorphlineId() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; morphline = new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline1", new MorphlineContext.Builder().build(), null); assertNotNull(morphline); }
@Test public void testCompileWithUnknownMorphlineId() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; try { new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline2", new MorphlineContext.Builder().build(), null); fail(); } catch (MorphlineCompilationException e) { ; // expected } }
@Test public void testCompileWithMissingMorphline() throws Exception { String file = "test-morphlines/compileWithMissingMorphline"; try { new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "morphline1", new MorphlineContext.Builder().build(), null); fail(); } catch (MorphlineCompilationException e) { ; // expected } }
/** Usage: java ... <morphline.conf> <dataFile1> ... <dataFileN> */ public static void main(String[] args) throws IOException { // compile morphline.conf file on the fly File morphlineFile = new File(args[0]); String morphlineId = null; MorphlineContext morphlineContext = new MorphlineContext.Builder().build(); Command morphline = new Compiler().compile(morphlineFile, morphlineId, morphlineContext, null); // process each input data file Notifications.notifyBeginTransaction(morphline); try { for (int i = 1; i < args.length; i++) { InputStream in = new BufferedInputStream(new FileInputStream(new File(args[i]))); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); Notifications.notifyStartSession(morphline); boolean success = morphline.process(record); if (!success) { System.out.println("Morphline failed to process record: " + record); } in.close(); } Notifications.notifyCommitTransaction(morphline); } catch (RuntimeException e) { Notifications.notifyRollbackTransaction(morphline); morphlineContext.getExceptionHandler().handleException(e, null); } Notifications.notifyShutdown(morphline); } }
@Test public void testCompile() throws Exception { String file = "test-morphlines/pipeWithTwoBasicCommands"; morphline = new Compiler().compile( new File(RESOURCES_DIR + "/" + file + ".conf"), "", new MorphlineContext.Builder().build(), null); assertNotNull(morphline); new Fields(); new Metrics(); }