/** * Creates and returns a new SparkRunner with specified options. * * @param options The PipelineOptions to use when executing the job. * @return A pipeline runner that will execute with specified options. */ public static SparkRunner fromOptions(PipelineOptions options) { SparkPipelineOptions sparkOptions = PipelineOptionsValidator.validate(SparkPipelineOptions.class, options); if (sparkOptions.getFilesToStage() == null) { sparkOptions.setFilesToStage( detectClassPathResourcesToStage(SparkRunner.class.getClassLoader())); LOG.info( "PipelineOptions.filesToStage was not specified. " + "Defaulting to files from the classpath: will stage {} files. " + "Enable logging at DEBUG level to see which files will be staged.", sparkOptions.getFilesToStage().size()); LOG.debug("Classpath elements: {}", sparkOptions.getFilesToStage()); } return new SparkRunner(sparkOptions); }
/** * Construct a runner from the provided options. * * @param options Properties which configure the runner. * @return The newly created runner. */ public static FlinkRunner fromOptions(PipelineOptions options) { FlinkPipelineOptions flinkOptions = PipelineOptionsValidator.validate(FlinkPipelineOptions.class, options); ArrayList<String> missing = new ArrayList<>(); if (flinkOptions.getAppName() == null) { missing.add("appName"); } if (missing.size() > 0) { throw new IllegalArgumentException( "Missing required values: " + Joiner.on(',').join(missing)); } if (flinkOptions.getFilesToStage() == null) { flinkOptions.setFilesToStage( detectClassPathResourcesToStage(FlinkRunner.class.getClassLoader())); LOG.info( "PipelineOptions.filesToStage was not specified. " + "Defaulting to files from the classpath: will stage {} files. " + "Enable logging at DEBUG level to see which files will be staged.", flinkOptions.getFilesToStage().size()); LOG.debug("Classpath elements: {}", flinkOptions.getFilesToStage()); } return new FlinkRunner(flinkOptions); }
@Test public void detectClassPathResourceWithNonFileResources() throws Exception { String url = "http://www.google.com/all-the-secrets.jar"; URLClassLoader classLoader = new URLClassLoader(new URL[] {new URL(url)}); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Unable to convert url (" + url + ") to file."); PipelineResources.detectClassPathResourcesToStage(classLoader); }
/** * Construct a runner from the provided options. * * @param options Properties which configure the runner. * @return The newly created runner. */ public static FlinkRunner fromOptions(PipelineOptions options) { options.setRunner(FlinkRunner.class); FlinkPipelineOptions flinkOptions = PipelineOptionsValidator.validate(FlinkPipelineOptions.class, options); ArrayList<String> missing = new ArrayList<>(); if (flinkOptions.getAppName() == null) { missing.add("appName"); } if (missing.size() > 0) { throw new IllegalArgumentException( "Missing required values: " + Joiner.on(',').join(missing)); } if (flinkOptions.getFilesToStage() == null) { flinkOptions.setFilesToStage( detectClassPathResourcesToStage(FlinkRunner.class.getClassLoader())); LOG.info( "PipelineOptions.filesToStage was not specified. " + "Defaulting to files from the classpath: will stage {} files. " + "Enable logging at DEBUG level to see which files will be staged.", flinkOptions.getFilesToStage().size()); LOG.debug("Classpath elements: {}", flinkOptions.getFilesToStage()); } return new FlinkRunner(flinkOptions); }
pathsToStage.addAll(detectClassPathResourcesToStage(PortableRunner.class.getClassLoader())); if (pathsToStage.isEmpty()) { throw new IllegalArgumentException("No classpath elements found.");
@Test public void detectClassPathResourceWithFileResources() throws Exception { File file = tmpFolder.newFile("file"); File file2 = tmpFolder.newFile("file2"); URLClassLoader classLoader = new URLClassLoader(new URL[] {file.toURI().toURL(), file2.toURI().toURL()}); assertEquals( ImmutableList.of(file.getAbsolutePath(), file2.getAbsolutePath()), PipelineResources.detectClassPathResourcesToStage(classLoader)); }
pipeline, flinkOptions, detectClassPathResourcesToStage(FlinkJobInvoker.class.getClassLoader()));
flinkOptions, serverConfig.flinkConfDir, detectClassPathResourcesToStage(FlinkJobInvoker.class.getClassLoader()));
@Test public void detectClassPathResourcesWithUnsupportedClassLoader() { ClassLoader mockClassLoader = Mockito.mock(ClassLoader.class); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Unable to use ClassLoader to detect classpath elements."); PipelineResources.detectClassPathResourcesToStage(mockClassLoader); }
detectClassPathResourcesToStage(DataflowRunner.class.getClassLoader())); if (dataflowOptions.getFilesToStage().isEmpty()) { throw new IllegalArgumentException("No files to stage has been found.");