@OnScheduled protected void setDefaultConfiguration(ProcessContext context) throws IOException { DefaultConfiguration.set(getConfiguration( context.getProperty(CONF_XML_FILES).evaluateAttributeExpressions().getValue())); }
protected static Configuration getConfiguration(String configFiles) { Configuration conf = DefaultConfiguration.get(); if (configFiles == null || configFiles.isEmpty()) { return conf; } for (String file : COMMA.split(configFiles)) { // process each resource only once if (conf.getResource(file) == null) { // use Path instead of String to get the file from the FS conf.addResource(new Path(file)); } } return conf; }
private static void addToConfiguration(Properties hadoopConfiguration) { // base the new Configuration on the current defaults Configuration conf = new Configuration(DefaultConfiguration.get()); // add all of the properties as config settings for (String key : hadoopConfiguration.stringPropertyNames()) { String value = hadoopConfiguration.getProperty(key); conf.set(key, value); } // replace the original Configuration DefaultConfiguration.set(conf); addedConf = true; }
DefaultConfiguration jooqConfiguration = new DefaultConfiguration(); jooqConfiguration.set(connectionProvider()); jooqConfiguration.set(new DefaultExecuteListenerProvider(jooqToSpringExceptionTransformer())); jooqConfiguration.set(dialect);
@Override @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR", justification="Delegate set by setConf") public RecordReader<E, Void> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { Configuration conf = Hadoop.TaskAttemptContext.getConfiguration.invoke(taskAttemptContext); DefaultConfiguration.init(conf); return delegate.createRecordReader(inputSplit, taskAttemptContext); }
@Test public void testFindsHDFS() throws Exception { // set the default configuration that the loader will use Configuration existing = DefaultConfiguration.get(); DefaultConfiguration.set(getConfiguration()); FileSystemDataset<GenericRecord> dataset = Datasets.load("dataset:hdfs:/tmp/datasets/ns/strings"); Assert.assertNotNull("Dataset should be found", dataset); Assert.assertEquals("Dataset should be located in HDFS", "hdfs", dataset.getFileSystem().getUri().getScheme()); // replace the original config so the other tests are not affected DefaultConfiguration.set(existing); }
@Override @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR", justification="Delegate set by setConf") public RecordReader<E, Void> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { Configuration conf = Hadoop.TaskAttemptContext.getConfiguration.invoke(taskAttemptContext); DefaultConfiguration.init(conf); return delegate.createRecordReader(inputSplit, taskAttemptContext); }
try { inputSchema = getSchema(inputSchemaProperty, DefaultConfiguration.get()); } catch (SchemaNotFoundException e) { getLogger().error("Cannot find schema: " + inputSchemaProperty); try { outputSchema = getSchema(outputSchemaProperty, DefaultConfiguration.get()); } catch (SchemaNotFoundException e) { getLogger().error("Cannot find schema: " + outputSchemaProperty);
@SuppressWarnings({"deprecation", "unchecked"}) private static <E> View<E> load(Configuration conf) { Class<E> type; try { type = (Class<E>)conf.getClass(KITE_TYPE, GenericData.Record.class); } catch (RuntimeException e) { if (e.getCause() instanceof ClassNotFoundException) { throw new TypeNotFoundException(String.format( "The Java class %s for the entity type could not be found", conf.get(KITE_TYPE)), e.getCause()); } else { throw e; } } DefaultConfiguration.set(conf); String schemaStr = conf.get(KITE_READER_SCHEMA); Schema projection = null; if (schemaStr != null) { projection = new Schema.Parser().parse(schemaStr); } String inputUri = conf.get(KITE_INPUT_URI); if (projection != null) { return Datasets.load(inputUri).asSchema(projection).asType(type); } else { return Datasets.load(inputUri, type); } }
@Override public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) { Configuration conf = Hadoop.TaskAttemptContext.getConfiguration.invoke(taskAttemptContext); DefaultConfiguration.init(conf); View<E> view = load(taskAttemptContext); return usePerTaskAttemptDatasets(view, conf) ? new MergeOutputCommitter<E>() : new NullOutputCommitter(); }
final Schema schema; try { schema = getSchema(schemaProperty, DefaultConfiguration.get()); } catch (SchemaNotFoundException e) { getLogger().error("Cannot find schema: " + schemaProperty);
/** * Starts the services in order, passing the previous service's modified * Configuration object to the next. * * @throws IOException */ public void start() throws IOException, InterruptedException { for (Service service : services) { service.configure(serviceConfig); logger.info("Running Minicluster Service: " + service.getClass().getName()); service.start(); serviceConfig.setHadoopConf(service.getHadoopConf()); // set the default configuration so that the minicluster is used DefaultConfiguration.set(serviceConfig.getHadoopConf()); } logger.info("All Minicluster Services running."); }
@Override public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) { Configuration conf = Hadoop.TaskAttemptContext.getConfiguration.invoke(taskAttemptContext); DefaultConfiguration.init(conf); View<E> view = load(taskAttemptContext); return usePerTaskAttemptDatasets(view) ? new MergeOutputCommitter<E>() : new NullOutputCommitter(); }
final Schema schema; try { schema = getSchema(schemaProperty, DefaultConfiguration.get()); } catch (SchemaNotFoundException e) { getLogger().error("Cannot find schema: " + schemaProperty);
@AfterClass public static void restoreDefaultConfiguration() { DefaultConfiguration.set(original); }
protected Configuration getConf() { if (!addedConf) { addToConfiguration(hadoopConfiguration); } // use the default return DefaultConfiguration.get(); }
@OnScheduled protected void setDefaultConfiguration(ProcessContext context) throws IOException { DefaultConfiguration.set(getConfiguration( context.getProperty(CONF_XML_FILES).evaluateAttributeExpressions().getValue())); }
public static Schema fromJSON(String name, File location) throws IOException { return fromJSON(name, FileSystem.getLocal(DefaultConfiguration.get()), new Path(location.getPath())); }
@Override public int run(String[] args) throws Exception { if (getConf() != null) { DefaultConfiguration.set(getConf());
public static Schema fromAvsc(File location) throws IOException { return fromAvsc( FileSystem.getLocal(DefaultConfiguration.get()), new Path(location.getPath())); }