protected static Configuration getConfiguration(String configFiles) { Configuration conf = DefaultConfiguration.get(); if (configFiles == null || configFiles.isEmpty()) { return conf; } for (String file : COMMA.split(configFiles)) { // process each resource only once if (conf.getResource(file) == null) { // use Path instead of String to get the file from the FS conf.addResource(new Path(file)); } } return conf; }
try { inputSchema = getSchema(inputSchemaProperty, DefaultConfiguration.get()); } catch (SchemaNotFoundException e) { getLogger().error("Cannot find schema: " + inputSchemaProperty); try { outputSchema = getSchema(outputSchemaProperty, DefaultConfiguration.get()); } catch (SchemaNotFoundException e) { getLogger().error("Cannot find schema: " + outputSchemaProperty);
final Schema schema; try { schema = getSchema(schemaProperty, DefaultConfiguration.get()); } catch (SchemaNotFoundException e) { getLogger().error("Cannot find schema: " + schemaProperty);
final Schema schema; try { schema = getSchema(schemaProperty, DefaultConfiguration.get()); } catch (SchemaNotFoundException e) { getLogger().error("Cannot find schema: " + schemaProperty);
public static Schema fromParquet(File location) throws IOException { return fromParquet( FileSystem.getLocal(DefaultConfiguration.get()), new Path(location.getPath())); }
public static Schema fromJSON(String name, File location) throws IOException { return fromJSON(name, FileSystem.getLocal(DefaultConfiguration.get()), new Path(location.getPath())); }
protected Configuration getConf() { if (!addedConf) { addToConfiguration(hadoopConfiguration); } // use the default return DefaultConfiguration.get(); }
public static Schema fromAvsc(File location) throws IOException { return fromAvsc( FileSystem.getLocal(DefaultConfiguration.get()), new Path(location.getPath())); }
public static Schema fromAvro(File location) throws IOException { return fromAvro( FileSystem.getLocal(DefaultConfiguration.get()), new Path(location.getPath())); }
private static void addToConfiguration(Properties hadoopConfiguration) { // base the new Configuration on the current defaults Configuration conf = new Configuration(DefaultConfiguration.get()); // add all of the properties as config settings for (String key : hadoopConfiguration.stringPropertyNames()) { String value = hadoopConfiguration.getProperty(key); conf.set(key, value); } // replace the original Configuration DefaultConfiguration.set(conf); addedConf = true; }
protected static Configuration getConfiguration(String configFiles) { Configuration conf = DefaultConfiguration.get(); if (configFiles == null || configFiles.isEmpty()) { return conf; } for (String file : COMMA.split(configFiles)) { // process each resource only once if (conf.getResource(file) == null) { // use Path instead of String to get the file from the FS conf.addResource(new Path(file)); } } return conf; }
public Builder() { this.properties = Maps.newHashMap(); this.conf = DefaultConfiguration.get(); try { this.defaultFS = FileSystem.get(conf).getUri(); } catch (IOException e) { throw new DatasetIOException("Cannot get the default FS", e); } }
@Override public DatasetRepository getFromOptions(Map<String, String> options) { Configuration conf = HBaseConfiguration.create(DefaultConfiguration.get()); String[] hostsAndPort = parseHostsAndPort(options.get("zk")); conf.set(HConstants.ZOOKEEPER_QUORUM, hostsAndPort[0]); String port = hostsAndPort[1]; if (port != null) { conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, port); } return new HBaseDatasetRepository.Builder().configuration(conf).build(); } });
@Override public DatasetRepository getFromOptions(Map<String, String> options) { Configuration conf = HBaseConfiguration.create(DefaultConfiguration.get()); String[] hostsAndPort = parseHostsAndPort(options.get("zk")); conf.set(HConstants.ZOOKEEPER_QUORUM, hostsAndPort[0]); String port = hostsAndPort[1]; if (port != null) { conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, port); } return new HBaseDatasetRepository.Builder().configuration(conf).build(); } });
@Override public DatasetRepository getFromOptions(Map<String, String> match) { LOG.debug("Managed URI options: {}", match); // make a modifiable copy and setup the MetaStore URI Configuration conf = newHiveConf(DefaultConfiguration.get()); // sanity check the URI setMetaStoreURI(conf, match); return new HiveManagedDatasetRepository.Builder() .configuration(conf) .build(); } }
@Override public DatasetRepository getFromOptions(Map<String, String> match) { LOG.debug("Managed URI options: {}", match); // make a modifiable copy and setup the MetaStore URI Configuration conf = newHiveConf(DefaultConfiguration.get()); // sanity check the URI setMetaStoreURI(conf, match); return new HiveManagedDatasetRepository.Builder() .configuration(conf) .build(); } }
/** * Build an instance of the configured {@link HiveManagedDatasetRepository}. * * @since 0.9.0 */ @SuppressWarnings("deprecation") public DatasetRepository build() { if (configuration == null) { this.configuration = DefaultConfiguration.get(); } if (rootDirectory != null) { return new HiveExternalDatasetRepository(configuration, rootDirectory); } else { return new HiveManagedDatasetRepository(configuration); } } }
@BeforeClass public static void saveDefaultConfiguration() { original = DefaultConfiguration.get(); }
@Test public void testFindsHDFS() throws Exception { // set the default configuration that the loader will use Configuration existing = DefaultConfiguration.get(); DefaultConfiguration.set(getConfiguration()); FileSystemDataset<GenericRecord> dataset = Datasets.load("dataset:hdfs:/tmp/datasets/ns/strings"); Assert.assertNotNull("Dataset should be found", dataset); Assert.assertEquals("Dataset should be located in HDFS", "hdfs", dataset.getFileSystem().getUri().getScheme()); // replace the original config so the other tests are not affected DefaultConfiguration.set(existing); }
@Test public void testSetupJobIsIdempotent() { DatasetKeyOutputFormat.MergeOutputCommitter<Object> outputCommitter = new DatasetKeyOutputFormat.MergeOutputCommitter<Object>(); Configuration conf = DefaultConfiguration.get(); DatasetKeyOutputFormat.configure(conf).appendTo(outputDataset); JobID jobId = new JobID("jt", 42); JobContext context = Hadoop.JobContext.ctor.newInstance(conf, jobId); // setup the job outputCommitter.setupJob(context); // call setup again to simulate an ApplicationMaster restart outputCommitter.setupJob(context); }