/** * Shamelessly cloned from GenericOptionsParser. */ public static String realFile(String newFile, Configuration conf) throws IOException { Path path = new Path(newFile); URI pathURI = path.toUri(); FileSystem fs; if (pathURI.getScheme() == null) { fs = FileSystem.getLocal(conf); } else { fs = path.getFileSystem(conf); } if (!fs.exists(path)) { return null; } String file = path.makeQualified(fs).toString(); return file; }
private void expandWildcard(List<String> finalPaths, Path path, FileSystem fs) throws IOException { FileStatus status = fs.getFileStatus(path); if (!status.isDirectory()) { throw new FileNotFoundException(path + " is not a directory."); } // get all the jars in the directory List<Path> jars = FileUtil.getJarsInDirectory(path.toString(), fs.equals(FileSystem.getLocal(conf))); if (jars.isEmpty()) { LOG.warn(path + " does not have jars in it. It will be ignored."); } else { for (Path jar: jars) { finalPaths.add(jar.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString()); } } }
private void dropPathAndUnregisterDeleteOnExit(Path path, Configuration conf, boolean localFs) { FileSystem fs = null; try { if (localFs) { fs = FileSystem.getLocal(conf); } else { fs = path.getFileSystem(conf); } fs.cancelDeleteOnExit(path); fs.delete(path, true); LOG.info("Deleted directory: {} on fs with scheme {}", path, fs.getScheme()); } catch (IllegalArgumentException | UnsupportedOperationException | IOException e) { LOG.error("Failed to delete path at {} on fs with scheme {}", path, (fs == null ? "Unknown-null" : fs.getScheme()), e); } }
/** * Copy dependency jars from local fs to HDFS. */ private void copyDependencyJarsToHdfs() throws IOException { if (!this.state.contains(ConfigurationKeys.JOB_JAR_FILES_KEY)) { return; } LocalFileSystem lfs = FileSystem.getLocal(this.conf); Path tmpJarFileDir = new Path(this.tmpOutputDir, "_gobblin_compaction_jars"); this.state.setProp(COMPACTION_JARS, tmpJarFileDir.toString()); this.fs.delete(tmpJarFileDir, true); for (String jarFile : this.state.getPropAsList(ConfigurationKeys.JOB_JAR_FILES_KEY)) { for (FileStatus status : lfs.globStatus(new Path(jarFile))) { Path tmpJarFile = new Path(this.fs.makeQualified(tmpJarFileDir), status.getPath().getName()); this.fs.copyFromLocalFile(status.getPath(), tmpJarFile); LOG.info(String.format("%s will be added to classpath", tmpJarFile)); } } }
ctx = new Context(); String[] dirStrings = StringUtils.getTrimmedStrings(newLocalDirs); ctx.localFS = FileSystem.getLocal(conf); int numDirs = dirStrings.length; ArrayList<Path> dirs = new ArrayList<Path>(numDirs); try { Path tmpDir = new Path(dirStrings[i]); if(ctx.localFS.mkdirs(tmpDir)|| ctx.localFS.exists(tmpDir)) { try { File tmpFile = tmpDir.isAbsolute() ? new File(ctx.localFS.makeQualified(tmpDir).toUri()) : new File(dirStrings[i]); dirs.add(new Path(tmpFile.getPath())); dfList.add(new DF(tmpFile, 30000)); } catch (DiskErrorException de) {
@BeforeClass public void setup() throws Exception { this.jdbcConnector = HiveJdbcConnector.newEmbeddedConnector(2); this.dataFile = new File(getClass().getClassLoader().getResource("hiveMaterializerTest/source/").toURI()); this.localHiveMetastore.dropDatabaseIfExists(this.dbName); this.localHiveMetastore.createTestDb(this.dbName); this.jdbcConnector.executeStatements( String.format("CREATE EXTERNAL TABLE %s.%s (id STRING, name String) PARTITIONED BY (%s String) " + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE", this.dbName, this.sourceTableName, this.partitionColumn), String.format("ALTER TABLE %s.%s ADD PARTITION (part = 'part1') LOCATION '%s'", this.dbName, this.sourceTableName, this.dataFile.getAbsolutePath() + "/part1"), String.format("ALTER TABLE %s.%s ADD PARTITION (part = 'part2') LOCATION '%s'", this.dbName, this.sourceTableName, this.dataFile.getAbsolutePath() + "/part2")); List<List<String>> allTable = executeStatementAndGetResults(this.jdbcConnector, String.format("SELECT * FROM %s.%s", this.dbName, this.sourceTableName), 3); Assert.assertEquals(allTable.size(), 8); List<List<String>> part1 = executeStatementAndGetResults(this.jdbcConnector, String.format("SELECT * FROM %s.%s WHERE %s='part1'", this.dbName, this.sourceTableName, this.partitionColumn), 3); Assert.assertEquals(part1.size(), 4); this.pool = HiveMetastoreClientPool.get(new Properties(), Optional.absent()); Table table; try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { table = new Table(client.get().getTable(this.dbName, this.sourceTableName)); } this.dataset = new HiveDataset(FileSystem.getLocal(new Configuration()), pool, table, new Properties()); }
@BeforeClass public void setUp() throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.outputPath = new Path(ParallelRunnerTest.class.getSimpleName()); }
@AfterClass public void tearDown() throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); fs.delete(this.testBasePath, true); }
private Path setupTempDirWithSingleOutputFile(Configuration hconf) throws IOException { Path tempDirPath = new Path("file://" + temporaryFolder.newFolder().getAbsolutePath()); Path taskOutputPath = new Path(tempDirPath, Utilities.getTaskId(hconf)); FileSystem.getLocal(hconf).create(taskOutputPath).close(); return tempDirPath; }
/** {@inheritDoc} */ @Override protected void beforeTestsStarted0() throws Exception { Configuration cfg = new Configuration(); setupFileSystems(cfg); // Init cache by correct LocalFileSystem implementation FileSystem.getLocal(cfg); }
@AfterClass @BeforeClass public void cleanup() throws Exception { FileSystem fs = FileSystem.getLocal(new Configuration()); fs.delete(new Path("gobblin-test-harness/testOutput"), true); } }
URI uri = (URI)pathObj; srcPath = new Path(uri); File dstPath = new File(jobLocDir.getAbsolutePath(), locName); FileSystem dstFs = FileSystem.getLocal(cfg); File archivesPath = new File(jobLocDir.getAbsolutePath(), ".cached-archives"); "[path=" + archivesPath + ", jobId=" + jobId + ']'); File archiveFile = new File(archivesPath, locName); FileUtil.copy(srcFs, srcPath, dstFs, new Path(archiveFile.toString()), false, cfg); FileUtil.copy(srcFs, srcPath, dstFs, new Path(dstPath.toString()), false, cfg);
@Test public void testCleanStateStore() throws IOException { File tmpDir = Files.createTempDir(); tmpDir.deleteOnExit(); FileSystem fs = FileSystem.getLocal(new Configuration()); FsDatasetStateStore store = new FsDatasetStateStore(fs, tmpDir.getAbsolutePath()); store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id1")); store.persistDatasetState("dataset1", new JobState.DatasetState("job1", "job1_id2")); store.persistDatasetState("dataset1", new JobState.DatasetState("job2", "job2_id1")); store.persistDatasetState("", new JobState.DatasetState("job3", "job3_id1")); Properties props = new Properties(); props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, tmpDir.getAbsolutePath()); props.setProperty("selection.timeBased.lookbackTime", "0m"); TimeBasedDatasetStoreDatasetFinder datasetFinder = new TimeBasedDatasetStoreDatasetFinder(fs, props); List<DatasetStoreDataset> datasets = datasetFinder.findDatasets(); for (DatasetStoreDataset dataset : datasets) { ((CleanableDataset) dataset).clean(); File jobDir = new File(tmpDir.getAbsolutePath(), dataset.getKey().getStoreName()); Assert.assertEquals(jobDir.list().length, 1); } } }
@BeforeClass public void setUp() throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.outputPath = new Path(SerializationUtilsTest.class.getSimpleName()); }
/** * Shamelessly cloned from GenericOptionsParser. */ public static String realFile(String newFile, Configuration conf) throws IOException { Path path = new Path(newFile); URI pathURI = path.toUri(); FileSystem fs; if (pathURI.getScheme() == null) { fs = FileSystem.getLocal(conf); } else { fs = path.getFileSystem(conf); } if (!fs.exists(path)) { return null; } String file = path.makeQualified(fs).toString(); return file; }
@Before public void openFileSystem() throws Exception { conf = new JobConf(); // all columns conf.set("columns", "userid,string1,subtype,decimal1,ts"); conf.set("columns.types", "bigint,string,double,decimal,timestamp"); // needed columns conf.set(ColumnProjectionUtils.READ_ALL_COLUMNS, "false"); conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0,2"); conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, "userid,subtype"); fs = FileSystem.getLocal(conf); testFilePath = new Path(workDir, "TestOrcFile." + testCaseName.getMethodName() + ".orc"); testFilePath2 = new Path(workDir, "TestOrcFile." + testCaseName.getMethodName() + ".2.orc"); fs.delete(testFilePath, false); fs.delete(testFilePath2, false); }
private void moveFile(Path sourcePath, Path targetPath, boolean isDfsDir) throws HiveException { try { String mesg = "Moving data to " + (isDfsDir ? "" : "local ") + "directory " + targetPath.toString(); String mesg_detail = " from " + sourcePath.toString(); console.printInfo(mesg, mesg_detail); FileSystem fs = sourcePath.getFileSystem(conf); if (isDfsDir) { moveFileInDfs (sourcePath, targetPath, fs); } else { // This is a local file FileSystem dstFs = FileSystem.getLocal(conf); moveFileFromDfsToLocal(sourcePath, targetPath, fs, dstFs); } } catch (Exception e) { throw new HiveException("Unable to move source " + sourcePath + " to destination " + targetPath, e); } }
@Test public void test() throws IOException, InterruptedException, ExecutionException, FSUtils.StreamLacksCapabilityException { Path f = new Path(TEST_UTIL.getDataTestDir(), "test"); FileSystem fs = FileSystem.getLocal(TEST_UTIL.getConfiguration()); AsyncFSOutput out = AsyncFSOutputHelper.createOutput(fs, f, false, true, fs.getDefaultReplication(f), fs.getDefaultBlockSize(f), GROUP, CHANNEL_CLASS); TestFanOutOneBlockAsyncDFSOutput.writeAndVerify(fs, f, out); } }
@Before public void setup() throws IOException { conf = new Configuration(); fs = FileSystem.getLocal(conf); key = new OrcFileKeyWrapper(); value = new OrcFileValueWrapper(); tmpPath = prepareTmpPath(); }