public String getHdfsWorkingDirectory() { return config.getHdfsWorkingDirectory(); }
@Override protected String pushdownRootPath() { String hdfsWorkingDirectory = this.kylinConfig.getHdfsWorkingDirectory(null); if (hdfsWorkingDirectory.endsWith("/")) return hdfsWorkingDirectory + "resources"; else return hdfsWorkingDirectory + "/" + "resources"; }
private String convertToAbsolutePath(String path) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Path basicPath = new Path(path); if (basicPath.toUri().getScheme() == null) return kylinConfig.getHdfsWorkingDirectory() + path; String[] paths = path.split("/resources/GlobalDict/"); if (paths.length == 2) return kylinConfig.getHdfsWorkingDirectory() + "/resources/GlobalDict/" + paths[1]; paths = path.split("/resources/SegmentDict/"); if (paths.length == 2) { return kylinConfig.getHdfsWorkingDirectory() + "/resources/SegmentDict/" + paths[1]; } else { throw new RuntimeException( "the basic directory of global dictionary only support the format which contains '/resources/GlobalDict/' or '/resources/SegmentDict/'"); } }
private static Path getCoprocessorHDFSDir(FileSystem fileSystem, KylinConfig config) throws IOException { String hdfsWorkingDirectory = config.getHdfsWorkingDirectory(); hdfsWorkingDirectory = HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory); Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor"); fileSystem.mkdirs(coprocessorDir); return coprocessorDir; }
/** * JIRA: https://issues.apache.org/jira/browse/KYLIN-2945 * if pass a absolute path, it may produce some problems like cannot find global dict after migration. * so convert to relative path can avoid it and be better to maintain flexibility. * */ private String convertToRelativePath(String path) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory(); if (!isSaveAbsolutePath && path.startsWith(hdfsWorkingDir)) { return path.substring(hdfsWorkingDir.length()); } return path; }
@Override public String copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException { if (baseDir.contains("resources/SegmentDict")) { logger.info("SegmentAppendTrieDict needn't to copy"); return baseDir; } checkArgument(baseDir.startsWith(srcConfig.getHdfsWorkingDirectory()), "Please check why current directory {} doesn't belong to source working directory {}", baseDir, srcConfig.getHdfsWorkingDirectory()); final String dstBaseDir = baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()); Long[] versions = listAllVersions(); if (versions.length == 0) { // empty dict, nothing to copy return dstBaseDir; } Path srcVersionDir = getVersionDir(versions[versions.length - 1]); Path dstVersionDir = new Path(srcVersionDir.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory())); FileSystem dstFS = dstVersionDir.getFileSystem(conf); if (dstFS.exists(dstVersionDir)) { dstFS.delete(dstVersionDir, true); } FileUtil.copy(fileSystem, srcVersionDir, dstFS, dstVersionDir, false, true, conf); return dstBaseDir; }
public static FileSystem getWorkingFileSystem(Configuration conf) throws IOException { Path workingPath = new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); return getFileSystem(workingPath, conf); }
public BaseBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) { KylinConfig config = KylinConfig.getInstanceFromEnv(); this.flatDesc = flatDesc; this.flatTableDatabase = config.getHiveDatabaseForIntermediateTable(); this.hdfsWorkingDir = config.getHdfsWorkingDirectory(); }
public static void main(String[] args) throws IOException { String path = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict/"; if (args.length > 0) { path = args[0]; } System.out.println("Recursive Check AppendTrieDictionary Slices in path " + path); AppendTrieDictionaryChecker checker = new AppendTrieDictionaryChecker(); if (checker.runChecker(path)) { System.exit(0); } else { System.exit(-1); } } }
public static FileSystem getWorkingFileSystem() throws IOException { return getFileSystem(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); }
protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) { return JobBuilderSupport.getJobWorkingDir(config.getHdfsWorkingDirectory(), jobFlow.getId()); }
private static void renameFoldersInHdfs(CubeInstance cube) { for (CubeSegment segment : cube.getSegments()) { String jobUuid = segment.getLastBuildJobID(); String src = JobBuilderSupport.getJobWorkingDir(srcConfig.getHdfsWorkingDirectory(), jobUuid); String tgt = JobBuilderSupport.getJobWorkingDir(dstConfig.getHdfsWorkingDirectory(), jobUuid); operations.add(new Opt(OptType.RENAME_FOLDER_IN_HDFS, new Object[] { src, tgt })); } }
protected void renameFoldersInHdfs(CubeInstance cube) throws IOException { for (CubeSegment segment : cube.getSegments()) { String jobUuid = segment.getLastBuildJobID(); String src = JobBuilderSupport.getJobWorkingDir(srcConfig.getHdfsWorkingDirectory(), jobUuid); String tgt = JobBuilderSupport.getJobWorkingDir(dstConfig.getHdfsWorkingDirectory(), jobUuid); operations.add(new Opt(OptType.RENAME_FOLDER_IN_HDFS, new Object[] { src, tgt })); } }
public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception { super(kylinConfig); Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme())); String path = metadataUrl.getParameter("path"); if (path == null) { // missing path is not expected, but don't fail it path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata"; logger.warn("Missing path, fall back to {}. ", path); } fs = HadoopUtil.getFileSystem(path); Path metadataPath = new Path(path); if (fs.exists(metadataPath) == false) { logger.warn("Path not exist in HDFS, create it: {}. ", path); createMetaFolder(metadataPath); } hdfsMetaPath = metadataPath; logger.info("hdfs meta path : {}", hdfsMetaPath); }
protected String getLocalWorkingDirectory() { String dir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(); if (dir.startsWith("file://")) dir = dir.substring("file://".length()); try { return new File(dir).getCanonicalPath(); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public void init(DictionaryInfo dictInfo, int baseId, String hdfsDir) throws IOException { sourceColumn = dictInfo.getSourceTable() + "." + dictInfo.getSourceColumn(); KylinConfig config = KylinConfig.getInstanceFromEnv(); int maxEntriesPerSlice = config.getAppendDictEntrySize(); if (hdfsDir == null) { //build in Kylin job server hdfsDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(); } //use UUID to make each segment dict in different HDFS dir and support concurrent build //use timestamp to make the segment dict easily to delete String baseDir = hdfsDir + "resources/SegmentDict" + dictInfo.getResourceDir() + "/" + RandomUtil.randomUUID().toString() + "_" + System.currentTimeMillis() + "/"; this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, false); this.baseId = baseId; }
@Test public void testHdfsWorkingDir() { KylinConfig conf = KylinConfig.getInstanceFromEnv(); String hdfsWorkingDirectory = conf.getHdfsWorkingDirectory(); assertTrue(hdfsWorkingDirectory.startsWith("file:/")); }
@Test public void TestGetJobWorkingDir() throws IOException { FileSystem fileSystem = FileSystem.get(new Configuration()); Path jobWorkDirPath = null; KylinConfig kylinConfig = mock(KylinConfig.class); try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig)) { when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true); when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/"); DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class); defaultChainedExecutable.setId(RandomUtil.randomUUID().toString()); String jobWorkingDir = HiveInputBase.getJobWorkingDir(defaultChainedExecutable, KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); jobWorkDirPath = new Path(jobWorkingDir); Assert.assertTrue(fileSystem.exists(jobWorkDirPath)); } finally { if (jobWorkDirPath != null) fileSystem.deleteOnExit(jobWorkDirPath); } }
@Override public void init(DictionaryInfo dictInfo, int baseId, String hdfsDir) throws IOException { sourceColumn = dictInfo.getSourceTable() + "_" + dictInfo.getSourceColumn(); lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentThread(); lock.lock(getLockPath(sourceColumn), Long.MAX_VALUE); int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize(); if (hdfsDir == null) { //build in Kylin job server hdfsDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(); } String baseDir = hdfsDir + "resources/GlobalDict" + dictInfo.getResourceDir() + "/"; try { this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, true); } catch (Throwable e) { lock.unlock(getLockPath(sourceColumn)); throw new RuntimeException( String.format(Locale.ROOT, "Failed to create global dictionary on %s ", sourceColumn), e); } this.baseId = baseId; }
@Before public void beforeTest() { staticCreateTestMetadata(); KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "50000"); BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/"; LOCAL_BASE_DIR = getLocalWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/"; }