public static void deleteHDFSMeta(String metaUrl) throws IOException { String realHdfsPath = StorageURL.valueOf(metaUrl).getParameter("path"); HadoopUtil.getFileSystem(realHdfsPath).delete(new Path(realHdfsPath), true); logger.info("Delete metadata in HDFS for this job: " + realHdfsPath); }
protected String getPath(KylinConfig kylinConfig) { StorageURL metadataUrl = kylinConfig.getMetadataUrl(); Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme())); return metadataUrl.getParameter("path"); } }
public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception { super(kylinConfig); Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme())); String path = metadataUrl.getParameter("path"); if (path == null) { // missing path is not expected, but don't fail it path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata"; logger.warn("Missing path, fall back to {}. ", path); } fs = HadoopUtil.getFileSystem(path); Path metadataPath = new Path(path); if (fs.exists(metadataPath) == false) { logger.warn("Path not exist in HDFS, create it: {}. ", path); createMetaFolder(metadataPath); } hdfsMetaPath = metadataPath; logger.info("hdfs meta path : {}", hdfsMetaPath); }
public static KylinConfig loadKylinConfigFromHdfs(String uri) { if (uri == null) throw new IllegalArgumentException("meta url should not be null"); if (!uri.contains("@hdfs")) throw new IllegalArgumentException("meta url should like @hdfs schema"); if (kylinConfigCache.get(uri) != null) { logger.info("KylinConfig cached for : {}", uri); return kylinConfigCache.get(uri); } logger.info("Ready to load KylinConfig from uri: {}", uri); KylinConfig config; FileSystem fs; String realHdfsPath = StorageURL.valueOf(uri).getParameter("path") + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE; try { fs = HadoopUtil.getFileSystem(realHdfsPath); InputStream is = fs.open(new Path(realHdfsPath)); Properties prop = KylinConfig.streamToProps(is); config = KylinConfig.createKylinConfig(prop); } catch (IOException e) { throw new RuntimeException(e); } kylinConfigCache.put(uri, config); return config; }
assertEquals("hbase", id.getScheme()); assertEquals(2, id.getAllParameters().size()); assertEquals("b", id.getParameter("a")); assertEquals("d", id.getParameter("c")); assertEquals("hello@hbase,a=b,c=d", id.toString()); assertEquals("hbase", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals("b", id.getParameter("a")); assertEquals("hello@hbase,a=b", id.toString()); assertEquals("hello@hbase,c=d", o.toString());
@Test public void testEdgeCases() { { StorageURL id = new StorageURL(""); assertEquals("kylin_metadata", id.getIdentifier()); assertEquals("", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("kylin_metadata", id.toString()); } { StorageURL id = new StorageURL("hello@"); assertEquals("hello", id.getIdentifier()); assertEquals("", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("hello", id.toString()); } { StorageURL id = new StorageURL("hello@hbase,a"); assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals("", id.getParameter("a")); assertEquals("hello@hbase,a", id.toString()); } }
protected String getPath(KylinConfig kylinConfig) { StorageURL metadataUrl = kylinConfig.getMetadataUrl(); Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme())); return metadataUrl.getParameter("path"); } }
public static void deleteHDFSMeta(String metaUrl) throws IOException { String realHdfsPath = StorageURL.valueOf(metaUrl).getParameter("path"); HadoopUtil.getFileSystem(realHdfsPath).delete(new Path(realHdfsPath), true); logger.info("Delete metadata in HDFS for this job: " + realHdfsPath); }
public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception { super(kylinConfig); Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme())); String path = metadataUrl.getParameter("path"); if (path == null) { // missing path is not expected, but don't fail it path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata"; logger.warn("Missing path, fall back to {}. ", path); } fs = HadoopUtil.getFileSystem(path); Path metadataPath = new Path(path); if (fs.exists(metadataPath) == false) { logger.warn("Path not exist in HDFS, create it: {}. ", path); createMetaFolder(metadataPath); } hdfsMetaPath = metadataPath; logger.info("hdfs meta path : {}", hdfsMetaPath); }
public static KylinConfig loadKylinConfigFromHdfs(String uri) { if (uri == null) throw new IllegalArgumentException("meta url should not be null"); if (!uri.contains("@hdfs")) throw new IllegalArgumentException("meta url should like @hdfs schema"); if (kylinConfigCache.get(uri) != null) { logger.info("KylinConfig cached for : {}", uri); return kylinConfigCache.get(uri); } logger.info("Ready to load KylinConfig from uri: {}", uri); KylinConfig config; FileSystem fs; String realHdfsPath = StorageURL.valueOf(uri).getParameter("path") + "/" + KylinConfig.KYLIN_CONF_PROPERTIES_FILE; try { fs = HadoopUtil.getFileSystem(realHdfsPath); InputStream is = fs.open(new Path(realHdfsPath)); Properties prop = KylinConfig.streamToProps(is); config = KylinConfig.createKylinConfig(prop); } catch (IOException e) { throw new RuntimeException(e); } kylinConfigCache.put(uri, config); return config; }