@Test public void testHDFS() { { StorageURL id = new StorageURL( "master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106"); assertEquals("master_ci_instance", id.getIdentifier()); assertEquals("hdfs", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals( "master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106", id.toString()); } }
public static void deleteHDFSMeta(String metaUrl) throws IOException { String realHdfsPath = StorageURL.valueOf(metaUrl).getParameter("path"); HadoopUtil.getFileSystem(realHdfsPath).delete(new Path(realHdfsPath), true); logger.info("Delete metadata in HDFS for this job: " + realHdfsPath); }
private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException { StorageURL url = kylinConfig.getMetadataUrl(); if (!url.getScheme().equals("hbase")) throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url); // control timeout for prompt error report Map<String, String> newParams = new LinkedHashMap<>(); newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod()); newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout()); newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber()); newParams.putAll(url.getAllParameters()); return url.copy(newParams); }
protected String getPath(KylinConfig kylinConfig) { StorageURL metadataUrl = kylinConfig.getMetadataUrl(); Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme())); return metadataUrl.getParameter("path"); } }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public String toString() { return getMetadataUrl().toString(); }
protected String getPath(KylinConfig kylinConfig) { return kylinConfig.getMetadataUrl().getIdentifier(); }
public static void checkScheme(StorageURL url) { Preconditions.checkState(JDBC_SCHEME.equals(url.getScheme())); }
private static Configuration newHBaseConfiguration(StorageURL url) { if (!"hbase".equals(url.getScheme())) throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties"); for (Entry<String, String> entry : url.getAllParameters().entrySet()) { conf.set(entry.getKey(), entry.getValue());
public StorageURL getMetadataUrl() { return StorageURL.valueOf(getOptional("kylin.metadata.url", "kylin_metadata@hbase")); }
@Override public StorageURL load(String metadataUrl) throws Exception { return new StorageURL(metadataUrl); } });
@Test public void testValueOfCache() { StorageURL id1 = StorageURL.valueOf("hello@hbase"); StorageURL id2 = StorageURL.valueOf("hello@hbase"); StorageURL id3 = StorageURL.valueOf("hello @ hbase"); StorageURL id4 = StorageURL.valueOf("hello@hbase,a=b"); assertTrue(id1 == id2); assertTrue(id1 != id3); assertTrue(id1.equals(id3)); assertTrue(id2 != id4); assertTrue(!id2.equals(id4)); } }
private Map<String, String> initDbcpProps(KylinConfig config) { // metadataUrl is like "kylin_default_instance@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=root,password=xxx" StorageURL metadataUrl = config.getMetadataUrl(); JDBCResourceStore.checkScheme(metadataUrl); LinkedHashMap<String, String> ret = new LinkedHashMap<>(metadataUrl.getAllParameters()); List<String> mandatoryItems = Arrays.asList("url", "username", PASSWORD); for (String item : mandatoryItems) { Preconditions.checkNotNull(ret.get(item), "Setting item \"" + item + "\" is mandatory for Jdbc connections."); } // Check whether password encrypted if ("true".equals(ret.get("passwordEncrypted"))) { String password = ret.get("password"); ret.put("password", EncryptUtil.decrypt(password)); ret.remove("passwordEncrypted"); } logger.info("Connecting to Jdbc with url:{0} by user {1}", ret.get("url"), ret.get("username")); putIfMissing(ret, "driverClassName", "com.mysql.jdbc.Driver"); putIfMissing(ret, "maxActive", "5"); putIfMissing(ret, "maxIdle", "5"); putIfMissing(ret, "maxWait", "1000"); putIfMissing(ret, "removeAbandoned", "true"); putIfMissing(ret, "removeAbandonedTimeout", "180"); putIfMissing(ret, "testOnBorrow", "true"); putIfMissing(ret, "testWhileIdle", "true"); putIfMissing(ret, "validationQuery", "select 1"); return ret; }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public static String replaceMetadataUrl(KylinConfig kylinConfig, String newUrl) { String oldUrl = kylinConfig.getMetadataUrl().toString(); kylinConfig.setProperty("kylin.metadata.url", newUrl); return oldUrl; }
public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception { super(kylinConfig); Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme())); String path = metadataUrl.getParameter("path"); if (path == null) { // missing path is not expected, but don't fail it path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata"; logger.warn("Missing path, fall back to {}. ", path); } fs = HadoopUtil.getFileSystem(path); Path metadataPath = new Path(path); if (fs.exists(metadataPath) == false) { logger.warn("Path not exist in HDFS, create it: {}. ", path); createMetaFolder(metadataPath); } hdfsMetaPath = metadataPath; logger.info("hdfs meta path : {}", hdfsMetaPath); }
public String getMetadataUrlPrefix() { return getMetadataUrl().getIdentifier(); }
private static void checkAndGetHbaseUrl() { StorageURL srcMetadataUrl = srcConfig.getMetadataUrl(); StorageURL dstMetadataUrl = dstConfig.getMetadataUrl(); logger.info("src metadata url is " + srcMetadataUrl); logger.info("dst metadata url is " + dstMetadataUrl); if (!"hbase".equals(srcMetadataUrl.getScheme()) || !"hbase".equals(dstMetadataUrl.getScheme())) throw new IllegalStateException("Both metadata urls should be hbase metadata url"); }
private static Configuration newHBaseConfiguration(StorageURL url) { if (!"hbase".equals(url.getScheme())) throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties"); for (Entry<String, String> entry : url.getAllParameters().entrySet()) { conf.set(entry.getKey(), entry.getValue());
public StorageURL getStorageUrl() { String url = getOptional("kylin.storage.url", "default@hbase"); // for backward compatibility if ("hbase".equals(url)) url = "default@hbase"; return StorageURL.valueOf(url); }