private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException { StorageURL url = kylinConfig.getMetadataUrl(); if (!url.getScheme().equals("hbase")) throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url); // control timeout for prompt error report Map<String, String> newParams = new LinkedHashMap<>(); newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod()); newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout()); newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber()); newParams.putAll(url.getAllParameters()); return url.copy(newParams); }
for (Entry<String, String> entry : url.getAllParameters().entrySet()) { conf.set(entry.getKey(), entry.getValue());
private Map<String, String> initDbcpProps(KylinConfig config) { // metadataUrl is like "kylin_default_instance@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=root,password=xxx" StorageURL metadataUrl = config.getMetadataUrl(); JDBCResourceStore.checkScheme(metadataUrl); LinkedHashMap<String, String> ret = new LinkedHashMap<>(metadataUrl.getAllParameters()); List<String> mandatoryItems = Arrays.asList("url", "username", PASSWORD); for (String item : mandatoryItems) { Preconditions.checkNotNull(ret.get(item), "Setting item \"" + item + "\" is mandatory for Jdbc connections."); } // Check whether password encrypted if ("true".equals(ret.get("passwordEncrypted"))) { String password = ret.get("password"); ret.put("password", EncryptUtil.decrypt(password)); ret.remove("passwordEncrypted"); } logger.info("Connecting to Jdbc with url:{0} by user {1}", ret.get("url"), ret.get("username")); putIfMissing(ret, "driverClassName", "com.mysql.jdbc.Driver"); putIfMissing(ret, "maxActive", "5"); putIfMissing(ret, "maxIdle", "5"); putIfMissing(ret, "maxWait", "1000"); putIfMissing(ret, "removeAbandoned", "true"); putIfMissing(ret, "removeAbandonedTimeout", "180"); putIfMissing(ret, "testOnBorrow", "true"); putIfMissing(ret, "testWhileIdle", "true"); putIfMissing(ret, "validationQuery", "select 1"); return ret; }
@Test public void testHDFS() { { StorageURL id = new StorageURL( "master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106"); assertEquals("master_ci_instance", id.getIdentifier()); assertEquals("hdfs", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals( "master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106", id.toString()); } }
@Test public void testEdgeCases() { { StorageURL id = new StorageURL(""); assertEquals("kylin_metadata", id.getIdentifier()); assertEquals("", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("kylin_metadata", id.toString()); } { StorageURL id = new StorageURL("hello@"); assertEquals("hello", id.getIdentifier()); assertEquals("", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("hello", id.toString()); } { StorageURL id = new StorageURL("hello@hbase,a"); assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals("", id.getParameter("a")); assertEquals("hello@hbase,a", id.toString()); } }
assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("hello@hbase", id.toString()); assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(2, id.getAllParameters().size()); assertEquals("b", id.getParameter("a")); assertEquals("d", id.getParameter("c")); StorageURL id = o.copy(o2.getAllParameters()); assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals("b", id.getParameter("a")); assertEquals("hello@hbase,a=b", id.toString());
private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException { StorageURL url = kylinConfig.getMetadataUrl(); if (!url.getScheme().equals("hbase")) throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url); // control timeout for prompt error report Map<String, String> newParams = new LinkedHashMap<>(); newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod()); newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout()); newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber()); newParams.putAll(url.getAllParameters()); return url.copy(newParams); }
for (Entry<String, String> entry : url.getAllParameters().entrySet()) { conf.set(entry.getKey(), entry.getValue());
private Map<String, String> initDbcpProps(KylinConfig config) { // metadataUrl is like "kylin_default_instance@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=root,password=xxx" StorageURL metadataUrl = config.getMetadataUrl(); JDBCResourceStore.checkScheme(metadataUrl); LinkedHashMap<String, String> ret = new LinkedHashMap<>(metadataUrl.getAllParameters()); List<String> mandatoryItems = Arrays.asList("url", "username", PASSWORD); for (String item : mandatoryItems) { Preconditions.checkNotNull(ret.get(item), "Setting item \"" + item + "\" is mandatory for Jdbc connections."); } // Check whether password encrypted if ("true".equals(ret.get("passwordEncrypted"))) { String password = ret.get("password"); ret.put("password", EncryptUtil.decrypt(password)); ret.remove("passwordEncrypted"); } logger.info("Connecting to Jdbc with url:{0} by user {1}", ret.get("url"), ret.get("username")); putIfMissing(ret, "driverClassName", "com.mysql.jdbc.Driver"); putIfMissing(ret, "maxActive", "5"); putIfMissing(ret, "maxIdle", "5"); putIfMissing(ret, "maxWait", "1000"); putIfMissing(ret, "removeAbandoned", "true"); putIfMissing(ret, "removeAbandonedTimeout", "180"); putIfMissing(ret, "testOnBorrow", "true"); putIfMissing(ret, "testWhileIdle", "true"); putIfMissing(ret, "validationQuery", "select 1"); return ret; }