protected String getPath(KylinConfig kylinConfig) { return kylinConfig.getMetadataUrl().getIdentifier(); }
public String getMetadataUrlPrefix() { return getMetadataUrl().getIdentifier(); }
public static KylinConfig loadKylinPropsAndMetadata() throws IOException { File metaDir = new File("meta"); if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) { System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath()); logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Map<String, String> paramsMap = new HashMap<>(); paramsMap.put("path", metaDir.getAbsolutePath()); StorageURL storageURL = new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "ifile", paramsMap); kylinConfig.setMetadataUrl(storageURL.toString()); return kylinConfig; } else { return KylinConfig.getInstanceFromEnv(); } }
public HBaseResourceStore(KylinConfig kylinConfig) throws IOException { super(kylinConfig); metadataUrl = buildMetadataUrl(kylinConfig); tableName = metadataUrl.getIdentifier(); createHTableIfNeeded(tableName); kvSizeLimit = Integer .parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760")); }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
@Override protected String createMetaStoreUUID() throws IOException { try (final Admin hbaseAdmin = HBaseConnection.get(metadataUrl).getAdmin()) { final String metaStoreName = metadataUrl.getIdentifier(); final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName)); String uuid = desc.getValue(HBaseConnection.HTABLE_UUID_TAG); if (uuid != null) return uuid; return UUID.randomUUID().toString(); } catch (Exception e) { return null; } }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public JDBCResourceStore(KylinConfig kylinConfig) throws SQLException, IOException { super(kylinConfig); StorageURL metadataUrl = kylinConfig.getMetadataUrl(); checkScheme(metadataUrl); this.metadataIdentifier = metadataUrl.getIdentifier(); this.tableNames[0] = metadataIdentifier; this.tableNames[1] = metadataIdentifier + "_log"; this.connectionManager = JDBCConnectionManager.getConnectionManager(); for (int i = 0; i < tableNames.length; i++) { createTableIfNeeded(tableNames[i]); } }
@Test public void testHDFS() { { StorageURL id = new StorageURL( "master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106"); assertEquals("master_ci_instance", id.getIdentifier()); assertEquals("hdfs", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals( "master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106", id.toString()); } }
@Test public void testEdgeCases() { { StorageURL id = new StorageURL(""); assertEquals("kylin_metadata", id.getIdentifier()); assertEquals("", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("kylin_metadata", id.toString()); } { StorageURL id = new StorageURL("hello@"); assertEquals("hello", id.getIdentifier()); assertEquals("", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("hello", id.toString()); } { StorageURL id = new StorageURL("hello@hbase,a"); assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals("", id.getParameter("a")); assertEquals("hello@hbase,a", id.toString()); } }
assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(2, id.getAllParameters().size()); StorageURL o2 = new StorageURL("hello@hbase,a=b"); StorageURL id = o.copy(o2.getAllParameters()); assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(1, id.getAllParameters().size());
public String getMetadataUrlPrefix() { return getMetadataUrl().getIdentifier(); }
protected String getPath(KylinConfig kylinConfig) { return kylinConfig.getMetadataUrl().getIdentifier(); }
public static KylinConfig loadKylinPropsAndMetadata() throws IOException { File metaDir = new File("meta"); if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) { System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath()); logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Map<String, String> paramsMap = new HashMap<>(); paramsMap.put("path", metaDir.getAbsolutePath()); StorageURL storageURL = new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "ifile", paramsMap); kylinConfig.setMetadataUrl(storageURL.toString()); return kylinConfig; } else { return KylinConfig.getInstanceFromEnv(); } }
@Override protected String createMetaStoreUUID() throws IOException { try (final Admin hbaseAdmin = HBaseConnection.get(metadataUrl).getAdmin()) { final String metaStoreName = metadataUrl.getIdentifier(); final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName)); String uuid = desc.getValue(HBaseConnection.HTABLE_UUID_TAG); if (uuid != null) return uuid; return UUID.randomUUID().toString(); } catch (Exception e) { return null; } }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public HBaseResourceStore(KylinConfig kylinConfig) throws IOException { super(kylinConfig); metadataUrl = buildMetadataUrl(kylinConfig); tableName = metadataUrl.getIdentifier(); createHTableIfNeeded(tableName); kvSizeLimit = Integer .parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760")); }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public JDBCResourceStore(KylinConfig kylinConfig) throws SQLException, IOException { super(kylinConfig); StorageURL metadataUrl = kylinConfig.getMetadataUrl(); checkScheme(metadataUrl); this.metadataIdentifier = metadataUrl.getIdentifier(); this.tableNames[0] = metadataIdentifier; this.tableNames[1] = metadataIdentifier + "_log"; this.connectionManager = JDBCConnectionManager.getConnectionManager(); for (int i = 0; i < tableNames.length; i++) { createTableIfNeeded(tableNames[i]); } }