public String toString() { return getMetadataUrl().toString(); }
public static KylinConfig loadKylinPropsAndMetadata() throws IOException { File metaDir = new File("meta"); if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) { System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath()); logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Map<String, String> paramsMap = new HashMap<>(); paramsMap.put("path", metaDir.getAbsolutePath()); StorageURL storageURL = new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "ifile", paramsMap); kylinConfig.setMetadataUrl(storageURL.toString()); return kylinConfig; } else { return KylinConfig.getInstanceFromEnv(); } }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public static String replaceMetadataUrl(KylinConfig kylinConfig, String newUrl) { String oldUrl = kylinConfig.getMetadataUrl().toString(); kylinConfig.setProperty("kylin.metadata.url", newUrl); return oldUrl; }
@Override public void run() { holder[0] = KylinConfig.getInstanceFromEnv().getMetadataUrl().toString(); } });
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta")); CuboidReducer reducer = new CuboidReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); }
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("./meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta")); NDCuboidMapper mapper = new NDCuboidMapper(); CuboidReducer reducer = new CuboidReducer(); mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer); }
@Before public void setup() throws Exception { this.createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta")); CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor(); codec = new MeasureCodec(desc.getMeasures()); }
@Test public void testFileStore() throws Exception { KylinConfig config = KylinConfig.getInstanceFromEnv(); ResourceStoreTest.testAStore(config.getMetadataUrl().toString(), config); }
@Test public void testThreadLocalOverride() throws InterruptedException { final String metadata1 = "meta1"; final String metadata2 = "meta2"; // set system KylinConfig KylinConfig sysConfig = KylinConfig.getInstanceFromEnv(); sysConfig.setMetadataUrl(metadata1); assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString()); // test thread-local override KylinConfig threadConfig = KylinConfig.createKylinConfig(new Properties()); threadConfig.setMetadataUrl(metadata2); try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(threadConfig)) { assertEquals(metadata2, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString()); // other threads still use system KylinConfig final String[] holder = new String[1]; Thread child = new Thread(new Runnable() { @Override public void run() { holder[0] = KylinConfig.getInstanceFromEnv().getMetadataUrl().toString(); } }); child.start(); child.join(); assertEquals(metadata1, holder[0]); } }
@Test public void testHDFS() { { StorageURL id = new StorageURL( "master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106"); assertEquals("master_ci_instance", id.getIdentifier()); assertEquals("hdfs", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals( "master_ci_instance@hdfs,path=hdfs://sandbox.hortonworks.com:8020/kylin/master_ci_instance/metadata/f112fe00-6f99-4f8e-b075-d57c08501106", id.toString()); } }
assertEquals("hbase", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("hello@hbase", id.toString()); assertEquals("b", id.getParameter("a")); assertEquals("d", id.getParameter("c")); assertEquals("hello@hbase,a=b,c=d", id.toString()); assertEquals(1, id.getAllParameters().size()); assertEquals("b", id.getParameter("a")); assertEquals("hello@hbase,a=b", id.toString()); assertEquals("hello@hbase,c=d", o.toString()); assertEquals("hello@hbase,a=b", o2.toString());
@Test public void testEdgeCases() { { StorageURL id = new StorageURL(""); assertEquals("kylin_metadata", id.getIdentifier()); assertEquals("", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("kylin_metadata", id.toString()); } { StorageURL id = new StorageURL("hello@"); assertEquals("hello", id.getIdentifier()); assertEquals("", id.getScheme()); assertEquals(0, id.getAllParameters().size()); assertEquals("hello", id.toString()); } { StorageURL id = new StorageURL("hello@hbase,a"); assertEquals("hello", id.getIdentifier()); assertEquals("hbase", id.getScheme()); assertEquals(1, id.getAllParameters().size()); assertEquals("", id.getParameter("a")); assertEquals("hello@hbase,a", id.toString()); } }
public String toString() { return getMetadataUrl().toString(); }
public static KylinConfig loadKylinPropsAndMetadata() throws IOException { File metaDir = new File("meta"); if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) { System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath()); logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Map<String, String> paramsMap = new HashMap<>(); paramsMap.put("path", metaDir.getAbsolutePath()); StorageURL storageURL = new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "ifile", paramsMap); kylinConfig.setMetadataUrl(storageURL.toString()); return kylinConfig; } else { return KylinConfig.getInstanceFromEnv(); } }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }