@Override protected String getReadableResourcePathImpl(String resPath) { return tableName + "(key='" + resPath + "')@" + kylinConfig.getMetadataUrl(); }
@Override protected String getReadableResourcePathImpl(String resPath) { return metadataIdentifier + "(key='" + resPath + "')@" + kylinConfig.getMetadataUrl(); }
private static void checkAndGetHbaseUrl() { StorageURL srcMetadataUrl = srcConfig.getMetadataUrl(); StorageURL dstMetadataUrl = dstConfig.getMetadataUrl(); logger.info("src metadata url is " + srcMetadataUrl); logger.info("dst metadata url is " + dstMetadataUrl); if (!"hbase".equals(srcMetadataUrl.getScheme()) || !"hbase".equals(dstMetadataUrl.getScheme())) throw new IllegalStateException("Both metadata urls should be hbase metadata url"); }
public FileResourceStore(KylinConfig kylinConfig) { super(kylinConfig); root = new File(getPath(kylinConfig)).getAbsoluteFile(); if (root.exists() == false) throw new IllegalArgumentException( "File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath()); }
protected void checkAndGetHbaseUrl() { StorageURL srcMetadataUrl = srcConfig.getMetadataUrl(); StorageURL dstMetadataUrl = dstConfig.getMetadataUrl(); logger.info("src metadata url is " + srcMetadataUrl); logger.info("dst metadata url is " + dstMetadataUrl); if (!"hbase".equals(srcMetadataUrl.getScheme()) || !"hbase".equals(dstMetadataUrl.getScheme())) throw new IllegalStateException("Both metadata urls should be hbase metadata url"); }
protected String getPath(KylinConfig kylinConfig) { StorageURL metadataUrl = kylinConfig.getMetadataUrl(); Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme())); return metadataUrl.getParameter("path"); } }
protected void cleanUnusedHBaseTables() throws IOException { if ("hbase".equals(config.getStorageUrl().getScheme()) && !"".equals(config.getMetadataUrl().getScheme())) { final int deleteTimeoutMin = 10; // Unit minute try { // use reflection to isolate NoClassDef errors when HBase is not available Class hbaseCleanUpUtil = Class.forName("org.apache.kylin.rest.job.StorageCleanJobHbaseUtil"); Method cleanUnusedHBaseTables = hbaseCleanUpUtil.getDeclaredMethod("cleanUnusedHBaseTables", boolean.class, int.class); hbaseGarbageTables = (List<String>) cleanUnusedHBaseTables.invoke(hbaseCleanUpUtil, delete, deleteTimeoutMin); } catch (Throwable e) { logger.error("Error during HBase clean up", e); } } }
private void putResourceCheckpoint(String resPath, ContentWriter content, long ts) throws IOException { logger.trace("Directly saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")"); beforeChange(resPath); putResourceWithRetry(resPath, content, ts); }
/** * delete a resource, does nothing on a folder */ final public void deleteResource(String resPath) throws IOException { logger.trace("Deleting resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")"); deleteResourceCheckpoint(norm(resPath)); }
private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException { StorageURL url = kylinConfig.getMetadataUrl(); if (!url.getScheme().equals("hbase")) throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url); // control timeout for prompt error report Map<String, String> newParams = new LinkedHashMap<>(); newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod()); newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout()); newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber()); newParams.putAll(url.getAllParameters()); return url.copy(newParams); }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) { Map<String, String> param = new HashMap<>(); param.put("path", getDumpMetadataPath(jobId)); return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString(); } }
private boolean checkTableExist(String tableName) throws IOException { StorageURL metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl(); try (Admin admin = HBaseConnection.get(metadataUrl).getAdmin()) { return admin.tableExists(TableName.valueOf(tableName)); } }
public JDBCResourceStore(KylinConfig kylinConfig) throws SQLException, IOException { super(kylinConfig); StorageURL metadataUrl = kylinConfig.getMetadataUrl(); checkScheme(metadataUrl); this.metadataIdentifier = metadataUrl.getIdentifier(); this.tableNames[0] = metadataIdentifier; this.tableNames[1] = metadataIdentifier + "_log"; this.connectionManager = JDBCConnectionManager.getConnectionManager(); for (int i = 0; i < tableNames.length; i++) { createTableIfNeeded(tableNames[i]); } }
@Override public void run() { holder[0] = KylinConfig.getInstanceFromEnv().getMetadataUrl().toString(); } });
public static String replaceMetadataUrl(KylinConfig kylinConfig, String newUrl) { String oldUrl = kylinConfig.getMetadataUrl().toString(); kylinConfig.setProperty("kylin.metadata.url", newUrl); return oldUrl; }
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta")); CuboidReducer reducer = new CuboidReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); }
@Before public void setUp() throws Exception { createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("./meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta")); NDCuboidMapper mapper = new NDCuboidMapper(); CuboidReducer reducer = new CuboidReducer(); mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer); }
@Before public void setup() throws Exception { this.createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta")); CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor(); codec = new MeasureCodec(desc.getMeasures()); }
@Test public void testFileStore() throws Exception { KylinConfig config = KylinConfig.getInstanceFromEnv(); ResourceStoreTest.testAStore(config.getMetadataUrl().toString(), config); }