public Map<Integer, String> getStorageEngines() { Map<Integer, String> r = Maps.newLinkedHashMap(); // ref constants in IStorageAware r.put(0, "org.apache.kylin.storage.hbase.HBaseStorage"); r.put(1, "org.apache.kylin.storage.hybrid.HybridStorage"); r.put(2, "org.apache.kylin.storage.hbase.HBaseStorage"); r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.storage.provider."))); return r; }
final protected Map<String, String> getPropertiesByPrefix(String prefix) { Map<String, String> result = Maps.newLinkedHashMap(); for (Entry<Object, Object> entry : getAllProperties().entrySet()) { String key = (String) entry.getKey(); if (key.startsWith(prefix)) { result.put(key.substring(prefix.length()), (String) entry.getValue()); } } return result; }
public int getBadQueryDefaultDetectIntervalSeconds() { int time = (int) (getQueryTimeoutSeconds() * getBadQueryDefaultAlertingCoefficient()); // half of query timeout if (time == 0) { time = 60; // 60 sec } return time; }
public String getKylinMetricsSubjectSuffix() { return getOptional("kylin.metric.subject-suffix", getDeployEnv()); }
public String getKylinJobJarPath() { final String jobJar = getOptional(KYLIN_ENGINE_MR_JOB_JAR); if (StringUtils.isNotEmpty(jobJar)) { return jobJar; } String kylinHome = getKylinHome(); if (StringUtils.isEmpty(kylinHome)) { return ""; } return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN); }
public String getMetastoreBigCellHdfsDirectory() { if (cachedBigCellDirectory != null) return cachedBigCellDirectory; String root = getOptional("kylin.env.hdfs-metastore-bigcell-dir"); if (root == null) { return getJdbcHdfsWorkingDirectory(); } Path path = new Path(root); if (!path.isAbsolute()) throw new IllegalArgumentException( "kylin.env.hdfs-metastore-bigcell-dir must be absolute, but got " + root); // make sure path is qualified try { FileSystem fs = HadoopUtil.getReadFileSystem(); path = fs.makeQualified(path); } catch (IOException e) { throw new RuntimeException(e); } root = new Path(path, StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).toString(); if (!root.endsWith("/")) root += "/"; cachedBigCellDirectory = root; if (cachedBigCellDirectory.startsWith(FILE_SCHEME)) { cachedBigCellDirectory = cachedBigCellDirectory.replace(FILE_SCHEME, "file://"); } else if (cachedBigCellDirectory.startsWith(MAPRFS_SCHEME)) { cachedBigCellDirectory = cachedBigCellDirectory.replace(MAPRFS_SCHEME, "maprfs://"); } return cachedBigCellDirectory; }
public String getReadHdfsWorkingDirectory() { if (StringUtils.isNotEmpty(getHBaseClusterFs())) { Path workingDir = new Path(getHdfsWorkingDirectory()); return new Path(getHBaseClusterFs(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/"; } return getHdfsWorkingDirectory(); }
public String getHdfsWorkingDirectory(String project) { if (isProjectIsolationEnabled() && project != null) { return new Path(getHdfsWorkingDirectory(), project).toString() + "/"; } else { return getHdfsWorkingDirectory(); } }
private String getJdbcHdfsWorkingDirectory() { if (StringUtils.isNotEmpty(getJdbcFileSystem())) { Path workingDir = new Path(getReadHdfsWorkingDirectory()); return new Path(getJdbcFileSystem(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/"; } return getReadHdfsWorkingDirectory(); }
public static String getSparkHome() { String sparkHome = System.getenv("SPARK_HOME"); if (StringUtils.isNotEmpty(sparkHome)) { logger.info("SPARK_HOME was set to {}", sparkHome); return sparkHome; } sparkHome = System.getProperty("SPARK_HOME"); if (StringUtils.isNotEmpty(sparkHome)) { logger.info("SPARK_HOME was set to {}", sparkHome); return sparkHome; } return getKylinHome() + File.separator + "spark"; }
public String getCoprocessorLocalJar() { final String coprocessorJar = getOptional(KYLIN_STORAGE_HBASE_COPROCESSOR_LOCAL_JAR); if (StringUtils.isNotEmpty(coprocessorJar)) { return coprocessorJar; } String kylinHome = getKylinHome(); if (StringUtils.isEmpty(kylinHome)) { throw new RuntimeException("getCoprocessorLocalJar needs KYLIN_HOME"); } return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN); }
public String getMetastoreBigCellHdfsDirectory() { if (cachedBigCellDirectory != null) return cachedBigCellDirectory; String root = getOptional("kylin.env.hdfs-metastore-bigcell-dir"); if (root == null) { return getJdbcHdfsWorkingDirectory(); } Path path = new Path(root); if (!path.isAbsolute()) throw new IllegalArgumentException( "kylin.env.hdfs-metastore-bigcell-dir must be absolute, but got " + root); // make sure path is qualified try { FileSystem fs = HadoopUtil.getReadFileSystem(); path = fs.makeQualified(path); } catch (IOException e) { throw new RuntimeException(e); } root = new Path(path, StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).toString(); if (!root.endsWith("/")) root += "/"; cachedBigCellDirectory = root; if (cachedBigCellDirectory.startsWith(FILE_SCHEME)) { cachedBigCellDirectory = cachedBigCellDirectory.replace(FILE_SCHEME, "file://"); } else if (cachedBigCellDirectory.startsWith(MAPRFS_SCHEME)) { cachedBigCellDirectory = cachedBigCellDirectory.replace(MAPRFS_SCHEME, "maprfs://"); } return cachedBigCellDirectory; }
public String getReadHdfsWorkingDirectory() { if (StringUtils.isNotEmpty(getHBaseClusterFs())) { Path workingDir = new Path(getHdfsWorkingDirectory()); return new Path(getHBaseClusterFs(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/"; } return getHdfsWorkingDirectory(); }
public String getHdfsWorkingDirectory(String project) { if (isProjectIsolationEnabled() && project != null) { return new Path(getHdfsWorkingDirectory(), project).toString() + "/"; } else { return getHdfsWorkingDirectory(); } }
private String getJdbcHdfsWorkingDirectory() { if (StringUtils.isNotEmpty(getJdbcFileSystem())) { Path workingDir = new Path(getReadHdfsWorkingDirectory()); return new Path(getJdbcFileSystem(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/"; } return getReadHdfsWorkingDirectory(); }
public String getKylinMetricsSubjectSuffix() { return getOptional("kylin.metric.subject-suffix", getDeployEnv()); }
public static String getSparkHome() { String sparkHome = System.getenv("SPARK_HOME"); if (StringUtils.isNotEmpty(sparkHome)) { logger.info("SPARK_HOME was set to {}", sparkHome); return sparkHome; } sparkHome = System.getProperty("SPARK_HOME"); if (StringUtils.isNotEmpty(sparkHome)) { logger.info("SPARK_HOME was set to {}", sparkHome); return sparkHome; } return getKylinHome() + File.separator + "spark"; }
public String getKylinJobJarPath() { final String jobJar = getOptional(KYLIN_ENGINE_MR_JOB_JAR); if (StringUtils.isNotEmpty(jobJar)) { return jobJar; } String kylinHome = getKylinHome(); if (StringUtils.isEmpty(kylinHome)) { return ""; } return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN); }
public Map<Integer, String> getJobEngines() { Map<Integer, String> r = Maps.newLinkedHashMap(); // ref constants in IEngineAware r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine"); //IEngineAware.ID_MR_V1 r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); //IEngineAware.ID_MR_V2 r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2"); //IEngineAware.ID_SPARK r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider."))); return r; }
public int getBadQueryDefaultDetectIntervalSeconds() { int time = (int) (getQueryTimeoutSeconds() * getBadQueryDefaultAlertingCoefficient()); // half of query timeout if (time == 0) { time = 60; // 60 sec } return time; }