public String getKafkaConsumerHadoopJobConf() { File kafkaConsumerFile = getKafkaConsumerFile(); return OptionsHelper.convertToFileURL(kafkaConsumerFile.getAbsolutePath()); }
private String getHadoopJobConfFilePath(String suffix, boolean appendSuffix) throws IOException { String hadoopJobConfFile; if (suffix != null && appendSuffix) { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + "_" + suffix.toLowerCase(Locale.ROOT) + ".xml"); } else { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + ".xml"); } File jobConfig = getJobConfig(hadoopJobConfFile); if (jobConfig == null || !jobConfig.exists()) { logger.warn("fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); jobConfig = getJobConfig(HADOOP_JOB_CONF_FILENAME + ".xml"); if (jobConfig == null || !jobConfig.exists()) { logger.error("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); throw new RuntimeException("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); } } return OptionsHelper.convertToFileURL(jobConfig.getAbsolutePath()); }
String fileUrl = OptionsHelper.convertToFileURL(confFile.getAbsolutePath());
String metaDirURI = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); if (metaDirURI.startsWith("/")) // note Path on windows is like "d:/../..." metaDirURI = "file://" + metaDirURI;
protected void dumpKylinPropsAndMetadata(String prj, Set<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); FileUtils.forceDelete(tmp); // we need a directory, so delete the file first File metaDir = new File(tmp, "meta"); metaDir.mkdirs(); // write kylin.properties File kylinPropsFile = new File(metaDir, "kylin.properties"); kylinConfig.exportToFile(kylinPropsFile); if (prj != null) { dumpList.add(ProjectManager.getInstance(kylinConfig).getProject(prj).getResourcePath()); } if (prj != null) { dumpList.add(ProjectManager.getInstance(kylinConfig).getProject(prj).getResourcePath()); } // write resources JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir, dumpList); // hadoop distributed cache String hdfsMetaDir = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); if (hdfsMetaDir.startsWith("/")) // note Path on windows is like "d:/../..." hdfsMetaDir = "file://" + hdfsMetaDir; else hdfsMetaDir = "file:///" + hdfsMetaDir; logger.info("HDFS meta dir is: " + hdfsMetaDir); appendTmpFiles(hdfsMetaDir, conf); }
public String getKafkaConsumerHadoopJobConf() { File kafkaConsumerFile = getKafkaConsumerFile(); return OptionsHelper.convertToFileURL(kafkaConsumerFile.getAbsolutePath()); }
private String getHadoopJobConfFilePath(String suffix, boolean appendSuffix) throws IOException { String hadoopJobConfFile; if (suffix != null && appendSuffix) { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + "_" + suffix.toLowerCase(Locale.ROOT) + ".xml"); } else { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + ".xml"); } File jobConfig = getJobConfig(hadoopJobConfFile); if (jobConfig == null || !jobConfig.exists()) { logger.warn("fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); jobConfig = getJobConfig(HADOOP_JOB_CONF_FILENAME + ".xml"); if (jobConfig == null || !jobConfig.exists()) { logger.error("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); throw new RuntimeException("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); } } return OptionsHelper.convertToFileURL(jobConfig.getAbsolutePath()); }
String fileUrl = OptionsHelper.convertToFileURL(confFile.getAbsolutePath());
protected void dumpKylinPropsAndMetadata(String prj, Set<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); FileUtils.forceDelete(tmp); // we need a directory, so delete the file first File metaDir = new File(tmp, "meta"); metaDir.mkdirs(); // write kylin.properties File kylinPropsFile = new File(metaDir, "kylin.properties"); kylinConfig.exportToFile(kylinPropsFile); if (prj != null) { dumpList.add(ProjectManager.getInstance(kylinConfig).getProject(prj).getResourcePath()); } if (prj != null) { dumpList.add(ProjectManager.getInstance(kylinConfig).getProject(prj).getResourcePath()); } // write resources JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir, dumpList); // hadoop distributed cache String hdfsMetaDir = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); if (hdfsMetaDir.startsWith("/")) // note Path on windows is like "d:/../..." hdfsMetaDir = "file://" + hdfsMetaDir; else hdfsMetaDir = "file:///" + hdfsMetaDir; logger.info("HDFS meta dir is: " + hdfsMetaDir); appendTmpFiles(hdfsMetaDir, conf); }
String metaDirURI = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); if (metaDirURI.startsWith("/")) // note Path on windows is like "d:/../..." metaDirURI = "file://" + metaDirURI;