private void afterIIUpdated(IIInstance updatedII) { iiMap.put(updatedII.getName(), updatedII); }
public IIInstance updateII(IIInstance ii) throws IOException { logger.info("Updating II instance '" + ii.getName()); // save resource saveResource(ii); logger.info("II with " + ii.getSegments().size() + " segments is saved"); return ii; }
private void attachMetadata(IIInstance ii) throws IOException { Configuration conf = job.getConfiguration(); attachKylinPropsAndMetadata(ii, conf); IISegment seg = ii.getFirstSegment(); conf.set(BatchConstants.CFG_II_NAME, ii.getName()); conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName()); }
private String getIIDistinctColumnsPath(IISegment seg, String jobUuid) { return getJobWorkingDir(jobUuid) + "/" + seg.getIIInstance().getName() + "/ii_distinct_columns"; }
private String getHFilePath(IISegment seg, String jobId) { return getJobWorkingDir(jobId) + "/" + seg.getIIInstance().getName() + "/hfile/"; }
private IIJob initialJob(IISegment seg, String type) { IIJob result = new IIJob(); SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone(engineConfig.getTimeZone())); result.setIIName(seg.getIIInstance().getName()); result.setSegmentId(seg.getUuid()); result.setName(seg.getIIInstance().getName() + " - " + seg.getName() + " - " + type + " - " + format.format(new Date(System.currentTimeMillis()))); result.setSubmitter(this.submitter); return result; }
public void removeIICache(IIInstance ii) { iiMap.remove(ii.getName()); for (IISegment segment : ii.getSegments()) { usedStorageLocation.remove(segment.getName()); } }
private synchronized IIInstance loadIIInstance(String path) throws IOException { ResourceStore store = getStore(); logger.debug("Loading IIInstance " + store.getReadableResourcePath(path)); IIInstance IIInstance = null; try { IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER); IIInstance.setConfig(config); if (StringUtils.isBlank(IIInstance.getName())) throw new IllegalStateException("IIInstance name must not be blank"); iiMap.putLocal(IIInstance.getName(), IIInstance); for (IISegment segment : IIInstance.getSegments()) { usedStorageLocation.add(segment.getName()); } return IIInstance; } catch (Exception e) { logger.error("Error during load ii instance " + path, e); return null; } }
private HadoopShellExecutable createBuildDictionaryStep(IISegment seg, String factDistinctColumnsPath) { // base cuboid job HadoopShellExecutable buildDictionaryStep = new HadoopShellExecutable(); buildDictionaryStep.setName(ExecutableConstants.STEP_NAME_BUILD_DICTIONARY); StringBuilder cmd = new StringBuilder(); appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName()); appendExecCmdParameters(cmd, "input", factDistinctColumnsPath); buildDictionaryStep.setJobParams(cmd.toString()); buildDictionaryStep.setJobClass(CreateInvertedIndexDictionaryJob.class); return buildDictionaryStep; }
private MapReduceExecutable createFactDistinctColumnsStep(IISegment seg, String factTableName, String jobId, String output) { MapReduceExecutable result = new MapReduceExecutable(); result.setName(ExecutableConstants.STEP_NAME_FACT_DISTINCT_COLUMNS); result.setMapReduceJobClass(IIDistinctColumnsJob.class); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd, engineConfig); appendExecCmdParameters(cmd, "tablename", factTableName); appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName()); appendExecCmdParameters(cmd, "output", output); appendExecCmdParameters(cmd, "jobname", "Kylin_Fact_Distinct_Columns_" + seg.getIIInstance().getName() + "_Step"); result.setMapReduceParams(cmd.toString()); return result; }
private HadoopShellExecutable createCreateHTableStep(IISegment seg) { HadoopShellExecutable createHtableStep = new HadoopShellExecutable(); createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE); StringBuilder cmd = new StringBuilder(); appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName()); appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier()); createHtableStep.setJobParams(cmd.toString()); createHtableStep.setJobClass(IICreateHTableJob.class); return createHtableStep; }
private HadoopShellExecutable createBulkLoadStep(IISegment seg, String jobId) { HadoopShellExecutable bulkLoadStep = new HadoopShellExecutable(); bulkLoadStep.setName(ExecutableConstants.STEP_NAME_BULK_LOAD_HFILE); StringBuilder cmd = new StringBuilder(); appendExecCmdParameters(cmd, "input", getHFilePath(seg, jobId)); appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier()); appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName()); bulkLoadStep.setJobParams(cmd.toString()); bulkLoadStep.setJobClass(IIBulkLoadJob.class); return bulkLoadStep; }
private MapReduceExecutable createConvertToHfileStep(IISegment seg, String inputPath, String jobId) { MapReduceExecutable createHFilesStep = new MapReduceExecutable(); createHFilesStep.setName(ExecutableConstants.STEP_NAME_CONVERT_II_TO_HFILE); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd, engineConfig); appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName()); appendExecCmdParameters(cmd, "input", inputPath); appendExecCmdParameters(cmd, "output", getHFilePath(seg, jobId)); appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier()); appendExecCmdParameters(cmd, "jobname", "Kylin_HFile_Generator_" + seg.getIIInstance().getName() + "_Step"); createHFilesStep.setMapReduceParams(cmd.toString()); createHFilesStep.setMapReduceJobClass(IICreateHFileJob.class); return createHFilesStep; }
private MapReduceExecutable createInvertedIndexStep(IISegment seg, String intermediateHiveTable, String iiOutputTempPath) { // base cuboid job MapReduceExecutable buildIIStep = new MapReduceExecutable(); StringBuilder cmd = new StringBuilder(); appendMapReduceParameters(cmd, engineConfig); buildIIStep.setName(ExecutableConstants.STEP_NAME_BUILD_II); appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName()); appendExecCmdParameters(cmd, "tablename", intermediateHiveTable); appendExecCmdParameters(cmd, "output", iiOutputTempPath); appendExecCmdParameters(cmd, "jobname", ExecutableConstants.STEP_NAME_BUILD_II); buildIIStep.setMapReduceParams(cmd.toString()); buildIIStep.setMapReduceJobClass(InvertedIndexJob.class); return buildIIStep; }
String tablename = seg.getStorageLocationIdentifier(); allTablesNeedToBeDropped.remove(tablename); log.info("Remove table " + tablename + " from drop list, as the table belongs to ii " + ii.getName() + " with status " + ii.getStatus());
public IIInstance createII(IIInstance ii) throws IOException { if (this.getII(ii.getName()) != null) throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists."); this.updateII(ii); // FIXME need to pass in project name String projectName = ProjectInstance.DEFAULT_PROJECT_NAME; ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner()); return ii; }
private void attachMetadata(IIInstance ii) throws IOException { Configuration conf = job.getConfiguration(); attachKylinPropsAndMetadata(ii, conf); IISegment seg = ii.getFirstSegment(); conf.set(BatchConstants.CFG_II_NAME, ii.getName()); conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName()); }
public IIJob buildJob(IISegment seg) { checkPreconditions(seg); IIJob result = initialJob(seg, "BUILD"); final String jobId = result.getId(); final IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(seg.getIIDesc()); final String intermediateHiveTableName = getIntermediateHiveTableName(intermediateTableDesc, jobId); final String factDistinctColumnsPath = getIIDistinctColumnsPath(seg, jobId); final String iiRootPath = getJobWorkingDir(jobId) + "/" + seg.getIIInstance().getName() + "/"; final String iiPath = iiRootPath + "*"; final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId); result.addTask(intermediateHiveTableStep); result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId, factDistinctColumnsPath)); result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath)); result.addTask(createInvertedIndexStep(seg, intermediateHiveTableName, iiRootPath)); // create htable step result.addTask(createCreateHTableStep(seg)); // generate hfiles step result.addTask(createConvertToHfileStep(seg, iiPath, jobId)); // bulk load step result.addTask(createBulkLoadStep(seg, jobId)); return result; }
private HadoopShellExecutable createBulkLoadStep(IISegment seg, String jobId) { HadoopShellExecutable bulkLoadStep = new HadoopShellExecutable(); bulkLoadStep.setName(ExecutableConstants.STEP_NAME_BULK_LOAD_HFILE); StringBuilder cmd = new StringBuilder(); appendExecCmdParameters(cmd, "input", getHFilePath(seg, jobId)); appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier()); appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName()); bulkLoadStep.setJobParams(cmd.toString()); bulkLoadStep.setJobClass(IIBulkLoadJob.class); return bulkLoadStep; }