private HadoopShellExecutable createCreateHTableStep(IISegment seg) { HadoopShellExecutable createHtableStep = new HadoopShellExecutable(); createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE); StringBuilder cmd = new StringBuilder(); appendExecCmdParameters(cmd, "iiname", seg.getIIInstance().getName()); appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier()); createHtableStep.setJobParams(cmd.toString()); createHtableStep.setJobClass(IICreateHTableJob.class); return createHtableStep; }
public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, HConnection conn) throws Throwable { String tableName = segment.getStorageLocationIdentifier(); table = conn.getTable(tableName); factTableName = segment.getIIDesc().getFactTableName(); rewriteMeasureParameters(measures, segment.getColumns()); this.measures = measures; this.columns = segment.getColumns(); this.columnNames = getColumnNames(columns);
private IIJob initialJob(IISegment seg, String type) { IIJob result = new IIJob(); SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone(engineConfig.getTimeZone())); result.setIIName(seg.getIIInstance().getName()); result.setSegmentId(seg.getUuid()); result.setName(seg.getIIInstance().getName() + " - " + seg.getName() + " - " + type + " - " + format.format(new Date(System.currentTimeMillis()))); result.setSubmitter(this.submitter); return result; }
/** * @param IIInstance * @param startDate (pass 0 if full build) * @param endDate (pass 0 if full build) * @return */ public IISegment buildSegment(IIInstance IIInstance, long startDate, long endDate) { IISegment segment = new IISegment(); String incrementalSegName = IISegment.getSegmentName(startDate, endDate); segment.setUuid(UUID.randomUUID().toString()); segment.setName(incrementalSegName); segment.setCreateTimeUTC(System.currentTimeMillis()); segment.setDateRangeStart(startDate); segment.setDateRangeEnd(endDate); segment.setStatus(SegmentStatusEnum.NEW); segment.setStorageLocationIdentifier(generateStorageLocation()); segment.setIIInstance(IIInstance); return segment; }
@Override public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) { String tableName = seg.getStorageLocationIdentifier(); //HConnection is cached, so need not be closed HConnection conn = HBaseConnection.get(context.getConnUrl()); try { return new EndpointTupleIterator(seg, sqlDigest.filter, sqlDigest.groupbyColumns, new ArrayList<>(sqlDigest.aggregations), context, conn); } catch (Throwable e) { e.printStackTrace(); throw new IllegalStateException("Error when connecting to II htable " + tableName, e); } } }
public IIJob buildJob(IISegment seg) { checkPreconditions(seg); IIJob result = initialJob(seg, "BUILD"); final String jobId = result.getId(); final IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(seg.getIIDesc()); final String intermediateHiveTableName = getIntermediateHiveTableName(intermediateTableDesc, jobId); final String factDistinctColumnsPath = getIIDistinctColumnsPath(seg, jobId); final String iiRootPath = getJobWorkingDir(jobId) + "/" + seg.getIIInstance().getName() + "/"; final String iiPath = iiRootPath + "*"; final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId); result.addTask(intermediateHiveTableStep); result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId, factDistinctColumnsPath)); result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath)); result.addTask(createInvertedIndexStep(seg, intermediateHiveTableName, iiRootPath)); // create htable step result.addTask(createCreateHTableStep(seg)); // generate hfiles step result.addTask(createConvertToHfileStep(seg, iiPath, jobId)); // bulk load step result.addTask(createBulkLoadStep(seg, jobId)); return result; }
public IIDesc getIIDesc() { return getIIInstance().getDescriptor(); }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME); String input = getOptionValue(OPTION_INPUT_PATH); String iiname = getOptionValue(OPTION_II_NAME); FileSystem fs = FileSystem.get(getConf()); FsPermission permission = new FsPermission((short) 0777); fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission); int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName }); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiname); IISegment seg = ii.getFirstSegment(); seg.setStorageLocationIdentifier(tableName); seg.setStatus(SegmentStatusEnum.READY); mgr.updateII(ii); return hbaseExitCode; } catch (Exception e) { printUsage(options); throw e; } }
public TableRecordInfo(IISegment iiSegment) { this(iiSegment.getIIDesc()); }
public void buildInvertedIndexDictionary(IISegment iiSeg, String factColumnsPath) throws IOException { logger.info("Start building ii dictionary"); DictionaryManager dictMgr = getDictionaryManager(); IIDesc iiDesc = iiSeg.getIIInstance().getDescriptor(); for (TblColRef column : iiDesc.listAllColumns()) { logger.info("Dealing with column {}", column); if (iiDesc.isMetricsCol(column)) { continue; } DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath); iiSeg.putDictResPath(column, dict.getResourcePath()); } saveResource(iiSeg.getIIInstance()); }
private void attachMetadata(IIInstance ii) throws IOException { Configuration conf = job.getConfiguration(); attachKylinPropsAndMetadata(ii, conf); IISegment seg = ii.getFirstSegment(); conf.set(BatchConstants.CFG_II_NAME, ii.getName()); conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName()); }
public TableRecordInfo(IISegment iiSegment) { seg = iiSegment; desc = seg.getIIInstance().getDescriptor(); allColumns = desc.listAllColumns(); nColumns = allColumns.size(); dictionaries = new Dictionary<?>[nColumns]; measureSerializers = new FixedLenMeasureCodec<?>[nColumns]; DictionaryManager dictMgr = DictionaryManager.getInstance(desc.getConfig()); int index = 0; for (TblColRef tblColRef : desc.listAllColumns()) { ColumnDesc col = tblColRef.getColumn(); if (desc.isMetricsCol(index)) { measureSerializers[index] = FixedLenMeasureCodec.get(col.getType()); } else { String dictPath = seg.getDictResPath(tblColRef); try { dictionaries[index] = dictMgr.getDictionary(dictPath); } catch (IOException e) { throw new RuntimeException("dictionary " + dictPath + " does not exist ", e); } } index++; } digest = createDigest(); }
/** * @param IIInstance * @param startDate (pass 0 if full build) * @param endDate (pass 0 if full build) * @return */ public IISegment buildSegment(IIInstance IIInstance, long startDate, long endDate) { IISegment segment = new IISegment(); String incrementalSegName = IISegment.getSegmentName(startDate, endDate); segment.setUuid(UUID.randomUUID().toString()); segment.setName(incrementalSegName); segment.setCreateTimeUTC(System.currentTimeMillis()); segment.setDateRangeStart(startDate); segment.setDateRangeEnd(endDate); segment.setStatus(SegmentStatusEnum.NEW); segment.setStorageLocationIdentifier(generateStorageLocation()); segment.setIIInstance(IIInstance); return segment; }
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<String>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) { for (IISegment seg : ii.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } return result; } }
public IIJob buildJob(IISegment seg) { checkPreconditions(seg); IIJob result = initialJob(seg, "BUILD"); final String jobId = result.getId(); final IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(seg.getIIDesc()); final String intermediateHiveTableName = getIntermediateHiveTableName(intermediateTableDesc, jobId); final String factDistinctColumnsPath = getIIDistinctColumnsPath(seg, jobId); final String iiRootPath = getJobWorkingDir(jobId) + "/" + seg.getIIInstance().getName() + "/"; final String iiPath = iiRootPath + "*"; final AbstractExecutable intermediateHiveTableStep = createIntermediateHiveTableStep(intermediateTableDesc, jobId); result.addTask(intermediateHiveTableStep); result.addTask(createFactDistinctColumnsStep(seg, intermediateHiveTableName, jobId, factDistinctColumnsPath)); result.addTask(createBuildDictionaryStep(seg, factDistinctColumnsPath)); result.addTask(createInvertedIndexStep(seg, intermediateHiveTableName, iiRootPath)); // create htable step result.addTask(createCreateHTableStep(seg)); // generate hfiles step result.addTask(createConvertToHfileStep(seg, iiPath, jobId)); // bulk load step result.addTask(createBulkLoadStep(seg, jobId)); return result; }
private String getIIDistinctColumnsPath(IISegment seg, String jobUuid) { return getJobWorkingDir(jobUuid) + "/" + seg.getIIInstance().getName() + "/ii_distinct_columns"; }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_HTABLE_NAME); options.addOption(OPTION_II_NAME); parseOptions(options, args); String tableName = getOptionValue(OPTION_HTABLE_NAME); String input = getOptionValue(OPTION_INPUT_PATH); String iiname = getOptionValue(OPTION_II_NAME); FileSystem fs = FileSystem.get(getConf()); FsPermission permission = new FsPermission((short) 0777); fs.setPermission(new Path(input, IIDesc.HBASE_FAMILY), permission); int hbaseExitCode = ToolRunner.run(new LoadIncrementalHFiles(getConf()), new String[] { input, tableName }); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); IIInstance ii = mgr.getII(iiname); IISegment seg = ii.getFirstSegment(); seg.setStorageLocationIdentifier(tableName); seg.setStatus(SegmentStatusEnum.READY); mgr.updateII(ii); return hbaseExitCode; } catch (Exception e) { printUsage(options); throw e; } }
@Override public IJoinedFlatTableDesc getJoinedFlatTableDesc() { return new IIJoinedFlatTableDesc(this.getIIDesc()); } }