public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
private void saveExtSnapshotIfNeeded(CubeManager cubeManager, CubeInstance cube, CubeSegment segment) throws IOException { String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO); if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) { return; } Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr); logger.info("update ext lookup snapshots:{}", extLookupSnapshotMap); List<SnapshotTableDesc> snapshotTableDescList = cube.getDescriptor().getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) { String tableName = snapshotTableDesc.getTableName(); if (snapshotTableDesc.isExtSnapshotTable()) { String newSnapshotResPath = extLookupSnapshotMap.get(tableName); if (newSnapshotResPath == null || newSnapshotResPath.isEmpty()) { continue; } if (snapshotTableDesc.isGlobal()) { if (!newSnapshotResPath.equals(cube.getSnapshotResPath(tableName))) { cubeManager.updateCubeLookupSnapshot(cube, tableName, newSnapshotResPath); } } else { segment.putSnapshotResPath(tableName, newSnapshotResPath); } } } }
public List<String> getAllExtLookupSnapshotTypes() { List<String> result = Lists.newArrayList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) { if (snapshotTableDesc.isExtSnapshotTable()) { result.add(snapshotTableDesc.getStorageType()); } } return result; }
public boolean isExtSnapshotTable(String tableName) { SnapshotTableDesc desc = getSnapshotTableDesc(tableName); if (desc == null) { return false; } return desc.isExtSnapshotTable(); }
public SnapshotTableDesc getSnapshotTableDesc(String tableName) { for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) { if (snapshotTableDesc.getTableName().equalsIgnoreCase(tableName)) { return snapshotTableDesc; } } return null; }
private void addExtMaterializeLookupTableSteps(final LookupSnapshotBuildJob result, SnapshotTableDesc snapshotTableDesc) { LookupMaterializeContext lookupMaterializeContext = new LookupMaterializeContext(result); ILookupMaterializer materializer = MRUtil.getExtLookupMaterializer(snapshotTableDesc.getStorageType()); materializer.materializeLookupTable(lookupMaterializeContext, cube, lookupTable); UpdateCubeAfterSnapshotStep afterSnapshotStep = new UpdateCubeAfterSnapshotStep(); afterSnapshotStep.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_LOOKUP_TABLE_UPDATE_CUBE); afterSnapshotStep.getParams().put(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO, lookupMaterializeContext.getAllLookupSnapshotsInString()); LookupExecutableUtil.setCubeName(cube.getName(), afterSnapshotStep.getParams()); LookupExecutableUtil.setLookupTableName(lookupTable, afterSnapshotStep.getParams()); LookupExecutableUtil.setSegments(segments, afterSnapshotStep.getParams()); LookupExecutableUtil.setJobID(result.getId(), afterSnapshotStep.getParams()); result.addTask(afterSnapshotStep); }
public boolean isGlobalSnapshotTable(String tableName) { SnapshotTableDesc desc = getSnapshotTableDesc(tableName); if (desc == null) { return false; } return desc.isGlobal(); }
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
public List<String> getAllExtLookupSnapshotTypes() { List<String> result = Lists.newArrayList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) { if (snapshotTableDesc.isExtSnapshotTable()) { result.add(snapshotTableDesc.getStorageType()); } } return result; }
public LookupSnapshotBuildJob build() { logger.info("new job to build lookup snapshot:{} for cube:{}", lookupTable, cube.getName()); LookupSnapshotBuildJob result = LookupSnapshotBuildJob.createJob(cube, lookupTable, submitter, kylinConfig); CubeDesc cubeDesc = cube.getDescriptor(); SnapshotTableDesc snapshotTableDesc = cubeDesc.getSnapshotTableDesc(lookupTable); if (snapshotTableDesc != null && snapshotTableDesc.isExtSnapshotTable()) { addExtMaterializeLookupTableSteps(result, snapshotTableDesc); } else { addInMetaStoreMaterializeLookupTableSteps(result); } return result; }
public SnapshotTableDesc getSnapshotTableDesc(String tableName) { for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) { if (snapshotTableDesc.getTableName().equalsIgnoreCase(tableName)) { return snapshotTableDesc; } } return null; }
private void addExtMaterializeLookupTableSteps(final LookupSnapshotBuildJob result, SnapshotTableDesc snapshotTableDesc) { LookupMaterializeContext lookupMaterializeContext = new LookupMaterializeContext(result); ILookupMaterializer materializer = MRUtil.getExtLookupMaterializer(snapshotTableDesc.getStorageType()); materializer.materializeLookupTable(lookupMaterializeContext, cube, lookupTable); UpdateCubeAfterSnapshotStep afterSnapshotStep = new UpdateCubeAfterSnapshotStep(); afterSnapshotStep.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_LOOKUP_TABLE_UPDATE_CUBE); afterSnapshotStep.getParams().put(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO, lookupMaterializeContext.getAllLookupSnapshotsInString()); LookupExecutableUtil.setCubeName(cube.getName(), afterSnapshotStep.getParams()); LookupExecutableUtil.setLookupTableName(lookupTable, afterSnapshotStep.getParams()); LookupExecutableUtil.setSegments(segments, afterSnapshotStep.getParams()); LookupExecutableUtil.setJobID(result.getId(), afterSnapshotStep.getParams()); result.addTask(afterSnapshotStep); }
private String getSnapshotResPath(CubeSegment cubeSegment, String tableName, SnapshotTableDesc snapshotTableDesc) { String snapshotResPath; if (snapshotTableDesc == null || !snapshotTableDesc.isGlobal()) { snapshotResPath = cubeSegment.getSnapshotResPath(tableName); } else { snapshotResPath = cubeSegment.getCubeInstance().getSnapshotResPath(tableName); } if (snapshotResPath == null) { throw new IllegalStateException("No snapshot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); } return snapshotResPath; }
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
private void saveExtSnapshotIfNeeded(CubeManager cubeManager, CubeInstance cube, CubeSegment segment) throws IOException { String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO); if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) { return; } Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr); logger.info("update ext lookup snapshots:{}", extLookupSnapshotMap); List<SnapshotTableDesc> snapshotTableDescList = cube.getDescriptor().getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) { String tableName = snapshotTableDesc.getTableName(); if (snapshotTableDesc.isExtSnapshotTable()) { String newSnapshotResPath = extLookupSnapshotMap.get(tableName); if (newSnapshotResPath == null || newSnapshotResPath.isEmpty()) { continue; } if (snapshotTableDesc.isGlobal()) { if (!newSnapshotResPath.equals(cube.getSnapshotResPath(tableName))) { cubeManager.updateCubeLookupSnapshot(cube, tableName, newSnapshotResPath); } } else { segment.putSnapshotResPath(tableName, newSnapshotResPath); } } } }
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
public ILookupTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) { String tableName = join.getPKSide().getTableIdentity(); CubeDesc cubeDesc = cubeSegment.getCubeDesc(); SnapshotTableDesc snapshotTableDesc = cubeDesc.getSnapshotTableDesc(tableName); if (snapshotTableDesc == null || !snapshotTableDesc.isExtSnapshotTable()) { return getInMemLookupTable(cubeSegment, join, snapshotTableDesc); } else { return getExtLookupTable(cubeSegment, tableName, snapshotTableDesc); } }
public boolean isGlobalSnapshotTable(String tableName) { SnapshotTableDesc desc = getSnapshotTableDesc(tableName); if (desc == null) { return false; } return desc.isGlobal(); }
public boolean isExtSnapshotTable(String tableName) { SnapshotTableDesc desc = getSnapshotTableDesc(tableName); if (desc == null) { return false; } return desc.isExtSnapshotTable(); }
private String getSnapshotResPath(CubeSegment cubeSegment, String tableName, SnapshotTableDesc snapshotTableDesc) { String snapshotResPath; if (snapshotTableDesc == null || !snapshotTableDesc.isGlobal()) { snapshotResPath = cubeSegment.getSnapshotResPath(tableName); } else { snapshotResPath = cubeSegment.getCubeInstance().getSnapshotResPath(tableName); } if (snapshotResPath == null) { throw new IllegalStateException("No snapshot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); } return snapshotResPath; }