private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); if (tableDesc.isView()) throw new IllegalArgumentException("Build snapshot of hive view \'" + table + "\' not supported."); SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceManager.createReadableTable(tableDesc, null), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } }
public List<TableSnapshotResponse> getLookupTableSnapshots(String project, String tableName) throws IOException { TableDesc tableDesc = getTableManager().getTableDesc(tableName, project); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null); TableSignature signature = hiveTable.getSignature(); return internalGetLookupTableSnapshots(tableName, signature); }
removeSnapshotIfExist(extSnapshotInfoManager, kylinConfig, tableName, lookupSnapshotID); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, jobId);
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { KylinConfig kylinConfig = context.getConfig(); CubeManager cubeManager = CubeManager.getInstance(kylinConfig); TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig); SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig); CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams())); List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams()); String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams()); CubeDesc cubeDesc = cube.getDescriptor(); try { TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject()); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null); logger.info("take snapshot for table:" + lookupTableName); SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig()); logger.info("update snapshot path to cube metadata"); if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) { LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName, snapshot.getResourcePath()); } else { LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName, snapshot.getResourcePath()); } return new ExecuteResult(); } catch (IOException e) { logger.error("fail to build snapshot for:" + lookupTableName, e); return ExecuteResult.createError(e); } }
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException { // work on copy instead of cached objects CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid()); TableMetadataManager metaMgr = getTableManager(); SnapshotManager snapshotMgr = getSnapshotManager(); TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject())); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid); SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig()); CubeDesc cubeDesc = cubeSeg.getCubeDesc(); if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) { segCopy.putSnapshotResPath(lookupTable, snapshot.getResourcePath()); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(segCopy); updateCube(update); // Update the input cubeSeg after the resource store updated cubeSeg.putSnapshotResPath(lookupTable, segCopy.getSnapshotResPath(lookupTable)); } else { CubeUpdate cubeUpdate = new CubeUpdate(cubeCopy); Map<String, String> map = Maps.newHashMap(); map.put(lookupTable, snapshot.getResourcePath()); cubeUpdate.setUpdateTableSnapshotPath(map); updateCube(cubeUpdate); cubeSeg.getCubeInstance().putSnapshotResPath(lookupTable, snapshot.getResourcePath()); } return snapshot; } }
private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); if (tableDesc.isView()) throw new IllegalArgumentException("Build snapshot of hive view \'" + table + "\' not supported."); SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceManager.createReadableTable(tableDesc, null), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } }
removeSnapshotIfExist(extSnapshotInfoManager, kylinConfig, tableName, lookupSnapshotID); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, jobId);
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { KylinConfig kylinConfig = context.getConfig(); CubeManager cubeManager = CubeManager.getInstance(kylinConfig); TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig); SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig); CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams())); List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams()); String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams()); CubeDesc cubeDesc = cube.getDescriptor(); try { TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject()); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null); logger.info("take snapshot for table:" + lookupTableName); SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig()); logger.info("update snapshot path to cube metadata"); if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) { LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName, snapshot.getResourcePath()); } else { LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName, snapshot.getResourcePath()); } return new ExecuteResult(); } catch (IOException e) { logger.error("fail to build snapshot for:" + lookupTableName, e); return ExecuteResult.createError(e); } }
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException { // work on copy instead of cached objects CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid()); TableMetadataManager metaMgr = getTableManager(); SnapshotManager snapshotMgr = getSnapshotManager(); TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject())); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid); SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig()); CubeDesc cubeDesc = cubeSeg.getCubeDesc(); if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) { segCopy.putSnapshotResPath(lookupTable, snapshot.getResourcePath()); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(segCopy); updateCube(update); // Update the input cubeSeg after the resource store updated cubeSeg.putSnapshotResPath(lookupTable, segCopy.getSnapshotResPath(lookupTable)); } else { CubeUpdate cubeUpdate = new CubeUpdate(cubeCopy); Map<String, String> map = Maps.newHashMap(); map.put(lookupTable, snapshot.getResourcePath()); cubeUpdate.setUpdateTableSnapshotPath(map); updateCube(cubeUpdate); cubeSeg.getCubeInstance().putSnapshotResPath(lookupTable, snapshot.getResourcePath()); } return snapshot; } }