private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); if (tableDesc.isView()) throw new IllegalArgumentException("Build snapshot of hive view \'" + table + "\' not supported."); SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceManager.createReadableTable(tableDesc, null), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } }
public static ISource getDefaultSource() { final KylinConfig sysConfig = KylinConfig.getInstanceFromEnv(); return getSource(new ISourceAware() { @Override public int getSourceType() { return sysConfig.getDefaultSource(); } @Override public KylinConfig getConfig() { return sysConfig; } }); }
public static IMRTableInputFormat getTableInputFormat(TableDesc tableDesc, String uuid) { return SourceManager.createEngineAdapter(tableDesc, IMRInput.class).getTableInputFormat(tableDesc, uuid); }
public static ISource getSource(ISourceAware aware) { return getInstance(aware.getConfig()).getCachedSource(aware); }
public ISource getProjectSource(String projectName) { ProjectInstance projectInstance = ProjectManager.getInstance(systemConfig).getProject(projectName); if (projectInstance != null) return getCachedSource(projectInstance); else return getDefaultSource(); }
/** * * @param project * @param database * @return * @throws Exception */ public List<String> getSourceTableNames(String project, String database) throws Exception { ISourceMetadataExplorer explr = SourceManager.getInstance(getConfig()).getProjectSource(project) .getSourceMetadataExplorer(); return explr.listTables(database); }
protected List<String> getHiveTables() throws Exception { ISourceMetadataExplorer explr = SourceManager.getDefaultSource().getSourceMetadataExplorer(); return explr.listTables(config.getHiveDatabaseForIntermediateTable()); }
SourceManager sourceManager = SourceManager.getInstance(KylinConfig.getInstanceFromEnv()); ISource source = sourceManager.getCachedSource(desc); source.unloadTable(tableName, project); return rtn;
public ISource getProjectSource(String projectName) { ProjectInstance projectInstance = ProjectManager.getInstance(systemConfig).getProject(projectName); if (projectInstance != null) return getCachedSource(projectInstance); else return getDefaultSource(); }
/** * * @param project * @return * @throws Exception */ public List<String> getSourceDbNames(String project) throws Exception { ISourceMetadataExplorer explr = SourceManager.getInstance(getConfig()).getProjectSource(project) .getSourceMetadataExplorer(); return explr.listDatabases(); }
public List<TableSnapshotResponse> getLookupTableSnapshots(String project, String tableName) throws IOException { TableDesc tableDesc = getTableManager().getTableDesc(tableName, project); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null); TableSignature signature = hiveTable.getSignature(); return internalGetLookupTableSnapshots(tableName, signature); }
public static IReadableTable createReadableTable(TableDesc table, String uuid) { return getSource(table).createReadableTable(table, uuid); }
public static IMRBatchMergeInputSide getBatchMergeInputSide(CubeSegment seg) { return (IMRBatchMergeInputSide)SourceManager.createEngineAdapter(seg, IMRInput.class).getBatchMergeInputSide(seg); }
public static ISource getSource(ISourceAware aware) { return getInstance(aware.getConfig()).getCachedSource(aware); }
removeSnapshotIfExist(extSnapshotInfoManager, kylinConfig, tableName, lookupSnapshotID); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, jobId);
public static <T> T createEngineAdapter(ISourceAware table, Class<T> engineInterface) { return getSource(table).adaptToBuildEngine(engineInterface); }
public static ISparkBatchMergeInputSide getBatchMergeInputSide(CubeSegment seg) { return (ISparkBatchMergeInputSide)SourceManager.createEngineAdapter(seg, ISparkInput.class).getBatchMergeInputSide(seg); }
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
public static List<String> getMRDependentResources(TableDesc table) { return getSource(table).getSourceMetadataExplorer().getRelatedKylinResources(table); } }
public static IMRBatchCubingInputSide getBatchCubingInputSide(CubeSegment seg) { IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(seg); return (IMRBatchCubingInputSide)SourceManager.createEngineAdapter(seg, IMRInput.class).getBatchCubingInputSide(flatDesc); }