/** * Get table extended info. Keys are defined in {@link MetadataConstants} * * @param tableName * @return */ public TableExtDesc getTableExt(String tableName, String prj) { TableDesc t = getTableDesc(tableName, prj); if (t == null) return null; return getTableExt(t); }
@Override protected void doSetup(Context context) throws IOException { Configuration conf = context.getConfiguration(); bindCurrentConfiguration(conf); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String project = conf.get(BatchConstants.CFG_PROJECT_NAME); String tableName = conf.get(BatchConstants.CFG_TABLE_NAME); tableDesc = TableMetadataManager.getInstance(config).getTableDesc(tableName, project); tableInputFormat = MRUtil.getTableInputFormat(tableDesc, conf.get(BatchConstants.ARG_CUBING_JOB_ID)); }
public void removeSourceTable(String tableIdentity, String prj) throws IOException { try (AutoLock lock = srcTableMapLock.lockForWrite()) { TableDesc t = getTableDesc(tableIdentity, prj); if (t == null) return; srcTableCrud.delete(t); } }
private List<String> getPkValues(ModelDataGenerator modelGen, Map<String, String> config, List<String> dftPkValues) throws IOException { String pkColName = config.get("pk"); if (pkColName == null) return dftPkValues; int cut = pkColName.lastIndexOf('.'); String pkTableName = pkColName.substring(0, cut); pkColName = pkColName.substring(cut + 1); KylinConfig kylinConfig = modelGen.getModle().getConfig(); String project = modelGen.getModle().getProject(); ColumnDesc pkcol = TableMetadataManager.getInstance(kylinConfig)// .getTableDesc(pkTableName, project).findColumnByName(pkColName); return modelGen.getPkValues(pkcol); }
private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); if (tableDesc.isView()) throw new IllegalArgumentException("Build snapshot of hive view \'" + table + "\' not supported."); SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceManager.createReadableTable(tableDesc, null), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } }
public void updateSnapshotLocalCache(String project, String tableName, String snapshotID) { ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID); TableDesc tableDesc = getTableManager().getTableDesc(tableName, project); if (extTableSnapshotInfo == null) { throw new IllegalArgumentException( "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID); } LookupProviderFactory.rebuildLocalCache(tableDesc, extTableSnapshotInfo); }
public List<TableSnapshotResponse> getLookupTableSnapshots(String project, String tableName) throws IOException { TableDesc tableDesc = getTableManager().getTableDesc(tableName, project); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null); TableSignature signature = hiveTable.getSignature(); return internalGetLookupTableSnapshots(tableName, signature); }
public TableDesc getTableDescByName(String tableName, boolean withExt, String prj) { aclEvaluate.checkProjectReadPermission(prj); TableDesc table = getTableManager().getTableDesc(tableName, prj); if (withExt) { aclEvaluate.checkProjectWritePermission(prj); table = cloneTableDesc(table, prj); } return table; }
public void removeTableDescFromProject(String tableId, String projectName) throws IOException { try (AutoLock lock = prjMapLock.lockForWrite()) { TableMetadataManager metaMgr = getTableManager(); ProjectInstance projectInstance = getProject(projectName); TableDesc table = metaMgr.getTableDesc(tableId, projectName); if (table == null) { throw new IllegalStateException("Cannot find table '" + tableId + "' in metadata manager"); } projectInstance.removeTable(table.getIdentity()); save(projectInstance); } }
public ProjectInstance addTableDescToProject(String[] tableIdentities, String projectName) throws IOException { try (AutoLock lock = prjMapLock.lockForWrite()) { TableMetadataManager metaMgr = getTableManager(); ProjectInstance projectInstance = getProject(projectName); for (String tableId : tableIdentities) { TableDesc table = metaMgr.getTableDesc(tableId, projectName); if (table == null) { throw new IllegalStateException("Cannot find table '" + tableId + "' in metadata manager"); } projectInstance.addTable(table.getIdentity()); } return save(projectInstance); } }
private ILookupTable getExtLookupTable(CubeSegment cubeSegment, String tableName, SnapshotTableDesc snapshotTableDesc) { String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot( snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getExtLookupTable(tableDesc, extTableSnapshot); }
private ILookupTable getInMemLookupTable(CubeSegment cubeSegment, JoinDesc join, SnapshotTableDesc snapshotTableDesc) { String tableName = join.getPKSide().getTableIdentity(); String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); String[] pkCols = join.getPrimaryKey(); try { SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getInMemLookupTable(tableDesc, pkCols, snapshot); } catch (IOException e) { throw new IllegalStateException( "Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); } }
public LookupTable<String> initLookupTable() throws Exception { TableMetadataManager metaMgr = TableMetadataManager.getInstance(config); String tableName = "EDW.TEST_CAL_DT"; String[] pkCols = new String[] { "CAL_DT" }; String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot"; SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = metaMgr.getTableDesc(tableName, "default"); LookupTable<String> lt = new LookupStringTable(tableDesc, pkCols, snapshot); System.out.println(lt); return lt; }
public CubeInstance rebuildLookupSnapshot(CubeInstance cube, String segmentName, String lookupTable) throws IOException { aclEvaluate.checkProjectOperationPermission(cube); Message msg = MsgPicker.getMsg(); TableDesc tableDesc = getTableManager().getTableDesc(lookupTable, cube.getProject()); if (tableDesc.isView()) { throw new BadRequestException( String.format(Locale.ROOT, msg.getREBUILD_SNAPSHOT_OF_VIEW(), tableDesc.getName())); } CubeSegment seg = cube.getSegment(segmentName, SegmentStatusEnum.READY); getCubeManager().buildSnapshotTable(seg, lookupTable, null); return cube; }
@Before public void setup() throws Exception { this.createTestMetadata(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); tableDesc = metadataManager.getTableDesc("TEST_COUNTRY", "default"); }
public JobInstance submitLookupSnapshotJob(CubeInstance cube, String lookupTable, List<String> segmentIDs, String submitter) throws IOException { Message msg = MsgPicker.getMsg(); TableDesc tableDesc = getTableManager().getTableDesc(lookupTable, cube.getProject()); if (tableDesc.isView()) { throw new BadRequestException( String.format(Locale.ROOT, msg.getREBUILD_SNAPSHOT_OF_VIEW(), tableDesc.getName())); } LookupSnapshotBuildJob job = new LookupSnapshotJobBuilder(cube, lookupTable, segmentIDs, submitter).build(); getExecutableManager().addJob(job); JobInstance jobInstance = getLookupSnapshotBuildJobInstance(job); return jobInstance; }
@Before public void setup() throws Exception { this.createTestMetadata(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); tableDesc = metadataManager.getTableDesc("TEST_COUNTRY", "default"); }
@Before public void setup() throws Exception { createTestMetadata(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); this.random = new Random(); tableDesc = metadataManager.getTableDesc("TEST_COUNTRY", "default"); sourceRowNum = 10000; genTestData(); lookupTable = new RocksDBLookupTable(tableDesc, new String[] { "COUNTRY" }, "lookup_cache/TEST_COUNTRY"); }
@Before public void setup() throws Exception { createTestMetadata(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); kylinConfig = getTestConfig(); tableDesc = metadataManager.getTableDesc(TABLE_COUNTRY, "default"); cleanCache(); LookupProviderFactory.registerLookupProvider(MOCK_EXT_LOOKUP, MockedLookupProvider.class.getName()); }
@Test public void testFindTableByName() throws Exception { TableDesc table = getInstance(getTestConfig()).getTableDesc("EDW.TEST_CAL_DT", "default"); Assert.assertNotNull(table); Assert.assertEquals("EDW.TEST_CAL_DT", table.getIdentity()); }