private TableMetadataManager getMetadataManager() { return TableMetadataManager.getInstance(config); }
@Override protected void doSetup(Context context) throws IOException { Configuration conf = context.getConfiguration(); bindCurrentConfiguration(conf); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String project = conf.get(BatchConstants.CFG_PROJECT_NAME); String tableName = conf.get(BatchConstants.CFG_TABLE_NAME); tableDesc = TableMetadataManager.getInstance(config).getTableDesc(tableName, project); tableInputFormat = MRUtil.getTableInputFormat(tableDesc, conf.get(BatchConstants.ARG_CUBING_JOB_ID)); }
private ExtTableSnapshotInfo load(String resourcePath) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); ExtTableSnapshotInfo snapshot = store.getResource(resourcePath, SNAPSHOT_SERIALIZER); return snapshot; }
public void removeSnapshot(String tableName, String snapshotID) throws IOException { String snapshotResPath = ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID); snapshotCache.invalidate(snapshotResPath); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); store.deleteResource(snapshotResPath); }
public List<SnapshotTable> getSnapshots(String tableName, TableSignature sourceTableSignature) throws IOException { List<SnapshotTable> result = Lists.newArrayList(); String tableSnapshotsPath = SnapshotTable.getResourceDir(tableName); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); result.addAll(store.getAllResources(tableSnapshotsPath, SnapshotTableSerializer.INFO_SERIALIZER)); if (sourceTableSignature != null) { String oldTableSnapshotsPath = SnapshotTable.getOldResourceDir(sourceTableSignature); result.addAll(store.getAllResources(oldTableSnapshotsPath, SnapshotTableSerializer.INFO_SERIALIZER)); } return result; }
public void save(ExtTableSnapshotInfo snapshot) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); String path = snapshot.getResourcePath(); store.checkAndPutResource(path, snapshot, SNAPSHOT_SERIALIZER); }
public List<ExtTableSnapshotInfo> getSnapshots(String tableName) throws IOException { String tableSnapshotsPath = ExtTableSnapshotInfo.getResourceDir(tableName); ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); return store.getAllResources(tableSnapshotsPath, SNAPSHOT_SERIALIZER); }
private List<String> getPkValues(ModelDataGenerator modelGen, Map<String, String> config, List<String> dftPkValues) throws IOException { String pkColName = config.get("pk"); if (pkColName == null) return dftPkValues; int cut = pkColName.lastIndexOf('.'); String pkTableName = pkColName.substring(0, cut); pkColName = pkColName.substring(cut + 1); KylinConfig kylinConfig = modelGen.getModle().getConfig(); String project = modelGen.getModle().getProject(); ColumnDesc pkcol = TableMetadataManager.getInstance(kylinConfig)// .getTableDesc(pkTableName, project).findColumnByName(pkColName); return modelGen.getPkValues(pkcol); }
private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); if (tableDesc.isView()) throw new IllegalArgumentException("Build snapshot of hive view \'" + table + "\' not supported."); SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceManager.createReadableTable(tableDesc, null), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } }
@Override public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException { //clean up the current project's table desc TableMetadataManager.getInstance(config).resetProjectSpecificTableDesc(project); try (AutoLock lock = modelMapLock.lockForWrite()) { for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) { crud.reloadQuietly(model); } } }
@Test public void testGetInstance() throws Exception { Assert.assertNotNull(getInstance(getTestConfig())); Assert.assertNotNull(getInstance(getTestConfig()).listAllTables(null)); Assert.assertTrue(getInstance(getTestConfig()).listAllTables(null).size() > 0); }
@Test public void testTableExtCompatibility() throws IOException { String tableName = "DEFAULT.WIDE_TABLE"; Map<String, String> oldTableExt = new HashMap<>(); oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4"); mockUpOldTableExtJson(tableName, oldTableExt); TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName, "default"); Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality()); getInstance(getTestConfig()).removeTableExt(tableName, "default"); }
public LookupTable<String> initLookupTable() throws Exception { TableMetadataManager metaMgr = TableMetadataManager.getInstance(config); String tableName = "EDW.TEST_CAL_DT"; String[] pkCols = new String[] { "CAL_DT" }; String snapshotResPath = "/table_snapshot/TEST_CAL_DT.csv/4af48c94-86de-4e22-a4fd-c49b06cbaa4f.snapshot"; SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = metaMgr.getTableDesc(tableName, "default"); LookupTable<String> lt = new LookupStringTable(tableDesc, pkCols, snapshot); System.out.println(lt); return lt; }
@Test public void testListAllTables() throws Exception { List<TableDesc> tables = getInstance(getTestConfig()).listAllTables(null); Assert.assertNotNull(tables); Assert.assertTrue(tables.size() > 0); }
@Before public void setup() throws Exception { this.createTestMetadata(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); tableDesc = metadataManager.getTableDesc("TEST_COUNTRY", "default"); }
@Before public void setup() throws Exception { this.createTestMetadata(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); tableDesc = metadataManager.getTableDesc("TEST_COUNTRY", "default"); }
@Before public void setup() throws Exception { createTestMetadata(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); this.random = new Random(); tableDesc = metadataManager.getTableDesc("TEST_COUNTRY", "default"); sourceRowNum = 10000; genTestData(); lookupTable = new RocksDBLookupTable(tableDesc, new String[] { "COUNTRY" }, "lookup_cache/TEST_COUNTRY"); }
@Before public void setup() throws Exception { createTestMetadata(); TableMetadataManager metadataManager = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); kylinConfig = getTestConfig(); tableDesc = metadataManager.getTableDesc(TABLE_COUNTRY, "default"); cleanCache(); LookupProviderFactory.registerLookupProvider(MOCK_EXT_LOOKUP, MockedLookupProvider.class.getName()); }
private void addTables(DataModelDesc modelDesc) throws IOException { if (modelDesc != null) { //fixme should get all tbls in prj not only in cubes when back up by prj. for (TableRef tableRef : modelDesc.getAllTables()) { addRequired(tableRef.getTableDesc().getResourcePath()); addOptional(TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()) // .getTableExt(tableRef.getTableDesc()) // .getResourcePath()); // } addRequired(DataModelDesc.concatResourcePath(modelDesc.getName())); } }
@Test public void testFindTableByName() throws Exception { TableDesc table = getInstance(getTestConfig()).getTableDesc("EDW.TEST_CAL_DT", "default"); Assert.assertNotNull(table); Assert.assertEquals("EDW.TEST_CAL_DT", table.getIdentity()); }