public static List<String> getMRDependentResources(TableDesc table) { return getSource(table).getSourceMetadataExplorer().getRelatedKylinResources(table); } }
public static <T> T createEngineAdapter(ISourceAware table, Class<T> engineInterface) { return getSource(table).adaptToBuildEngine(engineInterface); }
@Override public void onRemoval(RemovalNotification<String, ISource> entry) { ISource s = entry.getValue(); if (s != null) { try { s.close(); } catch (Throwable e) { logger.error("Failed to close ISource: {}", s.getClass().getName(), e); } } } }).build();
@Test public void testBasics() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); ISourceMetadataExplorer explorer = source.getSourceMetadataExplorer(); ISampleDataDeployer deployer = source.getSampleDataDeployer(); Assert.assertTrue(source instanceof JdbcSource); Assert.assertTrue(explorer instanceof JdbcExplorer); Assert.assertTrue(deployer instanceof JdbcExplorer); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); Class adaptTo = Object.class; expectedCannotAdaptEx.expect(RuntimeException.class); expectedCannotAdaptEx.expectMessage("Cannot adapt to " + adaptTo); source.adaptToBuildEngine(adaptTo); TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); IReadableTable table = source.createReadableTable(tblManager.getTableDesc("test_kylin_fact", "default"), null); Assert.assertTrue(table instanceof JdbcTable); source.close(); } }
@Test public void testGenSqoopCmd_Partition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"")); source.close(); }
public static IReadableTable createReadableTable(TableDesc table, String uuid) { return getSource(table).createReadableTable(table, uuid); }
source.unloadTable(tableName, project); return rtn;
SourcePartition src = new SourcePartition(tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd); src = source.enrichSourcePartitionBeforeBuild(cube, src); newSeg = getCubeManager().appendSegment(cube, src); job = EngineFactory.createBatchCubingJob(newSeg, submitter);
@Test public void testGenSqoopCmd_WithLookupShardBy() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\"")); source.close(); }
@Test public void testBasics() throws Exception { TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); TableDesc tblDesc = tblManager.getTableDesc("test_kylin_fact", "default"); IReadableTable table = SourceManager.getSource(new JdbcSourceTest.JdbcSourceAware()) .createReadableTable(tblDesc, null); // test TableReader try (IReadableTable.TableReader reader = table.getReader()) { Assert.assertTrue(reader instanceof JdbcTableReader); Assert.assertTrue(table instanceof JdbcTable); Assert.assertTrue(reader.next()); String[] row = reader.getRow(); Assert.assertNotNull(row); Assert.assertEquals(tblDesc.getColumnCount(), row.length); } // test basics Assert.assertTrue(table.exists()); IReadableTable.TableSignature sign = table.getSignature(); Assert.assertNotNull(sign); Assert.assertEquals(String.format(Locale.ROOT, "%s.%s", tblDesc.getDatabase(), tblDesc.getName()), sign.getPath()); Assert.assertTrue(sign.getLastModifiedTime() > 0); }
public List<Pair<TableDesc, TableExtDesc>> extractHiveTableMeta(String[] tables, String project) throws Exception { // de-dup SetMultimap<String, String> db2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { String[] parts = HadoopUtil.parseHiveTableName(fullTableName); db2tables.put(parts[0], parts[1]); } // load all tables first List<Pair<TableDesc, TableExtDesc>> allMeta = Lists.newArrayList(); ProjectInstance projectInstance = getProjectManager().getProject(project); ISourceMetadataExplorer explr = SourceManager.getSource(projectInstance).getSourceMetadataExplorer(); for (Map.Entry<String, String> entry : db2tables.entries()) { Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project); TableDesc tableDesc = pair.getFirst(); Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase(Locale.ROOT))); Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase(Locale.ROOT))); Preconditions.checkState(tableDesc.getIdentity() .equals(entry.getKey().toUpperCase(Locale.ROOT) + "." + entry.getValue().toUpperCase(Locale.ROOT))); TableExtDesc extDesc = pair.getSecond(); Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity())); allMeta.add(pair); } return allMeta; }
@Test public void testGenSqoopCmd_NoPartition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(0L, Long.MAX_VALUE)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue( cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\"")); source.close(); }
public static IReadableTable createReadableTable(TableDesc table, String uuid) { return getSource(table).createReadableTable(table, uuid); }
@Override public void onRemoval(RemovalNotification<String, ISource> entry) { ISource s = entry.getValue(); if (s != null) { try { s.close(); } catch (Throwable e) { logger.error("Failed to close ISource: {}", s.getClass().getName(), e); } } } }).build();
public static <T> T createEngineAdapter(ISourceAware table, Class<T> engineInterface) { return getSource(table).adaptToBuildEngine(engineInterface); }
protected List<String> getHiveTables() throws Exception { ISourceMetadataExplorer explr = SourceManager.getDefaultSource().getSourceMetadataExplorer(); return explr.listTables(config.getHiveDatabaseForIntermediateTable()); }
/** * * @param project * @param database * @return * @throws Exception */ public List<String> getSourceTableNames(String project, String database) throws Exception { ISourceMetadataExplorer explr = SourceManager.getInstance(getConfig()).getProjectSource(project) .getSourceMetadataExplorer(); return explr.listTables(database); }
/** * * @param project * @return * @throws Exception */ public List<String> getSourceDbNames(String project) throws Exception { ISourceMetadataExplorer explr = SourceManager.getInstance(getConfig()).getProjectSource(project) .getSourceMetadataExplorer(); return explr.listDatabases(); }
@BeforeClass public static void setUp() throws Exception { TestBase.setUp(); explorer = SourceManager.getSource(new JdbcSourceTest.JdbcSourceAware()).getSourceMetadataExplorer(); }