public static ISource getDefaultSource() { final KylinConfig sysConfig = KylinConfig.getInstanceFromEnv(); return getSource(new ISourceAware() { @Override public int getSourceType() { return sysConfig.getDefaultSource(); } @Override public KylinConfig getConfig() { return sysConfig; } }); }
public static IReadableTable createReadableTable(TableDesc table, String uuid) { return getSource(table).createReadableTable(table, uuid); }
public static <T> T createEngineAdapter(ISourceAware table, Class<T> engineInterface) { return getSource(table).adaptToBuildEngine(engineInterface); }
public static List<String> getMRDependentResources(TableDesc table) { return getSource(table).getSourceMetadataExplorer().getRelatedKylinResources(table); } }
public List<Pair<TableDesc, TableExtDesc>> extractHiveTableMeta(String[] tables, String project) throws Exception { // de-dup SetMultimap<String, String> db2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { String[] parts = HadoopUtil.parseHiveTableName(fullTableName); db2tables.put(parts[0], parts[1]); } // load all tables first List<Pair<TableDesc, TableExtDesc>> allMeta = Lists.newArrayList(); ProjectInstance projectInstance = getProjectManager().getProject(project); ISourceMetadataExplorer explr = SourceManager.getSource(projectInstance).getSourceMetadataExplorer(); for (Map.Entry<String, String> entry : db2tables.entries()) { Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project); TableDesc tableDesc = pair.getFirst(); Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase(Locale.ROOT))); Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase(Locale.ROOT))); Preconditions.checkState(tableDesc.getIdentity() .equals(entry.getKey().toUpperCase(Locale.ROOT) + "." + entry.getValue().toUpperCase(Locale.ROOT))); TableExtDesc extDesc = pair.getSecond(); Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity())); allMeta.add(pair); } return allMeta; }
@BeforeClass public static void setUp() throws Exception { TestBase.setUp(); explorer = SourceManager.getSource(new JdbcSourceTest.JdbcSourceAware()).getSourceMetadataExplorer(); }
ISampleDataDeployer sampleDataDeployer = SourceManager.getSource(model.getRootFactTable().getTableDesc()) .getSampleDataDeployer();
try { if (buildType == CubeBuildTypeEnum.BUILD) { ISource source = SourceManager.getSource(cube); SourcePartition src = new SourcePartition(tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd);
@Test public void testGenSqoopCmd_WithLookupShardBy() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\"")); source.close(); }
@Test public void testGenSqoopCmd_Partition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"")); source.close(); }
@Test public void testGenSqoopCmd_NoPartition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(0L, Long.MAX_VALUE)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue( cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\"")); source.close(); }
@Test public void testBasics() throws Exception { TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); TableDesc tblDesc = tblManager.getTableDesc("test_kylin_fact", "default"); IReadableTable table = SourceManager.getSource(new JdbcSourceTest.JdbcSourceAware()) .createReadableTable(tblDesc, null); // test TableReader try (IReadableTable.TableReader reader = table.getReader()) { Assert.assertTrue(reader instanceof JdbcTableReader); Assert.assertTrue(table instanceof JdbcTable); Assert.assertTrue(reader.next()); String[] row = reader.getRow(); Assert.assertNotNull(row); Assert.assertEquals(tblDesc.getColumnCount(), row.length); } // test basics Assert.assertTrue(table.exists()); IReadableTable.TableSignature sign = table.getSignature(); Assert.assertNotNull(sign); Assert.assertEquals(String.format(Locale.ROOT, "%s.%s", tblDesc.getDatabase(), tblDesc.getName()), sign.getPath()); Assert.assertTrue(sign.getLastModifiedTime() > 0); }
@Test public void testBasics() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); ISourceMetadataExplorer explorer = source.getSourceMetadataExplorer(); ISampleDataDeployer deployer = source.getSampleDataDeployer(); Assert.assertTrue(source instanceof JdbcSource); Assert.assertTrue(explorer instanceof JdbcExplorer); Assert.assertTrue(deployer instanceof JdbcExplorer); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); Class adaptTo = Object.class; expectedCannotAdaptEx.expect(RuntimeException.class); expectedCannotAdaptEx.expectMessage("Cannot adapt to " + adaptTo); source.adaptToBuildEngine(adaptTo); TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); IReadableTable table = source.createReadableTable(tblManager.getTableDesc("test_kylin_fact", "default"), null); Assert.assertTrue(table instanceof JdbcTable); source.close(); } }
public static ISource getDefaultSource() { final KylinConfig sysConfig = KylinConfig.getInstanceFromEnv(); return getSource(new ISourceAware() { @Override public int getSourceType() { return sysConfig.getDefaultSource(); } @Override public KylinConfig getConfig() { return sysConfig; } }); }
public static IReadableTable createReadableTable(TableDesc table, String uuid) { return getSource(table).createReadableTable(table, uuid); }
public static <T> T createEngineAdapter(ISourceAware table, Class<T> engineInterface) { return getSource(table).adaptToBuildEngine(engineInterface); }
public static List<String> getMRDependentResources(TableDesc table) { return getSource(table).getSourceMetadataExplorer().getRelatedKylinResources(table); } }