@Override public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) { return new CubeJoinedFlatTableDesc(cubeDesc); }
@Override public List<TblColRef> getFactColumns() { final List<TblColRef> factColumns = Lists.newArrayList(); for (TblColRef col : this.getAllColumns()) { if (col.getTableRef().equals(getDataModel().getRootFactTable())) { // only fetch the columns from fact table factColumns.add(col); } } return factColumns; }
private CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment /* can be null */, boolean includingDerived) { this.cubeDesc = cubeDesc; this.cubeSegment = cubeSegment; this.tableName = makeTableName(cubeDesc, cubeSegment); this.includingDerived = includingDerived; initParseCubeDesc(); }
protected void initParseCubeDesc() { if (this.includingDerived) { for (TblColRef col : cubeDesc.listDimensionColumnsIncludingDerived()) { initAddColumn(col); initAddColumn(col); for (int j = 0; j < colRefs.size(); j++) { TblColRef c = colRefs.get(j); initAddColumn(c); for (DictionaryDesc dictDesc : cubeDesc.getDictionaries()) { TblColRef c = dictDesc.getColumnRef(); initAddColumn(c); if (dictDesc.getResuseColumnRef() != null) { c = dictDesc.getResuseColumnRef(); initAddColumn(c);
protected void initParseCubeDesc() { if (this.includingDerived) { for (TblColRef col : cubeDesc.listDimensionColumnsIncludingDerived()) { initAddColumn(col); initAddColumn(col); for (int j = 0; j < colRefs.size(); j++) { TblColRef c = colRefs.get(j); initAddColumn(c); for (DictionaryDesc dictDesc : cubeDesc.getDictionaries()) { TblColRef c = dictDesc.getColumnRef(); initAddColumn(c); if (dictDesc.getResuseColumnRef() != null) { c = dictDesc.getResuseColumnRef(); initAddColumn(c);
@Override public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) { return new CubeJoinedFlatTableDesc(newSegment); }
@Override public List<TblColRef> getFactColumns() { final List<TblColRef> factColumns = Lists.newArrayList(); for (TblColRef col : this.getAllColumns()) { if (col.getTableRef().equals(getDataModel().getRootFactTable())) { // only fetch the columns from fact table factColumns.add(col); } } return factColumns; }
private CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment /* can be null */, boolean includingDerived) { this.cubeDesc = cubeDesc; this.cubeSegment = cubeSegment; this.tableName = makeTableName(cubeDesc, cubeSegment); this.includingDerived = includingDerived; initParseCubeDesc(); }
@Override public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) { return new CubeJoinedFlatTableDesc(cubeDesc); }
@Override public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) { return new CubeJoinedFlatTableDesc(newSegment); }
@Override public void configureJob(Job job) { job.setInputFormatClass(SequenceFileInputFormat.class); String jobId = job.getConfiguration().get(BatchConstants.ARG_CUBING_JOB_ID); IJoinedFlatTableDesc flatHiveTableDesc = new CubeJoinedFlatTableDesc(cubeSegment); String inputPath = JoinedFlatTable.getTableDir(flatHiveTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobId)); try { FileInputFormat.addInputPath(job, new Path(inputPath)); } catch (IOException e) { throw new IllegalStateException(e); } }
/** * Get SQL of a Cube * * @param cubeName Cube Name * @return * @throws IOException */ @RequestMapping(value = "/{cubeName}/sql", method = { RequestMethod.GET }, produces = { "application/json" }) @ResponseBody public GeneralResponse getSql(@PathVariable String cubeName) { checkCubeExists(cubeName); CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); IJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), true); String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc); GeneralResponse response = new GeneralResponse(); response.setProperty("sql", sql); return response; }
/** * Get SQL of a Cube segment * * @param cubeName Cube Name * @param segmentName Segment Name * @return * @throws IOException */ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = { RequestMethod.GET }, produces = { "application/json" }) @ResponseBody public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) { checkCubeExists(cubeName); CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); CubeSegment segment = cube.getSegment(segmentName, null); if (segment == null) { throw new NotFoundException("Cannot find segment " + segmentName); } IJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(segment, true); String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc); GeneralResponse response = new GeneralResponse(); response.setProperty("sql", sql); return response; }
@Override protected void doSetup(Context context) throws IOException { Configuration conf = context.getConfiguration(); bindCurrentConfiguration(conf); config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); final CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); this.cubeSegment = cube.getSegmentById(conf.get(BatchConstants.CFG_CUBE_SEGMENT_ID)); this.delimiter = BatchConstants.SEQUENCE_FILE_DEFAULT_DELIMITER; //sequence file default delimiter logger.info("Use delimiter: " + delimiter); final KafkaConfigManager kafkaConfigManager = KafkaConfigManager.getInstance(config); final KafkaConfig kafkaConfig = kafkaConfigManager.getKafkaConfig(cubeSegment.getCubeInstance().getRootFactTable()); final IJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cubeSegment); final List<TblColRef> allColumns = flatTableDesc.getFactColumns(); try { streamingParser = StreamingParser.getStreamingParser(kafkaConfig.getParserName(), kafkaConfig.getAllParserProperties(), allColumns); } catch (ReflectiveOperationException e) { throw new IllegalArgumentException(e); } }
@Test public void testGenSqoopCmd_WithLookupShardBy() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\"")); source.close(); }
@Test public void testGenSqoopCmd_Partition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"")); source.close(); }
@Test public void testGenSqoopCmd_NoPartition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(0L, Long.MAX_VALUE)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue( cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\"")); source.close(); }
@Override public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) { return new CubeJoinedFlatTableDesc(cubeDesc); }
@Override public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) { return new CubeJoinedFlatTableDesc(newSegment); }
@Override public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) { return new CubeJoinedFlatTableDesc(newSegment); }