public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectJob()); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkTool.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkTool.getCubeDescOverrideProperties());
public static TableDesc generateKylinTable(KylinConfig kylinConfig, SinkTool sinkTool, String subject, List<Pair<String, String>> columns) { TableDesc kylinTable = new TableDesc(); Pair<String, String> tableNameSplits = ActiveReservoirReporter .getTableNameSplits(sinkTool.getTableNameForMetrics(subject)); kylinTable.setUuid(RandomUtil.randomUUID().toString()); kylinTable.setDatabase(tableNameSplits.getFirst()); kylinTable.setName(tableNameSplits.getSecond()); kylinTable.setTableType(null); kylinTable.setLastModified(0L); kylinTable.setSourceType(sinkTool.getSourceType()); ColumnDesc[] columnDescs = new ColumnDesc[columns.size()]; for (int i = 0; i < columns.size(); i++) { columnDescs[i] = new ColumnDesc(); Pair<String, String> entry = columns.get(i); columnDescs[i].setId(Integer.toString(i + 1)); columnDescs[i].setName(entry.getFirst()); columnDescs[i].setDatatype(entry.getSecond()); } kylinTable.setColumns(columnDescs); kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT); return kylinTable; } }
public static CubeInstance generateKylinCubeInstanceForMetricsQueryRPC(String owner, KylinConfig config, SinkTool sinkTool) { return generateKylinCubeInstance(owner, sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall())); }
@Test public void testReadSinkToolsJson() throws Exception { Set<SinkTool> sinkToolSet = readSinkToolsJson("src/main/resources/SCSinkTools.json"); for (SinkTool entry : sinkToolSet) { Map<String, String> props = entry.getCubeDescOverrideProperties(); assertEquals(props.get("kylin.cube.algorithm"), "INMEM"); } }
public static CubeInstance generateKylinCubeInstanceForMetricsQueryCube(String owner, KylinConfig config, SinkTool sinkTool) { return generateKylinCubeInstance(owner, sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube())); }
@Test public void testWriteSinkToolsJson() throws Exception { Map<String, String> cubeDescOverrideProperties = Maps.newHashMap(); cubeDescOverrideProperties.put("kylin.cube.algorithm", "INMEM"); HiveSinkTool hiveSinkTool = new HiveSinkTool(); hiveSinkTool.setCubeDescOverrideProperties(cubeDescOverrideProperties); String outputPath = "src/test/resources/SCSinkTools.json"; try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(outputPath))) { ObjectMapper mapper = new ObjectMapper(); mapper.enableDefaultTyping(); mapper.writeValue(os, Sets.newHashSet(hiveSinkTool)); } Set<SinkTool> sinkToolSet = readSinkToolsJson(outputPath); for (SinkTool entry : sinkToolSet) { Map<String, String> props = entry.getCubeDescOverrideProperties(); for (String key : cubeDescOverrideProperties.keySet()) { assertEquals(props.get(key), cubeDescOverrideProperties.get(key)); } } }
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall()); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkTool.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkTool.getCubeDescOverrideProperties());
public static CubeInstance generateKylinCubeInstanceForMetricsJobException(String owner, KylinConfig config, SinkTool sinkTool) { return generateKylinCubeInstance(owner, sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectJobException())); }
public static TableDesc generateKylinTable(KylinConfig kylinConfig, SinkTool sinkTool, String subject, List<Pair<String, String>> columns) { TableDesc kylinTable = new TableDesc(); Pair<String, String> tableNameSplits = ActiveReservoirReporter .getTableNameSplits(sinkTool.getTableNameForMetrics(subject)); kylinTable.setUuid(RandomUtil.randomUUID().toString()); kylinTable.setDatabase(tableNameSplits.getFirst()); kylinTable.setName(tableNameSplits.getSecond()); kylinTable.setTableType(null); kylinTable.setLastModified(0L); kylinTable.setSourceType(sinkTool.getSourceType()); ColumnDesc[] columnDescs = new ColumnDesc[columns.size()]; for (int i = 0; i < columns.size(); i++) { columnDescs[i] = new ColumnDesc(); Pair<String, String> entry = columns.get(i); columnDescs[i].setId(Integer.toString(i + 1)); columnDescs[i].setName(entry.getFirst()); columnDescs[i].setDatatype(entry.getSecond()); } kylinTable.setColumns(columnDescs); kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT); return kylinTable; } }
public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectJobException()); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkTool.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkTool.getCubeDescOverrideProperties());
public static CubeInstance generateKylinCubeInstanceForMetricsJob(String owner, KylinConfig config, SinkTool sinkTool) { return generateKylinCubeInstance(owner, sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectJob())); }
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube()); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkTool.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkTool.getCubeDescOverrideProperties());
public static CubeInstance generateKylinCubeInstanceForMetricsQuery(String owner, KylinConfig config, SinkTool sinkTool) { return generateKylinCubeInstance(owner, sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectQuery())); }
public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectQuery()); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkTool.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkTool.getCubeDescOverrideProperties());
public static DataModelDesc generateKylinModelForMetricsQueryCube(String owner, KylinConfig kylinConfig, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(kylinConfig.getKylinMetricsSubjectQueryCube()); return generateKylinModel(owner, tableName, getDimensionsForMetricsQueryCube(), getMeasuresForMetricsQueryCube(), getPartitionDesc(tableName)); }
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectJob()); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkTool.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkTool.getCubeDescOverrideProperties());
public static DataModelDesc generateKylinModelForMetricsJob(String owner, KylinConfig kylinConfig, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(kylinConfig.getKylinMetricsSubjectJob()); return generateKylinModel(owner, tableName, getDimensionsForMetricsJob(), getMeasuresForMetricsJob(), getPartitionDesc(tableName)); }
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall()); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkTool.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkTool.getCubeDescOverrideProperties());
public static DataModelDesc generateKylinModelForMetricsJobException(String owner, KylinConfig kylinConfig, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(kylinConfig.getKylinMetricsSubjectJobException()); return generateKylinModel(owner, tableName, getDimensionsForMetricsJobException(), getMeasuresForMetricsJobException(), getPartitionDesc(tableName)); }
public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, SinkTool sinkTool) { String tableName = sinkTool.getTableNameForMetrics(config.getKylinMetricsSubjectJobException()); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkTool.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkTool.getCubeDescOverrideProperties());