public static CubeDesc getCopyOf(CubeDesc cubeDesc) { CubeDesc newCubeDesc = new CubeDesc(); newCubeDesc.setName(cubeDesc.getName()); newCubeDesc.setDraft(cubeDesc.isDraft()); newCubeDesc.setModelName(cubeDesc.getModelName()); newCubeDesc.setDescription(cubeDesc.getDescription()); newCubeDesc.setNullStrings(cubeDesc.getNullStrings()); newCubeDesc.setDimensions(cubeDesc.getDimensions()); newCubeDesc.setMeasures(cubeDesc.getMeasures()); newCubeDesc.setDictionaries(cubeDesc.getDictionaries()); newCubeDesc.setRowkey(cubeDesc.getRowkey()); newCubeDesc.setHbaseMapping(cubeDesc.getHbaseMapping()); newCubeDesc.setSignature(cubeDesc.getSignature()); newCubeDesc.setNotifyList(cubeDesc.getNotifyList()); newCubeDesc.setStatusNeedNotify(cubeDesc.getStatusNeedNotify()); newCubeDesc.setAutoMergeTimeRanges(cubeDesc.getAutoMergeTimeRanges()); newCubeDesc.setPartitionDateStart(cubeDesc.getPartitionDateStart()); newCubeDesc.setPartitionDateEnd(cubeDesc.getPartitionDateEnd()); newCubeDesc.setVolatileRange(cubeDesc.getVolatileRange()); newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange()); newCubeDesc.setEngineType(cubeDesc.getEngineType()); newCubeDesc.setStorageType(cubeDesc.getStorageType()); newCubeDesc.setAggregationGroups(cubeDesc.getAggregationGroups()); newCubeDesc.setOverrideKylinProps(cubeDesc.getOverrideKylinProps()); newCubeDesc.setConfig((KylinConfigExt) cubeDesc.getConfig()); newCubeDesc.setPartitionOffsetStart(cubeDesc.getPartitionOffsetStart()); newCubeDesc.setVersion(cubeDesc.getVersion()); newCubeDesc.setParentForward(cubeDesc.getParentForward()); newCubeDesc.setSnapshotTableDescList(cubeDesc.getSnapshotTableDescList()); newCubeDesc.setMandatoryDimensionSetList(cubeDesc.getMandatoryDimensionSetList());
@Override public DataModelDesc getDataModel() { return cubeDesc.getModel(); }
if (cubeDesc.getUuid() == null || cubeDesc.getName() == null) throw new IllegalArgumentException(); if (cubeDescMap.containsKey(cubeDesc.getName())) throw new IllegalArgumentException("CubeDesc '" + cubeDesc.getName() + "' already exists"); if (cubeDesc.isDraft()) throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, cubeDesc.getName())); cubeDesc.init(config); } catch (Exception e) { logger.warn("Broken cube desc " + cubeDesc, e); cubeDesc.addError(e.toString()); if (cubeDesc.isBroken()) { return cubeDesc; cubeDesc.setSignature(cubeDesc.calculateSignature());
private void updateCubeDesc(CubeDesc cubeDesc) { try { String calculatedSign = cubeDesc.calculateSignature(); if (cubeDesc.getSignature() == null || (!cubeDesc.getSignature().equals(calculatedSign))) { cubeDesc.setSignature(calculatedSign); store.checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, CubeDesc.newSerializerForLowLevelAccess()); updatedResources.add(cubeDesc.getResourcePath()); } } catch (Exception e) { logger.error("error", e); errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); } }
@Override protected CubeDesc initEntityAfterReload(CubeDesc cubeDesc, String resourceName) { if (cubeDesc.isDraft()) throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, cubeDesc.getName())); try { cubeDesc.init(config); } catch (Exception e) { logger.warn(String.format(Locale.ROOT, BROKEN_CUBE_MSG, cubeDesc.resourceName()), e); cubeDesc.addError(e.toString()); } return cubeDesc; } };
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName()); if (StringUtils.isEmpty(dataModelDesc.getPartitionDesc().getPartitionDateColumn())) { logger.error("No incremental cube, no need to extend."); String newCubeDescName = renameCube(cubeDesc.getName()); String newCubeInstanceName = renameCube(cubeInstance.getName()); while (cubeDescManager.getCubeDesc(newCubeDescName) != null) CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc); newCubeDesc.setName(newCubeDescName); newCubeDesc.updateRandomUuid(); newCubeDesc.init(kylinConfig); newCubeDesc.setPartitionDateEnd(partitionDate); newCubeDesc.calculateSignature(); cubeDescManager.createCubeDesc(newCubeDesc); logger.info("CubeDesc was saved at: " + newCubeDesc.getResourcePath()); cubeDesc.setPartitionDateStart(partitionDate); cubeDesc.setEngineType(IEngineAware.ID_MR_V2); cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE); cubeDesc.calculateSignature(); cubeDescManager.updateCubeDesc(cubeDesc); logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
assertEquals(lc.getDimensions().size(), ic.getDimensions().size()); for (int i = 0, n = lc.getDimensions().size(); i < n; i++) { DimensionDesc ld = lc.getDimensions().get(i); DimensionDesc id = ic.getDimensions().get(i); assertEquals(ld.getTable(), id.getTable()); assertEquals(ld.getColumn(), id.getColumn()); List<MeasureDesc> lcMeasures = dropPercentile(lc.getMeasures()); List<MeasureDesc> icMeasures = ic.getMeasures(); assertEquals(lc.getAggregationGroups().size(), ic.getAggregationGroups().size()); for (int i = 0, n = lc.getAggregationGroups().size(); i < n; i++) { AggregationGroup lag = lc.getAggregationGroups().get(i); AggregationGroup iag = ic.getAggregationGroups().get(i); assertArrayEquals(lag.getIncludes(), iag.getIncludes()); assertArrayEquals(lag.getSelectRule().mandatoryDims, iag.getSelectRule().mandatoryDims); assertEquals(lc.listAllColumnDescs().size(), ic.listAllColumnDescs().size()); assertEquals(lc.listAllColumns().size(), ic.listAllColumns().size()); assertTrue(lc.listAllColumns().contains(lc.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID"))); assertTrue(ic.listAllColumns().contains(ic.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID")));
public KVGTRecordWriter(CubeDesc cubeDesc, CubeSegment cubeSegment) { this.cubeDesc = cubeDesc; this.cubeSegment = cubeSegment; this.measureCount = cubeDesc.getMeasures().size(); }
if (cubeDesc.getUuid() == null || cubeDesc.getName() == null) throw new IllegalArgumentException(); if (cubeDescMap.containsKey(cubeDesc.getName())) throw new IllegalArgumentException("CubeDesc '" + cubeDesc.getName() + "' already exists"); cubeDesc.init(config, getMetadataManager().getAllTablesMap()); } catch (IllegalStateException e) { cubeDesc.addError(e.getMessage(), true); if (!cubeDesc.getError().isEmpty()) { return cubeDesc; cubeDesc.setSignature(cubeDesc.calculateSignature()); String path = cubeDesc.getResourcePath(); getStore().putResource(path, cubeDesc, CUBE_DESC_SERIALIZER); cubeDescMap.put(cubeDesc.getName(), cubeDesc);
public Set<TblColRef> getShardByColumns() { return getRowkey().getShardByColumns(); }
private void checkCubeDesc(String descName) { CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName); Assert.assertNotNull(cubedesc1); DataModelDesc model = cubedesc1.getModel(); Assert.assertNotNull(model); Assert.assertTrue(model.getLookups().length > 0); List<DimensionDesc> dims = cubedesc1.getDimensions(); Assert.assertTrue(dims.size() > 0); for (DimensionDesc dim : dims) { Assert.assertTrue(dim.getColumn().length > 0); } Assert.assertTrue(cubedesc1.getMeasures().size() > 0); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName); Assert.assertTrue(cubes.size() > 0); }
private void validateColumnFamily(CubeDesc cubeDesc) { Set<String> columnFamilyMetricsSet = Sets.newHashSet(); for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) { for (String columnName : hBaseColumnDesc.getMeasureRefs()) { columnFamilyMetricsSet.add(columnName); } } } for (MeasureDesc measureDesc : cubeDesc.getMeasures()) { if (!columnFamilyMetricsSet.contains(measureDesc.getName())) { throw new BadRequestException("column family lack measure:" + measureDesc.getName()); } } if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) { throw new BadRequestException( "the number of input measure and the number of measure defined in cubedesc are not consistent"); } }
if (this.getConfig().isIgnoreCubeSignatureInconsistency()) { logger.info("Skip checking cube signature"); return true; KylinVersion cubeVersion = new KylinVersion(getVersion()); KylinVersion kylinVersion = KylinVersion.getCurrentVersion(); if (!kylinVersion.isCompatibleWith(cubeVersion)) { logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", getName(), cubeVersion, kylinVersion); return true; logger.info( "checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), cubeVersion); return true; if (StringUtils.isBlank(getSignature())) { return true; String calculated = calculateSignature(); String saved = getSignature(); return calculated.equals(saved);
private void upgradeCubeDesc() { logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT); for (String path : paths) { try { CubeDescUpgrader upgrade = new CubeDescUpgrader(path); CubeDesc ndesc = upgrade.upgrade(); ndesc.setSignature(ndesc.calculateSignature()); getStore().putResource(ndesc.getModel().getResourcePath(), ndesc.getModel(), MetadataManager.MODELDESC_SERIALIZER); getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER); updatedResources.add(ndesc.getResourcePath()); } catch (IOException e) { e.printStackTrace(); errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage()); } } }
protected void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException { CubeDesc cubeDesc = cube.getDescriptor(); String prj = cubeDesc.getProject(); metaResource.add(cube.getResourcePath()); metaResource.add(cubeDesc.getResourcePath()); metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName())); Set<TableRef> tblRefs = cubeDesc.getModel().getAllTables(); metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_RESOURCE_ROOT)); metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_EXD_RESOURCE_ROOT)); if (doMigrateSegment) { for (CubeSegment segment : cube.getSegments()) { metaResource.add(segment.getStatisticsResourcePath()); dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); } } if (doAclCopy) { metaResource.add(ACL_PREFIX + cube.getUuid()); metaResource.add(ACL_PREFIX + cube.getModel().getUuid()); } }
@Test public void testBadInit15() throws Exception { thrown.expect(IllegalStateException.class); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates"); HBaseColumnDesc colDesc = new HBaseColumnDesc(); colDesc.setQualifier("M"); colDesc.setMeasureRefs( new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" }); cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; cubeDesc.initMeasureReferenceToColumnFamily(); }
if ((cubeName == null || (exactMatch && cubeName.toLowerCase(Locale.ROOT).equals(c.getName().toLowerCase(Locale.ROOT))) || (!exactMatch && c.getName().toLowerCase(Locale.ROOT).contains(cubeName.toLowerCase(Locale.ROOT)))) && (modelName == null || modelName.toLowerCase(Locale.ROOT) .equals(c.getModelName().toLowerCase(Locale.ROOT)))) { if (c.getMeasures() != null) { for (MeasureDesc m : c.getMeasures()) { FunctionDesc f = m.getFunction(); if (f.getExpression().equals(PercentileMeasureType.FUNC_PERCENTILE)) {
@Test public void testBasics() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready"); CubeDesc desc = cube.getDescriptor(); //System.out.println(JsonUtil.writeValueAsIndentString(desc)); String signature = desc.calculateSignature(); desc.getModel().getPartitionDesc().setPartitionDateColumn("test_column"); assertTrue(signature.equals(desc.calculateSignature())); }
private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException { CubeDesc cubeDesc = cube.getDescriptor(); metaResource.add(cube.getResourcePath()); metaResource.add(cubeDesc.getResourcePath()); metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName())); for (TableRef table : cubeDesc.getModel().getAllTables()) { metaResource.add(table.getTableDesc().getResourcePath()); } for (CubeSegment segment : cube.getSegments()) { metaResource.add(segment.getStatisticsResourcePath()); dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); } }
public static Set<String> collectCubeMetadata(CubeInstance cube) { // cube, model_desc, cube_desc, table Set<String> dumpList = new LinkedHashSet<>(); dumpList.add(cube.getResourcePath()); dumpList.add(cube.getDescriptor().getModel().getResourcePath()); dumpList.add(cube.getDescriptor().getResourcePath()); dumpList.add(cube.getProjectInstance().getResourcePath()); for (TableRef tableRef : cube.getDescriptor().getModel().getAllTables()) { TableDesc table = tableRef.getTableDesc(); dumpList.add(table.getResourcePath()); dumpList.addAll(SourceManager.getMRDependentResources(table)); } return dumpList; }