@Override public DataModelDesc getDataModel() { return cubeDesc.getModel(); }
private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException { CubeDesc cubeDesc = cube.getDescriptor(); metaResource.add(cube.getResourcePath()); metaResource.add(cubeDesc.getResourcePath()); metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName())); for (TableRef table : cubeDesc.getModel().getAllTables()) { metaResource.add(table.getTableDesc().getResourcePath()); } for (CubeSegment segment : cube.getSegments()) { metaResource.add(segment.getStatisticsResourcePath()); dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); } }
@Override public DataModelDesc getDataModel() { return cubeDesc.getModel(); }
@Override public DataModelDesc getModel() { CubeDesc cubeDesc = this.getDescriptor(); if (cubeDesc != null) { return cubeDesc.getModel(); } else { return null; } }
@Override public DataModelDesc getModel() { return this.getCubeDesc().getModel(); }
private void normalizeColumnNames(String[] names) { if (names == null) return; for (int i = 0; i < names.length; i++) { TblColRef col = cubeDesc.getModel().findColumn(names[i]); names[i] = col.getIdentity(); } // check no dup Set<String> set = new HashSet<>(Arrays.asList(names)); if (set.size() < names.length) throw new IllegalStateException( "Columns in aggrgroup must not contain duplication: " + Arrays.asList(names)); }
void init(CubeDesc cubeDesc) { DataModelDesc model = cubeDesc.getModel(); column = column.toUpperCase(Locale.ROOT); colRef = model.findColumn(column); if (reuseColumn != null) { reuseColumn = reuseColumn.toUpperCase(Locale.ROOT); reuseColRef = model.findColumn(reuseColumn); } }
private void buildPartialCubeFullMask(RowKeyDesc rowKeyDesc) { Preconditions.checkState(this.includes != null); Preconditions.checkState(this.includes.length != 0); partialCubeFullMask = 0L; for (String dim : this.includes) { TblColRef hColumn = cubeDesc.getModel().findColumn(dim); Integer index = rowKeyDesc.getColumnBitIndex(hColumn); long bit = 1L << index; partialCubeFullMask |= bit; } }
public CubeJoinedFlatTableEnrich(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) { // != works due to object cache if (cubeDesc.getModel() != flatDesc.getDataModel()) throw new IllegalArgumentException(); this.cubeDesc = cubeDesc; this.flatDesc = flatDesc; parseCubeDesc(); }
/** * @param context * @param cube * @param value */ private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) { DataModelDesc model = cube.getModel(); try { model.findColumn(value); } catch (IllegalArgumentException e) { context.addResult(ResultLevel.ERROR, e.getMessage()); } }
public String getProject() { DataModelDesc modelDesc = getModel(); if (modelDesc == null) { // In case the model for cube not exists in metadata List<ProjectInstance> ownerPrj = ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name); if (ownerPrj.size() == 1) { return ownerPrj.get(0).getName(); } else { throw new IllegalStateException("No project found for cube " + name); } } else { return getModel().getProject(); } }
private void buildMandatoryColumnMask(RowKeyDesc rowKeyDesc) { mandatoryColumnMask = 0L; String[] mandatory_dims = this.selectRule.mandatoryDims; if (mandatory_dims == null || mandatory_dims.length == 0) { return; } for (String dim : mandatory_dims) { TblColRef hColumn = cubeDesc.getModel().findColumn(dim); Integer index = rowKeyDesc.getColumnBitIndex(hColumn); mandatoryColumnMask |= (1L << index); } }
private void checkCubeIsPartitioned(CubeInstance cube) { if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned() == false) { throw new IllegalStateException( "there is no partition date column specified, only full build is supported"); } }
@Override public void validate() throws IllegalStateException { if (cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned()) { if (!isOffsetCube() && dateRangeStart >= dateRangeEnd) throw new IllegalStateException("Invalid segment, dateRangeStart(" + dateRangeStart + ") must be smaller than dateRangeEnd(" + dateRangeEnd + ") in segment " + this); if (isOffsetCube() && sourceOffsetStart >= sourceOffsetEnd) throw new IllegalStateException("Invalid segment, sourceOffsetStart(" + sourceOffsetStart + ") must be smaller than sourceOffsetEnd(" + sourceOffsetEnd + ") in segment " + this); } }
public static Set<String> collectCubeMetadata(CubeInstance cube) { // cube, model_desc, cube_desc, table Set<String> dumpList = new LinkedHashSet<>(); dumpList.add(cube.getResourcePath()); dumpList.add(cube.getDescriptor().getModel().getResourcePath()); dumpList.add(cube.getDescriptor().getResourcePath()); dumpList.add(cube.getProjectInstance().getResourcePath()); for (TableRef tableRef : cube.getDescriptor().getModel().getAllTables()) { TableDesc table = tableRef.getTableDesc(); dumpList.add(table.getResourcePath()); dumpList.addAll(SourceManager.getMRDependentResources(table)); } return dumpList; }
private static void copyACL(CubeInstance cube, String projectName) { operations.add(new Opt(OptType.COPY_ACL, new Object[] { cube.getUuid(), cube.getDescriptor().getModel().getUuid(), projectName })); }
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
@Override public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow) { boolean onlyOneTable = cubeDesc.getModel().getLookupTables().size() == 0; final String baseLocation = getJobWorkingDir(jobFlow); if (onlyOneTable) { // directly use flat table location final String intermediateFactTable = flatDesc.getTableName(); final String tableLocation = baseLocation + "/" + intermediateFactTable; jobFlow.addTask(createSaveKafkaDataStep(jobFlow.getId(), tableLocation, seg)); intermediatePaths.add(tableLocation); } else { final String mockFactTableName = MetadataConstants.KYLIN_INTERMEDIATE_PREFIX + cubeName.toLowerCase(Locale.ROOT) + "_" + seg.getUuid().replaceAll("-", "_") + "_fact"; jobFlow.addTask(createSaveKafkaDataStep(jobFlow.getId(), baseLocation + "/" + mockFactTableName, seg)); jobFlow.addTask(createFlatTable(hiveTableDatabase, mockFactTableName, baseLocation, cubeName, cubeDesc, flatDesc, intermediateTables, intermediatePaths)); } }
public boolean needAutoMerge() { if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned()) return false; return this.getConfig().isAutoMergeEnabled() && this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0; }
@Test public void testBasics() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready"); CubeDesc desc = cube.getDescriptor(); //System.out.println(JsonUtil.writeValueAsIndentString(desc)); String signature = desc.calculateSignature(); desc.getModel().getPartitionDesc().setPartitionDateColumn("test_column"); assertTrue(signature.equals(desc.calculateSignature())); }