/** Returns the key for what this cuboid scheduler responsible for. */ public String getCuboidCacheKey() { return cubeDesc.getName(); }
@Override public String getCuboidCacheKey() { return CubeInstance.class.getSimpleName() + "-" + cubeDesc.getName(); }
protected String makeTableName(CubeDesc cubeDesc, CubeSegment cubeSegment) { if (cubeSegment == null) { return MetadataConstants.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase(Locale.ROOT); } else { return MetadataConstants.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase(Locale.ROOT) + "_" + cubeSegment.getUuid().replaceAll("-", "_"); } }
public Cuboid findMandatoryId(CubeDesc cubeDesc, long cuboidID) { Map<Long, Cuboid> cubeCache = schedulerCuboidCache.get(cubeDesc.getName()); if (cubeCache == null) { cubeCache = Maps.newConcurrentMap(); schedulerCuboidCache.put(cubeDesc.getName(), cubeCache); } Cuboid cuboid = cubeCache.get(cuboidID); if (cuboid == null) { cuboid = new Cuboid(cubeDesc, cuboidID, cuboidID); cubeCache.put(cuboidID, cuboid); } return cuboid; }
private void generateKylinCubeDescFileForSystemCube(String output, CubeDesc cubeDesc) throws IOException { saveSystemCubeMetadataToFile(output + D_CUBE_DESC + cubeDesc.getName() + ".json", cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); }
@Override protected CubeDesc initEntityAfterReload(CubeDesc cubeDesc, String resourceName) { if (cubeDesc.isDraft()) throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, cubeDesc.getName())); try { cubeDesc.init(config); } catch (Exception e) { logger.warn(String.format(Locale.ROOT, BROKEN_CUBE_MSG, cubeDesc.resourceName()), e); cubeDesc.addError(e.toString()); } return cubeDesc; } };
@Nullable @Override public RealizationEntry apply(@Nullable CubeDesc cubeDesc) { if (cubeDesc != null) { RealizationEntry entry = new RealizationEntry(); entry.setRealization(cubeDesc.getName()); entry.setType(RealizationType.CUBE); return entry; } return null; } }));
public void removeCubeDesc(CubeDesc cubeDesc) throws IOException { try (AutoLock lock = descMapLock.lockForWrite()) { crud.delete(cubeDesc); clearCuboidCache(cubeDesc.getName()); } }
private void updateCubeDesc(CubeDesc cubeDesc) { try { String calculatedSign = cubeDesc.calculateSignature(); if (cubeDesc.getSignature() == null || (!cubeDesc.getSignature().equals(calculatedSign))) { cubeDesc.setSignature(calculatedSign); store.checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, CubeDesc.newSerializerForLowLevelAccess()); updatedResources.add(cubeDesc.getResourcePath()); } } catch (Exception e) { logger.error("error", e); errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); } }
private <T> void build(RecordConsumeBlockingQueueController<T> input, ICuboidCollector collector) throws IOException { long startTime = System.currentTimeMillis(); logger.info("In Mem Cube Build start, {}", cubeDesc.getName()); baseCuboidMemTracker = new MemoryWaterLevel(); baseCuboidMemTracker.markLow(); // multiple threads to compute cuboid in parallel taskPending = new TreeSet<CuboidTask>(); taskCuboidCompleted.set(0); taskThreads = prepareTaskThreads(); taskThreadExceptions = new Throwable[taskThreadCount]; // build base cuboid resultCollector = collector; totalSumForSanityCheck = null; baseResult = createBaseCuboid(input); if (baseResult.nRows == 0) return; // plan memory budget baseCuboidMemTracker.markLow(); makeMemoryBudget(); // kick off N-D cuboid tasks and output addChildTasks(baseResult); start(taskThreads); // wait complete join(taskThreads); long endTime = System.currentTimeMillis(); logger.info("In Mem Cube Build end, {}, takes {} ms", cubeDesc.getName(), (endTime - startTime)); throwExceptionIfAny(); }
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')") public CubeInstance createCubeAndDesc(ProjectInstance project, CubeDesc desc) throws IOException { Message msg = MsgPicker.getMsg(); String cubeName = desc.getName(); if (getCubeManager().getCube(cubeName) != null) { throw new BadRequestException(String.format(Locale.ROOT, msg.getCUBE_ALREADY_EXIST(), cubeName)); } if (getCubeDescManager().getCubeDesc(desc.getName()) != null) { throw new BadRequestException(String.format(Locale.ROOT, msg.getCUBE_DESC_ALREADY_EXIST(), desc.getName())); } String owner = SecurityContextHolder.getContext().getAuthentication().getName(); CubeDesc createdDesc; CubeInstance createdCube; createdDesc = getCubeDescManager().createCubeDesc(desc); if (createdDesc.isBroken()) { throw new BadRequestException(createdDesc.getErrorsAsString()); } int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc, false); logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids"); createdCube = getCubeManager().createCube(cubeName, project.getName(), createdDesc, owner); return createdCube; }
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException { try (AutoLock lock = cubeMapLock.lockForWrite()) { logger.info("Creating cube '{}-->{}' from desc '{}'", projectName, cubeName, desc.getName()); // save cube resource CubeInstance cube = CubeInstance.create(cubeName, desc); cube.setOwner(owner); updateCubeWithRetry(new CubeUpdate(cube), 0); ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner); return cube; } }
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException { aclEvaluate.checkProjectWritePermission(cube); Message msg = MsgPicker.getMsg(); final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); if (!cubingJobs.isEmpty()) { throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cube.getName())); } //double check again if (!forceUpdate && !cube.getDescriptor().consistentWith(desc)) { throw new BadRequestException(String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC(), desc.getName())); } CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc); int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc, false); logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); ProjectManager projectManager = getProjectManager(); if (!isCubeInProject(newProjectName, cube)) { String owner = SecurityContextHolder.getContext().getAuthentication().getName(); ProjectInstance newProject = projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), newProjectName, owner); } return updatedCubeDesc; }
public void dropModel(DataModelDesc desc) throws IOException { aclEvaluate.checkProjectWritePermission(desc.getProjectInstance().getName()); Message msg = MsgPicker.getMsg(); //check cube desc exist List<CubeDesc> cubeDescs = getCubeDescManager().listAllDesc(); for (CubeDesc cubeDesc : cubeDescs) { if (cubeDesc.getModelName().equals(desc.getName())) { throw new BadRequestException( String.format(Locale.ROOT, msg.getDROP_REFERENCED_MODEL(), cubeDesc.getName())); } } getDataModelManager().dropModel(desc); }
public void validateCubeDesc(CubeDesc desc, boolean isDraft) { Message msg = MsgPicker.getMsg(); if (desc == null) { throw new BadRequestException(msg.getINVALID_CUBE_DEFINITION()); } String cubeName = desc.getName(); if (StringUtils.isEmpty(cubeName)) { logger.info("Cube name should not be empty."); throw new BadRequestException(msg.getEMPTY_CUBE_NAME()); } if (!ValidateUtil.isAlphanumericUnderscore(cubeName)) { logger.info("Invalid Cube name {}, only letters, numbers and underscore supported.", cubeName); throw new BadRequestException(String.format(Locale.ROOT, msg.getINVALID_CUBE_NAME(), cubeName)); } if (!isDraft) { DataModelDesc modelDesc = modelService.getDataModelManager().getDataModelDesc(desc.getModelName()); if (modelDesc == null) { throw new BadRequestException( String.format(Locale.ROOT, msg.getMODEL_NOT_FOUND(), desc.getModelName())); } if (modelDesc.isDraft()) { logger.info("Cannot use draft model."); throw new BadRequestException( String.format(Locale.ROOT, msg.getUSE_DRAFT_MODEL(), desc.getModelName())); } } }
public static CubeInstance create(String cubeName, CubeDesc cubeDesc) { CubeInstance cubeInstance = new CubeInstance(); cubeInstance.setConfig((KylinConfigExt) cubeDesc.getConfig()); cubeInstance.setName(cubeName); cubeInstance.setDisplayName(cubeName); cubeInstance.setDescName(cubeDesc.getName()); cubeInstance.setCreateTimeUTC(System.currentTimeMillis()); cubeInstance.setSegments(new Segments<CubeSegment>()); cubeInstance.setStatus(RealizationStatusEnum.DISABLED); cubeInstance.updateRandomUuid(); return cubeInstance; }
public CubeDesc updateCube(CubeInstance cube, CubeDesc desc, ProjectInstance project) throws IOException { aclEvaluate.checkProjectWritePermission(cube); Message msg = MsgPicker.getMsg(); String projectName = project.getName(); desc.setDraft(false); try { if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) { throw new BadRequestException( String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC(), desc.getName())); } desc = updateCubeAndDesc(cube, desc, projectName, true); } catch (AccessDeniedException accessDeniedException) { throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT()); } if (desc.isBroken()) { throw new BadRequestException(desc.getErrorsAsString()); } return desc; }
@Test public void testGenSqoopCmd_Partition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"")); source.close(); }
@Test public void testGenSqoopCmd_WithLookupShardBy() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\"")); source.close(); }
@Test public void testGenSqoopCmd_NoPartition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(0L, Long.MAX_VALUE)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue( cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\"")); source.close(); }