private void updateCubeDesc(CubeDesc cubeDesc) { try { String calculatedSign = cubeDesc.calculateSignature(); if (cubeDesc.getSignature() == null || (!cubeDesc.getSignature().equals(calculatedSign))) { cubeDesc.setSignature(calculatedSign); store.checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, CubeDesc.newSerializerForLowLevelAccess()); updatedResources.add(cubeDesc.getResourcePath()); } } catch (Exception e) { logger.error("error", e); errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); } }
void init(KylinConfig config) { CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(descName); checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", descName, name); if (cubeDesc.isBroken()) { setStatus(RealizationStatusEnum.DESCBROKEN); logger.error("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), name); logger.error("Errors: {}", cubeDesc.getErrorsAsString()); } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) { setStatus(RealizationStatusEnum.DISABLED); logger.info("cube {} changed from DESCBROKEN to DISABLED", name); } setConfig((KylinConfigExt) cubeDesc.getConfig()); }
private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException { CubeDesc cubeDesc = cube.getDescriptor(); metaResource.add(cube.getResourcePath()); metaResource.add(cubeDesc.getResourcePath()); metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName())); for (TableRef table : cubeDesc.getModel().getAllTables()) { metaResource.add(table.getTableDesc().getResourcePath()); } for (CubeSegment segment : cube.getSegments()) { metaResource.add(segment.getStatisticsResourcePath()); dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); } }
public static Set<String> collectCubeMetadata(CubeInstance cube) { // cube, model_desc, cube_desc, table Set<String> dumpList = new LinkedHashSet<>(); dumpList.add(cube.getResourcePath()); dumpList.add(cube.getDescriptor().getModel().getResourcePath()); dumpList.add(cube.getDescriptor().getResourcePath()); dumpList.add(cube.getProjectInstance().getResourcePath()); for (TableRef tableRef : cube.getDescriptor().getModel().getAllTables()) { TableDesc table = tableRef.getTableDesc(); dumpList.add(table.getResourcePath()); dumpList.addAll(SourceManager.getMRDependentResources(table)); } return dumpList; }
protected void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException { CubeDesc cubeDesc = cube.getDescriptor(); String prj = cubeDesc.getProject(); metaResource.add(cube.getResourcePath()); metaResource.add(cubeDesc.getResourcePath()); metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName())); Set<TableRef> tblRefs = cubeDesc.getModel().getAllTables(); metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_RESOURCE_ROOT)); metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_EXD_RESOURCE_ROOT)); if (doMigrateSegment) { for (CubeSegment segment : cube.getSegments()) { metaResource.add(segment.getStatisticsResourcePath()); dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); } } if (doAclCopy) { metaResource.add(ACL_PREFIX + cube.getUuid()); metaResource.add(ACL_PREFIX + cube.getModel().getUuid()); } }
newCubeDesc.calculateSignature(); cubeDescManager.createCubeDesc(newCubeDesc); logger.info("CubeDesc was saved at: " + newCubeDesc.getResourcePath()); cubeDesc.calculateSignature(); cubeDescManager.updateCubeDesc(cubeDesc); logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
newCubeDesc.calculateSignature(); cubeDescManager.createCubeDesc(newCubeDesc); logger.info("CubeDesc was saved at: " + newCubeDesc.getResourcePath()); cubeDesc.calculateSignature(); cubeDescManager.updateCubeDesc(cubeDesc); logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, List<String> dictAndSnapshot) throws IOException { CubeDesc cubeDesc = cube.getDescriptor(); metaResource.add(cube.getResourcePath()); metaResource.add(cubeDesc.getResourcePath()); for (String table : cubeDesc.getModel().getAllTables()) { metaResource.add(TableDesc.concatResourcePath(table.toUpperCase())); } for (CubeSegment segment : cube.getSegments()) { dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); } }
private void reloadAllCubeDesc() throws IOException { ResourceStore store = getStore(); logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); cubeDescMap.clear(); List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX); for (String path : paths) { CubeDesc desc; try { desc = loadCubeDesc(path); } catch (Exception e) { logger.error("Error loading cube desc " + path, e); continue; } if (path.equals(desc.getResourcePath()) == false) { logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath()); continue; } if (cubeDescMap.containsKey(desc.getName())) { logger.error("Dup CubeDesc name '" + desc.getName() + "' on path " + path); continue; } cubeDescMap.putLocal(desc.getName(), desc); } logger.debug("Loaded " + cubeDescMap.size() + " Cube(s)"); }
public void removeCubeDesc(CubeDesc cubeDesc) throws IOException { String path = cubeDesc.getResourcePath(); getStore().deleteResource(path); cubeDescMap.remove(cubeDesc.getName()); }
private void upgradeCubeDesc() { logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT); for (String path : paths) { try { CubeDescUpgrader upgrade = new CubeDescUpgrader(path); CubeDesc ndesc = upgrade.upgrade(); ndesc.setSignature(ndesc.calculateSignature()); getStore().putResource(ndesc.getModel().getResourcePath(), ndesc.getModel(), MetadataManager.MODELDESC_SERIALIZER); getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER); updatedResources.add(ndesc.getResourcePath()); } catch (IOException e) { e.printStackTrace(); errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage()); } } }
protected void attachKylinPropsAndMetadata(CubeInstance cube, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); tmp.delete(); // we need a directory, so delete the file first File metaDir = new File(tmp, "meta"); metaDir.mkdirs(); metaDir.getParentFile().deleteOnExit(); // write kylin.properties KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); File kylinPropsFile = new File(metaDir, "kylin.properties"); kylinConfig.writeProperties(kylinPropsFile); // write cube / model_desc / cube_desc / dict / table ArrayList<String> dumpList = new ArrayList<String>(); dumpList.add(cube.getResourcePath()); dumpList.add(cube.getDescriptor().getModel().getResourcePath()); dumpList.add(cube.getDescriptor().getResourcePath()); for (String tableName : cube.getDescriptor().getModel().getAllTables()) { TableDesc table = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName); dumpList.add(table.getResourcePath()); } for (CubeSegment segment : cube.getSegments()) { dumpList.addAll(segment.getDictionaryPaths()); } dumpResources(kylinConfig, metaDir, dumpList); // hadoop distributed cache conf.set("tmpfiles", "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath())); }
public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws IOException { logger.info("Dropping cube '" + cubeName + "'"); // load projects before remove cube from project ResourceStore store = getStore(); // delete cube instance and cube desc CubeInstance cube = getCube(cubeName); if (deleteDesc && cube.getDescriptor() != null) store.deleteResource(cube.getDescriptor().getResourcePath()); store.deleteResource(cube.getResourcePath()); // delete cube from project ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName); // clean cube cache this.afterCubeDropped(cube); return cube; }
String path = cubeDesc.getResourcePath(); getStore().putResource(path, cubeDesc, CUBE_DESC_SERIALIZER); cubeDescMap.put(cubeDesc.getName(), cubeDesc);
String path = desc.getResourcePath(); getStore().putResource(path, desc, CUBE_DESC_SERIALIZER);
private void updateCubeDesc(CubeDesc cubeDesc) { try { String calculatedSign = cubeDesc.calculateSignature(); if (cubeDesc.getSignature() == null || (!cubeDesc.getSignature().equals(calculatedSign))) { cubeDesc.setSignature(calculatedSign); store.checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, CubeDesc.newSerializerForLowLevelAccess()); updatedResources.add(cubeDesc.getResourcePath()); } } catch (Exception e) { logger.error("error", e); errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); } }
private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, List<String> dictAndSnapshot) throws IOException { CubeDesc cubeDesc = cube.getDescriptor(); metaResource.add(cube.getResourcePath()); metaResource.add(cubeDesc.getResourcePath()); metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName())); for (String table : cubeDesc.getModel().getAllTables()) { metaResource.add(TableDesc.concatResourcePath(table.toUpperCase())); } for (CubeSegment segment : cube.getSegments()) { dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); } }
void init(KylinConfig config) { CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(descName); checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", descName, name); if (cubeDesc.isBroken()) { setStatus(RealizationStatusEnum.DESCBROKEN); logger.error("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), name); logger.error("Errors: {}", cubeDesc.getErrorsAsString()); } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) { setStatus(RealizationStatusEnum.DISABLED); logger.info("cube {} changed from DESCBROKEN to DISABLED", name); } setConfig((KylinConfigExt) cubeDesc.getConfig()); }
public static Set<String> collectCubeMetadata(CubeInstance cube) { // cube, model_desc, cube_desc, table Set<String> dumpList = new LinkedHashSet<>(); dumpList.add(cube.getResourcePath()); dumpList.add(cube.getDescriptor().getModel().getResourcePath()); dumpList.add(cube.getDescriptor().getResourcePath()); dumpList.add(cube.getProjectInstance().getResourcePath()); for (TableRef tableRef : cube.getDescriptor().getModel().getAllTables()) { TableDesc table = tableRef.getTableDesc(); dumpList.add(table.getResourcePath()); dumpList.addAll(SourceManager.getMRDependentResources(table)); } return dumpList; }
public void removeCubeDesc(CubeDesc cubeDesc) throws IOException { String path = cubeDesc.getResourcePath(); getStore().deleteResource(path); cubeDescMap.remove(cubeDesc.getName()); Cuboid.reloadCache(cubeDesc.getName()); }