private void initConfig() { this.config = KylinConfigExt.createInstance(KylinConfig.getInstanceFromEnv(), this.overrideKylinProps); }
public static KylinConfigExt createInstance(KylinConfig kylinConfig, Map<String, String> overrides) { if (kylinConfig instanceof KylinConfigExt) { return new KylinConfigExt((KylinConfigExt) kylinConfig, overrides); } else { return new KylinConfigExt(kylinConfig, overrides); } }
public void validateAggregationGroupsCombination() { int index = 1; for (AggregationGroup agg : getAggregationGroups()) { try { long combination = agg.calculateCuboidCombination(); if (combination > config.getCubeAggrGroupMaxCombination()) { String msg = "Aggregation group " + index + " of Cube Desc " + this.name + " has too many combinations: " + combination + ". Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; throw new TooManyCuboidException(msg); } } catch (TooManyCuboidException e) { throw e; } catch (Exception e) { throw new IllegalStateException("Unknown error while calculating cuboid number for " + // "Aggregation group " + index + " of Cube Desc " + this.name, e); } index++; } }
this.config = KylinConfigExt.createInstance(config, allOverrideProps); checkArgument(this.rowkey.getRowKeyColumns().length <= this.config.getCubeRowkeyMaxSize(), "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions", this.rowkey.getRowKeyColumns().length);
@Test public void testBackwardCompatibility() { KylinConfig config = KylinConfig.getInstanceFromEnv(); final String oldk = "kylin.test.bcc.old.key"; final String newk = "kylin.test.bcc.new.key"; assertNull(config.getOptional(oldk)); assertNotNull(config.getOptional(newk)); Map<String, String> override = Maps.newHashMap(); override.put(oldk, "1"); KylinConfigExt ext = KylinConfigExt.createInstance(config, override); assertEquals(ext.getOptional(oldk), null); assertEquals(ext.getOptional(newk), "1"); assertNotEquals(config.getOptional(newk), "1"); config.setProperty(oldk, "2"); assertEquals(config.getOptional(newk), "2"); }
/** Get a column which can be used to cluster the source table. * To reduce memory footprint in base cuboid for global dict */ // TODO handle more than one ultra high cardinality columns use global dict in one cube TblColRef getClusteredByColumn() { if (getDistributedByColumn() != null) { return null; } if (dictionaries == null) { return null; } String clusterByColumn = config.getFlatHiveTableClusterByDictColumn(); for (DictionaryDesc dictDesc : dictionaries) { if (dictDesc.getColumnRef().getName().equalsIgnoreCase(clusterByColumn)) { return dictDesc.getColumnRef(); } } return null; }
public static void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList, KylinConfigExt kylinConfig, String metadataUrl) throws IOException { try (AutoDeleteDirectory tmpDir = new AutoDeleteDirectory("kylin_job_meta", ""); AutoDeleteDirectory metaDir = tmpDir.child("meta")) { // dump metadata JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir.getFile(), dumpList); // dump metadata dumpResources(kylinConfig, metaDir.getFile(), dumpList); // write kylin.properties Properties props = kylinConfig.exportToProperties(); props.setProperty("kylin.metadata.url", metadataUrl); File kylinPropsFile = new File(metaDir.getFile(), "kylin.properties"); try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) { props.store(os, kylinPropsFile.getAbsolutePath()); } KylinConfig dstConfig = KylinConfig.createKylinConfig(props); //upload metadata new ResourceTool().copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig); } } }
this.config = KylinConfigExt.createInstance(config, allOverrideProps); checkArgument(this.rowkey.getRowKeyColumns().length <= this.config.getCubeRowkeyMaxSize(), "Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions", this.rowkey.getRowKeyColumns().length);
/** Get a column which can be used to cluster the source table. * To reduce memory footprint in base cuboid for global dict */ // TODO handle more than one ultra high cardinality columns use global dict in one cube TblColRef getClusteredByColumn() { if (getDistributedByColumn() != null) { return null; } if (dictionaries == null) { return null; } String clusterByColumn = config.getFlatHiveTableClusterByDictColumn(); for (DictionaryDesc dictDesc : dictionaries) { if (dictDesc.getColumnRef().getName().equalsIgnoreCase(clusterByColumn)) { return dictDesc.getColumnRef(); } } return null; }
public static void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList, KylinConfigExt kylinConfig, String metadataUrl) throws IOException { try (AutoDeleteDirectory tmpDir = new AutoDeleteDirectory("kylin_job_meta", ""); AutoDeleteDirectory metaDir = tmpDir.child("meta")) { // dump metadata JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir.getFile(), dumpList); // dump metadata dumpResources(kylinConfig, metaDir.getFile(), dumpList); // write kylin.properties Properties props = kylinConfig.exportToProperties(); props.setProperty("kylin.metadata.url", metadataUrl); File kylinPropsFile = new File(metaDir.getFile(), "kylin.properties"); try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) { props.store(os, kylinPropsFile.getAbsolutePath()); } KylinConfig dstConfig = KylinConfig.createKylinConfig(props); //upload metadata new ResourceTool().copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig); } } }
@Test public void testExtShareTheBase() { KylinConfig config = KylinConfig.getInstanceFromEnv(); Map<String, String> override = Maps.newHashMap(); KylinConfig configExt = KylinConfigExt.createInstance(config, override); assertTrue(config.properties == configExt.properties); config.setProperty("1234", "1234"); assertEquals("1234", configExt.getOptional("1234")); }
public static KylinConfigExt createInstance(KylinConfig kylinConfig, Map<String, String> overrides) { if (kylinConfig instanceof KylinConfigExt) { return new KylinConfigExt((KylinConfigExt) kylinConfig, overrides); } else { return new KylinConfigExt(kylinConfig, overrides); } }
public void validateAggregationGroupsCombination() { int index = 1; for (AggregationGroup agg : getAggregationGroups()) { try { long combination = agg.calculateCuboidCombination(); if (combination > config.getCubeAggrGroupMaxCombination()) { String msg = "Aggregation group " + index + " of Cube Desc " + this.name + " has too many combinations: " + combination + ". Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; throw new TooManyCuboidException(msg); } } catch (TooManyCuboidException e) { throw e; } catch (Exception e) { throw new IllegalStateException("Unknown error while calculating cuboid number for " + // "Aggregation group " + index + " of Cube Desc " + this.name, e); } index++; } }
projectInstance.setConfig(KylinConfigExt.createInstance(config, overrides));
projectInstance.setConfig(KylinConfigExt.createInstance(config, overrides));
private void initConfig() { this.config = KylinConfigExt.createInstance(KylinConfig.getInstanceFromEnv(), this.overrideKylinProps); }