throws IOException { Random random = new Random(System.nanoTime()); StarTreeIndexSpec expectedStarTreeSpec = new StarTreeIndexSpec(); expectedStarTreeSpec.setDimensionsSplitOrder(expectedDimensionSplitOrder); expectedStarTreeSpec.setMaxLeafRecords(expectedMaxLeafRecords); expectedStarTreeSpec.setSkipMaterializationCardinalityThreshold(expectedSkipMaterializationThreshold); expectedStarTreeSpec.setSkipMaterializationForDimensions(expectedSkipMaterializationDimensions); expectedStarTreeSpec.setSkipStarNodeCreationForDimensions(expectedSkipStarNodeCreationForDimension); StarTreeIndexSpec actualStarTreeSpec = actualIndexingConfig.getStarTreeIndexSpec(); assertEquals(actualStarTreeSpec.getDimensionsSplitOrder(), expectedDimensionSplitOrder); assertEquals(actualStarTreeSpec.getMaxLeafRecords(), expectedMaxLeafRecords); assertEquals(actualStarTreeSpec.getSkipMaterializationCardinalityThreshold(), expectedSkipMaterializationThreshold); assertEquals(actualStarTreeSpec.getSkipMaterializationForDimensions(), expectedSkipMaterializationDimensions); assertEquals(actualStarTreeSpec.getSkipStarNodeCreationForDimensions(), expectedSkipStarNodeCreationForDimension);
private void initStarTreeIndex(JobContext context) throws IOException { String _starTreeIndexSpecFile = PinotOutputFormat.getStarTreeIndexSpec(context); if (_starTreeIndexSpecFile != null) { StarTreeIndexSpec starTreeIndexSpec = StarTreeIndexSpec.fromFile(new File(_starTreeIndexSpecFile)); // Specifying star-tree index file enables star tree generation, even if _enableStarTreeIndex is not specified. _segmentConfig.enableStarTreeIndex(starTreeIndexSpec); } else if (PinotOutputFormat.getEnableStarTreeIndex(context)) { _segmentConfig.enableStarTreeIndex(null); } } }
public static StarTreeIndexSpec fromStarTreeMetadata(StarTreeMetadata starTreeMetadata) { StarTreeIndexSpec starTreeIndexSpec = new StarTreeIndexSpec(); starTreeIndexSpec.setMaxLeafRecords(starTreeMetadata.getMaxLeafRecords()); starTreeIndexSpec.setDimensionsSplitOrder(starTreeMetadata.getDimensionsSplitOrder()); starTreeIndexSpec .setSkipStarNodeCreationForDimensions(Sets.newHashSet(starTreeMetadata.getSkipStarNodeCreationForDimensions())); starTreeIndexSpec .setSkipMaterializationForDimensions(Sets.newHashSet(starTreeMetadata.getSkipMaterializationForDimensions())); starTreeIndexSpec.setSkipMaterializationCardinalityThreshold(starTreeMetadata.getSkipMaterializationCardinality()); return starTreeIndexSpec; }
private void checkTableConfigWithStarTreeConfig(TableConfig tableConfig, TableConfig tableConfigToCompare) throws Exception { // Check that the segment assignment configuration does exist. Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); Assert.assertNotNull(tableConfigToCompare.getIndexingConfig().getStarTreeIndexSpec()); // Check that the configurations are correct. StarTreeIndexSpec starTreeIndexSpec = tableConfigToCompare.getIndexingConfig().getStarTreeIndexSpec(); Set<String> dims = new HashSet<>(); dims.add("dims"); Assert.assertEquals(starTreeIndexSpec.getDimensionsSplitOrder(), Collections.singletonList("dim")); Assert.assertEquals(starTreeIndexSpec.getMaxLeafRecords(), 5); Assert.assertEquals(starTreeIndexSpec.getSkipMaterializationCardinalityThreshold(), 1); Assert.assertEquals(starTreeIndexSpec.getSkipMaterializationForDimensions(), dims); Assert.assertEquals(starTreeIndexSpec.getSkipStarNodeCreationForDimensions(), dims); starTreeIndexSpec = StarTreeIndexSpec.fromJsonString(starTreeIndexSpec.toJsonString()); Assert.assertEquals(starTreeIndexSpec.getDimensionsSplitOrder(), Collections.singletonList("dim")); Assert.assertEquals(starTreeIndexSpec.getMaxLeafRecords(), 5); Assert.assertEquals(starTreeIndexSpec.getSkipMaterializationCardinalityThreshold(), 1); Assert.assertEquals(starTreeIndexSpec.getSkipMaterializationForDimensions(), dims); Assert.assertEquals(starTreeIndexSpec.getSkipStarNodeCreationForDimensions(), dims); }
starTreeIndexSpec = new StarTreeIndexSpec(); starTreeIndexSpec.setMaxLeafRecords(StarTreeIndexSpec.DEFAULT_MAX_LEAF_RECORDS); starTreeBuilderConfig.setOutDir(starTreeTempDir); starTreeBuilderConfig.setSchema(dataSchema); starTreeBuilderConfig.setDimensionsSplitOrder(starTreeIndexSpec.getDimensionsSplitOrder()); starTreeBuilderConfig.setSkipStarNodeCreationDimensions(starTreeIndexSpec.getSkipStarNodeCreationForDimensions()); starTreeBuilderConfig.setSkipMaterializationDimensions(starTreeIndexSpec.getSkipMaterializationForDimensions()); starTreeBuilderConfig .setSkipMaterializationCardinalityThreshold(starTreeIndexSpec.getSkipMaterializationCardinalityThreshold()); starTreeBuilderConfig.setMaxNumLeafRecords(starTreeIndexSpec.getMaxLeafRecords()); starTreeBuilderConfig.setExcludeSkipMaterializationDimensionsForStarTreeIndex( starTreeIndexSpec.isExcludeSkipMaterializationDimensionsForStarTreeIndex()); starTreeIndexSpec.setDimensionsSplitOrder(starTreeBuilder.getDimensionsSplitOrder()); starTreeIndexSpec.setSkipMaterializationForDimensions(starTreeBuilder.getSkipMaterializationDimensions());
properties.setProperty(STAR_TREE_SPLIT_ORDER, starTreeIndexSpec.getDimensionsSplitOrder()); properties.setProperty(STAR_TREE_MAX_LEAF_RECORDS, starTreeIndexSpec.getMaxLeafRecords()); properties.setProperty(STAR_TREE_SKIP_STAR_NODE_CREATION_FOR_DIMENSIONS, starTreeIndexSpec.getSkipStarNodeCreationForDimensions()); properties.setProperty(STAR_TREE_SKIP_MATERIALIZATION_CARDINALITY, starTreeIndexSpec.getSkipMaterializationCardinalityThreshold()); properties.setProperty(STAR_TREE_SKIP_MATERIALIZATION_FOR_DIMENSIONS, starTreeIndexSpec.getSkipMaterializationForDimensions());
private void setupStarTreeConfig(SegmentGeneratorConfig segmentGenConfig) { // StarTree related StarTreeIndexSpec starTreeIndexSpec = new StarTreeIndexSpec(); starTreeIndexSpec.setMaxLeafRecords(StarTreeIndexSpec.DEFAULT_MAX_LEAF_RECORDS); segmentGenConfig.enableStarTreeIndex(starTreeIndexSpec); LOGGER.info("segmentGenConfig Schema (w/o derived fields): "); printSchema(segmentGenConfig.getSchema()); }
segmentGeneratorConfig.setSegmentName(SEGMENT_NAME_STARTREE); segmentGeneratorConfig.setOutDir(INDEX_DIR_STARTREE.getAbsolutePath()); segmentGeneratorConfig.enableStarTreeIndex(new StarTreeIndexSpec());
private void setupSegment(File segmentDir) throws Exception { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA)); if (segmentDir.exists()) { FileUtils.deleteQuietly(segmentDir); } final SegmentGeneratorConfig config = SegmentTestUtils .getSegmentGenSpecWithSchemAndProjectedColumns(new File(filePath), segmentDir, "time_day", TimeUnit.DAYS, TABLE_NAME); config.setTableName(TABLE_NAME); config.setSegmentName(SEGMENT_NAME); StarTreeIndexSpec starTreeIndexSpec = new StarTreeIndexSpec(); starTreeIndexSpec.setDimensionsSplitOrder(DIMENSIONS_SPLIT_ORDER); starTreeIndexSpec.setMaxLeafRecords(MAX_LEAF_RECORDS); starTreeIndexSpec.setSkipMaterializationCardinalityThreshold(SKIP_CARDINALITY_THRESHOLD); starTreeIndexSpec.setSkipStarNodeCreationForDimensions(SKIP_STAR_NODE_CREATION_DIMENSTIONS); starTreeIndexSpec.setSkipMaterializationForDimensions(SKIP_MATERIALIZATION_DIMENSIONS); config.enableStarTreeIndex(starTreeIndexSpec); final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null); driver.init(config); driver.build(); }
StarTreeIndexSpec starTreeIndexSpec = new StarTreeIndexSpec(); starTreeIndexSpec.setMaxLeafRecords(10); config.enableStarTreeIndex(starTreeIndexSpec); config.setOutDir(segmentDirName);
StarTreeIndexSpec starTreeIndexSpec = new StarTreeIndexSpec(); Set<String> dims = new HashSet<>(); dims.add("dims"); starTreeIndexSpec.setDimensionsSplitOrder(Collections.singletonList("dim")); starTreeIndexSpec.setMaxLeafRecords(5); starTreeIndexSpec.setSkipMaterializationCardinalityThreshold(1); starTreeIndexSpec.setSkipMaterializationForDimensions(dims); starTreeIndexSpec.setSkipStarNodeCreationForDimensions(dims);
StarTreeIndexSpec starTreeIndexSpec = StarTreeIndexSpec.fromFile(new File(_starTreeIndexSpecFile));
/** * Helper method to perform the conversion. * @param columnarSegment Columnar segment directory to convert * @throws Exception */ private void convertSegment(File columnarSegment) throws Exception { PinotSegmentRecordReader pinotSegmentRecordReader = new PinotSegmentRecordReader(columnarSegment); SegmentGeneratorConfig config = new SegmentGeneratorConfig(pinotSegmentRecordReader.getSchema()); config.setDataDir(_inputDirName); config.setInputFilePath(columnarSegment.getAbsolutePath()); config.setFormat(FileFormat.PINOT); config.setOutDir(_outputDirName); config.setOverwrite(_overwrite); StarTreeIndexSpec starTreeIndexSpec = null; if (_starTreeConfigFileName != null) { starTreeIndexSpec = StarTreeIndexSpec.fromFile(new File(_starTreeConfigFileName)); } config.enableStarTreeIndex(starTreeIndexSpec); // Read the segment and table name from the segment's metadata. SegmentMetadata metadata = new SegmentMetadataImpl(columnarSegment); SegmentNameGenerator nameGenerator = new DefaultSegmentNameGenerator(metadata.getName()); config.setSegmentNameGenerator(nameGenerator); config.setTableName(metadata.getTableName()); SegmentIndexCreationDriver indexCreator = new SegmentIndexCreationDriverImpl(); indexCreator.init(config); indexCreator.build(); }