public SchemaBuilder addSingleValueDimension(@Nonnull String dimensionName, @Nonnull DataType dataType, @Nonnull Object defaultNullValue) { _schema.addField(new DimensionFieldSpec(dimensionName, dataType, true, defaultNullValue)); return this; }
if (!fieldSpec.isSingleValueField() || schema.isVirtualColumn(fieldSpec.getName())) { continue; String column = fieldSpec.getName(); uniqueSingleValueDimensions.add(column); loadValuesForSingleValueDimension(indexSegment, singleValueDimensionValuesMap, column);
DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(dimName, FieldSpec.DataType.STRING, true); schema.addField(dimensionFieldSpec); String dimName = schema.getDimensionFieldSpecs().get(i).getName(); map.put(dimName, dimName + "-v" + rowId % (NUM_DIMENSIONS - i)); String dimName = schema.getDimensionFieldSpecs().get(i).getName(); map.put(dimName, dimName + "-v" + RANDOM.nextInt(i * 100));
Assert.assertEquals(dimensionFieldSpec.getFieldType(), FieldSpec.FieldType.DIMENSION); Assert.assertEquals(dimensionFieldSpec.getName(), "svDimension"); Assert.assertEquals(dimensionFieldSpec.getDataType(), FieldSpec.DataType.INT); Assert.assertEquals(dimensionFieldSpec.isSingleValueField(), true); Assert.assertEquals(dimensionFieldSpec.getDefaultNullValue(), Integer.MIN_VALUE); Assert.assertEquals(dimensionFieldSpec.getFieldType(), FieldSpec.FieldType.DIMENSION); Assert.assertEquals(dimensionFieldSpec.getName(), "svDimensionWithDefault"); Assert.assertEquals(dimensionFieldSpec.getDataType(), FieldSpec.DataType.INT); Assert.assertEquals(dimensionFieldSpec.isSingleValueField(), true); Assert.assertEquals(dimensionFieldSpec.getDefaultNullValue(), 10); Assert.assertEquals(dimensionFieldSpec.getFieldType(), FieldSpec.FieldType.DIMENSION); Assert.assertEquals(dimensionFieldSpec.getName(), "mvDimension"); Assert.assertEquals(dimensionFieldSpec.getDataType(), FieldSpec.DataType.STRING); Assert.assertEquals(dimensionFieldSpec.isSingleValueField(), false); Assert.assertEquals(dimensionFieldSpec.getDefaultNullValue(), "null"); Assert.assertEquals(dimensionFieldSpec.getFieldType(), FieldSpec.FieldType.DIMENSION); Assert.assertEquals(dimensionFieldSpec.getName(), "mvDimensionWithDefault"); Assert.assertEquals(dimensionFieldSpec.getDataType(), FieldSpec.DataType.STRING); Assert.assertEquals(dimensionFieldSpec.isSingleValueField(), false); Assert.assertEquals(dimensionFieldSpec.getDefaultNullValue(), defaultString);
DimensionFieldSpec dimensionFieldSpec1 = JsonUtils.stringToObject(getRandomOrderJsonString(dimensionFields), DimensionFieldSpec.class); DimensionFieldSpec dimensionFieldSpec2 = new DimensionFieldSpec("dimension", BOOLEAN, true, false); Assert.assertEquals(dimensionFieldSpec1, dimensionFieldSpec2, ERROR_MESSAGE); Assert.assertEquals(dimensionFieldSpec1.getDefaultNullValue(), "false", ERROR_MESSAGE); new String[]{"\"name\":\"dimension\"", "\"dataType\":\"STRING\"", "\"singleValueField\":false", "\"defaultNullValue\":\"default\""}; dimensionFieldSpec1 = JsonUtils.stringToObject(getRandomOrderJsonString(dimensionFields), DimensionFieldSpec.class); dimensionFieldSpec2 = new DimensionFieldSpec("dimension", STRING, false, "default"); Assert.assertEquals(dimensionFieldSpec1, dimensionFieldSpec2, ERROR_MESSAGE); Assert.assertEquals(dimensionFieldSpec1.getDefaultNullValue(), "default", ERROR_MESSAGE);
/** * Rolls up input segments using segment converter. * @param schema input schema * @return a list of rolled-up segments */ private List<File> rollupSegments(Schema schema) throws Exception { // Compute group by columns for roll-up preparation (all dimensions + time column) List<String> groupByColumns = new ArrayList<>(); for (DimensionFieldSpec dimensionFieldSpec : schema.getDimensionFieldSpecs()) { groupByColumns.add(dimensionFieldSpec.getName()); } String timeColumn = schema.getTimeColumnName(); if (timeColumn != null) { groupByColumns.add(timeColumn); } // Initialize roll-up record transformer // TODO: add the support for roll-up with time granularity change RecordTransformer rollupRecordTransformer = (row) -> row; // Initialize roll-up record aggregator RecordAggregator rollupRecordAggregator = new RollupRecordAggregator(schema, _rolllupPreAggregateType); SegmentConverter rollupSegmentConverter = new SegmentConverter.Builder().setTableName(_tableName).setSegmentName(_segmentName) .setInputIndexDirs(_inputIndexDirs).setWorkingDir(_workingDir).setRecordTransformer(rollupRecordTransformer) .setRecordAggregator(rollupRecordAggregator).setGroupByColumns(groupByColumns) .setIndexingConfig(_indexingConfig).build(); return rollupSegmentConverter.convertSegment(); }
for (int i = 0; i < numDimensions; i++) { String dimName = "d" + (i + 1); DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(dimName, DataType.STRING, true); schema.addField(dimensionFieldSpec); for (int row = 0; row < ROWS; row++) { for (int i = 0; i < numDimensions; i++) { String dimName = schema.getDimensionFieldSpecs().get(i).getName(); map.put(dimName, dimName + "-v" + row % (numDimensions - i));
public SchemaBuilder addMultiValueDimension(@Nonnull String dimensionName, @Nonnull DataType dataType, @Nonnull Object defaultNullValue) { _schema.addField(new DimensionFieldSpec(dimensionName, dataType, false, defaultNullValue)); return this; }
DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(dimName, DataType.INT, true); schema.addField(dimensionFieldSpec); dimensionFieldSpec = new DimensionFieldSpec(dimName, DataType.STRING, true); schema.addField(dimensionFieldSpec); dimName = schema.getDimensionFieldSpecs().get(i).getName(); map.put(dimName, dimName + "-v" + row % (numDimensions - i));
private static void printSchema(Schema schema) { LOGGER.info("schemaName: {}", schema.getSchemaName()); LOGGER.info("Dimension columnNames: "); int i = 0; for (DimensionFieldSpec spec : schema.getDimensionFieldSpecs()) { String columnInfo = i + " " + spec.getName(); if (!spec.isSingleValueField()) { LOGGER.info(columnInfo + " Multi-Value."); } else { LOGGER.info(columnInfo); } i += 1; } LOGGER.info("Metric columnNames: "); i = 0; for (MetricFieldSpec spec : schema.getMetricFieldSpecs()) { String columnInfo = i + " " + spec.getName(); if (!spec.isSingleValueField()) { LOGGER.info(columnInfo + " Multi-Value."); } else { LOGGER.info(columnInfo); } i += 1; } LOGGER.info("Time column: {}", schema.getTimeColumnName()); }
public SchemaBuilder addSingleValueDimension(@Nonnull String dimensionName, @Nonnull DataType dataType) { _schema.addField(new DimensionFieldSpec(dimensionName, dataType, true)); return this; }
public SchemaBuilder addMultiValueDimension(@Nonnull String dimensionName, @Nonnull DataType dataType) { _schema.addField(new DimensionFieldSpec(dimensionName, dataType, false)); return this; }
public static void addBuiltInVirtualColumnsToSchema(Schema schema) { if (!schema.hasColumn("$docId")) { schema.addField(new DimensionFieldSpec("$docId", FieldSpec.DataType.INT, true, DocIdVirtualColumnProvider.class)); } if (!schema.hasColumn("$hostName")) { schema.addField( new DimensionFieldSpec("$hostName", FieldSpec.DataType.STRING, true, HostNameVirtualColumnProvider.class)); } if (!schema.hasColumn("$segmentName")) { schema.addField(new DimensionFieldSpec("$segmentName", FieldSpec.DataType.STRING, true, SegmentNameVirtualColumnProvider.class)); } } }
boolean isSingleValueField = isSingleValueField(field); if (fieldTypeMap == null) { pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); } else { FieldSpec.FieldType fieldType = fieldTypeMap.get(fieldName); switch (fieldType) { case DIMENSION: pinotSchema.addField(new DimensionFieldSpec(fieldName, dataType, isSingleValueField)); break; case METRIC:
/** * Helper method to build a schema with provided number of metric columns. * * @return Schema containing the given number of metric columns */ private static Schema buildSchema() { Schema schema = new Schema(); schema.addField(new DimensionFieldSpec(INT_COLUMN, FieldSpec.DataType.INT, true)); schema.addField(new DimensionFieldSpec(LONG_COLUMN, FieldSpec.DataType.LONG, true)); schema.addField(new DimensionFieldSpec(FLOAT_COLUMN, FieldSpec.DataType.FLOAT, true)); schema.addField(new DimensionFieldSpec(DOUBLE_COLUMN, FieldSpec.DataType.DOUBLE, true)); schema.addField(new DimensionFieldSpec(STRING_COLUMN, FieldSpec.DataType.STRING, true)); return schema; } }
/** * Setup to build a segment with raw indexes (no-dictionary) of various data types. * * @throws Exception */ @BeforeClass public void setup() throws Exception { Schema schema = new Schema(); schema.addField(new DimensionFieldSpec(INT_COLUMN, FieldSpec.DataType.INT, true)); schema.addField(new DimensionFieldSpec(LONG_COLUMN, FieldSpec.DataType.LONG, true)); schema.addField(new DimensionFieldSpec(FLOAT_COLUMN, FieldSpec.DataType.FLOAT, true)); schema.addField(new DimensionFieldSpec(DOUBLE_COLUMN, FieldSpec.DataType.DOUBLE, true)); schema.addField(new DimensionFieldSpec(STRING_COLUMN, FieldSpec.DataType.STRING, true)); _random = new Random(System.nanoTime()); _recordReader = buildIndex(schema); }
private FieldSpec buildSpec(DataGeneratorSpec genSpec, String column) { DataType dataType = genSpec.getDataTypesMap().get(column); FieldType fieldType = genSpec.getFieldTypesMap().get(column); FieldSpec spec; switch (fieldType) { case DIMENSION: spec = new DimensionFieldSpec(); break; case METRIC: spec = new MetricFieldSpec(); break; case TIME: spec = new TimeFieldSpec(column, dataType, genSpec.getTimeUnitMap().get(column)); break; default: throw new RuntimeException("Invalid Field type."); } spec.setName(column); spec.setDataType(dataType); spec.setSingleValueField(true); return spec; }
_indexDir.deleteOnExit(); FieldSpec fieldSpec = new DimensionFieldSpec(COLUMN_NAME, FieldSpec.DataType.STRING, true); schema.addField(fieldSpec);
/** * Setup to build a segment with raw indexes (no-dictionary) of various data types. * * @throws Exception */ @BeforeClass public void setup() throws Exception { _schema = new Schema(); _schema.addField(new DimensionFieldSpec(FIXED_BYTE_SORTED_COLUMN, FieldSpec.DataType.BYTES, true)); _schema.addField(new DimensionFieldSpec(FIXED_BYTES_UNSORTED_COLUMN, FieldSpec.DataType.BYTES, true)); _schema.addField(new DimensionFieldSpec(FIXED_BYTES_NO_DICT_COLUMN, FieldSpec.DataType.BYTES, true)); _schema.addField(new DimensionFieldSpec(VARIABLE_BYTES_COLUMN, FieldSpec.DataType.BYTES, true)); _random = new Random(System.nanoTime()); _recordReader = buildIndex(_schema); _segment = ImmutableSegmentLoader.load(new File(SEGMENT_DIR_NAME, SEGMENT_NAME), ReadMode.heap); }
DimensionFieldSpec dimensionFieldSpec = new DimensionFieldSpec(column, FieldSpec.DataType.STRING, true); schema.addField(dimensionFieldSpec);