ObjectNode message = JsonUtils.newObjectNode(); for (FieldSpec spec : pinotSchema.getDimensionFieldSpecs()) { message.set(spec.getName(), JsonUtils.objectToJsonNode(record.get(spec.getName()))); for (FieldSpec spec : pinotSchema.getDimensionFieldSpecs()) { message.set(spec.getName(), JsonUtils.objectToJsonNode(record.get(spec.getName())));
for (DimensionFieldSpec fieldSpec : schema.getDimensionFieldSpecs()) { if (!fieldSpec.isSingleValueField() || schema.isVirtualColumn(fieldSpec.getName())) { continue;
private static void printSchema(Schema schema) { LOGGER.info("schemaName: {}", schema.getSchemaName()); LOGGER.info("Dimension columnNames: "); int i = 0; for (DimensionFieldSpec spec : schema.getDimensionFieldSpecs()) { String columnInfo = i + " " + spec.getName(); if (!spec.isSingleValueField()) { LOGGER.info(columnInfo + " Multi-Value."); } else { LOGGER.info(columnInfo); } i += 1; } LOGGER.info("Metric columnNames: "); i = 0; for (MetricFieldSpec spec : schema.getMetricFieldSpecs()) { String columnInfo = i + " " + spec.getName(); if (!spec.isSingleValueField()) { LOGGER.info(columnInfo + " Multi-Value."); } else { LOGGER.info(columnInfo); } i += 1; } LOGGER.info("Time column: {}", schema.getTimeColumnName()); }
/** * Rolls up input segments using segment converter. * @param schema input schema * @return a list of rolled-up segments */ private List<File> rollupSegments(Schema schema) throws Exception { // Compute group by columns for roll-up preparation (all dimensions + time column) List<String> groupByColumns = new ArrayList<>(); for (DimensionFieldSpec dimensionFieldSpec : schema.getDimensionFieldSpecs()) { groupByColumns.add(dimensionFieldSpec.getName()); } String timeColumn = schema.getTimeColumnName(); if (timeColumn != null) { groupByColumns.add(timeColumn); } // Initialize roll-up record transformer // TODO: add the support for roll-up with time granularity change RecordTransformer rollupRecordTransformer = (row) -> row; // Initialize roll-up record aggregator RecordAggregator rollupRecordAggregator = new RollupRecordAggregator(schema, _rolllupPreAggregateType); SegmentConverter rollupSegmentConverter = new SegmentConverter.Builder().setTableName(_tableName).setSegmentName(_segmentName) .setInputIndexDirs(_inputIndexDirs).setWorkingDir(_workingDir).setRecordTransformer(rollupRecordTransformer) .setRecordAggregator(rollupRecordAggregator).setGroupByColumns(groupByColumns) .setIndexingConfig(_indexingConfig).build(); return rollupSegmentConverter.convertSegment(); }
String dimName = schema.getDimensionFieldSpecs().get(i).getName(); map.put(dimName, dimName + "-v" + rowId % (NUM_DIMENSIONS - i)); String dimName = schema.getDimensionFieldSpecs().get(i).getName(); map.put(dimName, dimName + "-v" + RANDOM.nextInt(i * 100));
for (int row = 0; row < ROWS; row++) { for (int i = 0; i < numDimensions; i++) { String dimName = schema.getDimensionFieldSpecs().get(i).getName(); map.put(dimName, dimName + "-v" + row % (numDimensions - i));
dimName = schema.getDimensionFieldSpecs().get(i).getName(); map.put(dimName, dimName + "-v" + row % (numDimensions - i));