public PinotRecord(Schema schema) { _pinotSchema = schema; _fieldNames = _pinotSchema.getColumnNames(); }
@Override public Set<String> getColumnNames() { return _schema.getColumnNames(); }
@Override public Set<String> getColumnNames() { return _segmentMetadata.getSchema().getColumnNames(); }
@Override public boolean prune(IndexSegment segment, ServerQueryRequest queryRequest) { Set<String> columnsInSchema = segment.getSegmentMetadata().getSchema().getColumnNames(); return !columnsInSchema.containsAll(queryRequest.getAllColumns()); }
private void addDerivedFieldsInSchema() { if (createHllIndex) { Collection<String> columnNames = dataSchema.getColumnNames(); HllConfig hllConfig = config.getHllConfig(); for (String derivedFieldName : hllConfig.getDerivedHllFieldToOriginMap().keySet()) { if (columnNames.contains(derivedFieldName)) { throw new IllegalArgumentException( "Cannot add derived field: " + derivedFieldName + " since it already exists in schema."); } else { dataSchema.addField( new MetricFieldSpec(derivedFieldName, FieldSpec.DataType.STRING, hllConfig.getHllFieldSize(), MetricFieldSpec.DerivedMetricType.HLL)); } } } }
@Override public T deserialize(PinotRecord record) throws IOException { ObjectNode jsonRecord = JsonUtils.newObjectNode(); for (String column : _schema.getColumnNames()) { jsonRecord.set(column, JsonUtils.objectToJsonNode(record.getValue(column))); } return JsonUtils.jsonNodeToObject(jsonRecord, getJsonReaderClass(_conf)); }
columnNames = new ArrayList<>(metadata.getSchema().getColumnNames()); Collections.sort(columnNames);
Collection<String> columnNames = _schema.getColumnNames(); _columnReaderMap = new HashMap<>(columnNames.size()); for (String columnName : columnNames) {
@Test public void testColumnStatsWithoutStarTree() { SegmentWithHllIndexCreateHelper helper = null; boolean hasException = false; try { LOGGER.debug("================ Without StarTree ================"); helper = new SegmentWithHllIndexCreateHelper("noStarTree", getClass().getClassLoader().getResource(AVRO_DATA), timeColumnName, timeUnit, "starTreeSegment"); SegmentIndexCreationDriver driver = helper.build(false, null); LOGGER.debug("================ Cardinality ================"); for (String name : helper.getSchema().getColumnNames()) { LOGGER.debug("* " + name + ": " + driver.getColumnStatisticsCollector(name).getCardinality()); } } catch (Exception e) { hasException = true; LOGGER.error(e.getMessage()); } finally { if (helper != null) { helper.cleanTempDir(); } Assert.assertEquals(hasException, false); } }
for (String columnName : schema.getColumnNames()) { if (schema.isVirtualColumn(columnName)) { FieldSpec fieldSpec = schema.getFieldSpecFor(columnName);
@Test public void testNoVirtualColumnsInSchema() { Schema schema = new Schema(); FieldSpec spec = new DimensionFieldSpec("col1", FieldSpec.DataType.STRING, true); schema.addField(spec); TimeFieldSpec tfs = new TimeFieldSpec("col1", FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "col2", FieldSpec.DataType.LONG, TimeUnit.DAYS); schema.addField(tfs); VirtualColumnProviderFactory.addBuiltInVirtualColumnsToSchema(schema); Assert.assertEquals(schema.getColumnNames().size(), 5); Assert.assertEquals(schema.getTimeFieldSpec().getIncomingGranularitySpec().getTimeType(), TimeUnit.MILLISECONDS); RealtimeSegmentConverter converter = new RealtimeSegmentConverter(null, "", schema, "testTable", "col1", "segment1", "col1"); Schema newSchema = converter.getUpdatedSchema(schema); Assert.assertEquals(newSchema.getColumnNames().size(), 2); Assert.assertEquals(newSchema.getTimeFieldSpec().getIncomingGranularitySpec().getTimeType(), TimeUnit.DAYS); } }
_tableName = segmentMetadata.getTableName(); _segmentName = segmentMetadata.getSegmentName(); _allColumns = schema.getColumnNames(); _schema = schema; _totalDocs = segmentMetadataPropertiesConfiguration.getInt(SEGMENT_TOTAL_DOCS);
SegmentIndexCreationDriver driver = helper.build(true, hllConfig); LOGGER.debug("================ Cardinality ================"); for (String name : helper.getSchema().getColumnNames()) { LOGGER.debug("* " + name + ": " + driver.getColumnStatisticsCollector(name).getCardinality());
@Test public void test() throws Exception { PinotSegmentRecordReader pinotReader = new PinotSegmentRecordReader(new File(SEGMENT_DIR_NAME, SEGMENT_NAME)); _recordReader.rewind(); while (pinotReader.hasNext()) { GenericRow expectedRow = _recordReader.next(); GenericRow actualRow = pinotReader.next(); for (String column : _schema.getColumnNames()) { byte[] actual = (byte[]) actualRow.getValue(column); byte[] expected = (byte[]) expectedRow.getValue(column); if (ByteArray.compare(actual, expected) != 0) { Assert.assertEquals(actualRow.getValue(column), expectedRow.getValue(column)); } } } // Ensure both record readers are exhausted, ie same number of rows. Assert.assertTrue(!_recordReader.hasNext()); pinotReader.close(); }