@Override public void postInject() { // Compute the actual default null value from its string representation _defaultNullValue = getDefaultNullValue(getFieldType(), _dataType, _stringDefaultNullValue); }
public static Object convertToDataTypeArray(String[] tokens, FieldSpec fieldSpec) { Object[] value; if ((tokens == null) || (tokens.length == 0)) { value = new Object[]{fieldSpec.getDefaultNullValue()}; } else { int length = tokens.length; value = new Object[length]; for (int i = 0; i < length; i++) { value[i] = convertToDataType(tokens[i], fieldSpec); } } return value; }
public static Object convertToDataTypeArray(ArrayList tokens, FieldSpec fieldSpec) { Object[] value; if ((tokens == null) || tokens.isEmpty()) { value = new Object[]{fieldSpec.getDefaultNullValue()}; } else { int length = tokens.size(); value = new Object[length]; for (int i = 0; i < length; i++) { Object token = tokens.get(i); if (token == null) { value[i] = fieldSpec.getDefaultNullValue(); } else { value[i] = convertToDataType(token.toString(), fieldSpec); } } } return value; }
public static Object convertToDataTypeSet(HashSet tokens, FieldSpec fieldSpec) { Object[] value; if ((tokens == null) || tokens.isEmpty()) { value = new Object[]{fieldSpec.getDefaultNullValue()}; } else { int length = tokens.size(); value = new Object[length]; int index = 0; for (Object token : tokens) { if (token == null) { value[index] = fieldSpec.getDefaultNullValue(); } else { value[index] = convertToDataType(token.toString(), fieldSpec); } index++; } } return value; }
public static Object convertToDataType(String token, FieldSpec fieldSpec) { if ((token == null) || token.isEmpty()) { return fieldSpec.getDefaultNullValue(); } DataType dataType = fieldSpec.getDataType(); switch (dataType) { case INT: return Integer.parseInt(token); case LONG: return Long.parseLong(token); case FLOAT: return Float.parseFloat(token); case DOUBLE: return Double.parseDouble(token); case STRING: return token; default: throw new IllegalStateException("Illegal data type: " + dataType); } }
protected void appendDefaultNullValue(ObjectNode jsonNode) { assert _defaultNullValue != null; if (!_defaultNullValue.equals(getDefaultNullValue(getFieldType(), _dataType, null))) { if (_defaultNullValue instanceof Number) { jsonNode.set("defaultNullValue", JsonUtils.objectToJsonNode(_defaultNullValue)); } else { jsonNode.put("defaultNullValue", getStringValue(_defaultNullValue)); } } }
public void setDefaultNullValue(@Nullable Object defaultNullValue) { if (defaultNullValue != null) { _stringDefaultNullValue = getStringValue(defaultNullValue); } if (_dataType != null) { _defaultNullValue = getDefaultNullValue(getFieldType(), _dataType, _stringDefaultNullValue); } }
public void setDataType(DataType dataType) { _dataType = dataType.getStoredType(); _defaultNullValue = getDefaultNullValue(getFieldType(), _dataType, _stringDefaultNullValue); }
public static Object extractValue(@Nullable JsonNode jsonValue, FieldSpec fieldSpec) { if (fieldSpec.isSingleValueField()) { if (jsonValue != null && !jsonValue.isNull()) { return extractSingleValue(jsonValue, fieldSpec.getDataType()); } else { return fieldSpec.getDefaultNullValue(); } } else { if (jsonValue != null && !jsonValue.isNull()) { if (jsonValue.isArray()) { int numValues = jsonValue.size(); if (numValues != 0) { Object[] values = new Object[numValues]; for (int i = 0; i < numValues; i++) { values[i] = extractSingleValue(jsonValue.get(i), fieldSpec.getDataType()); } return values; } else { return new Object[]{fieldSpec.getDefaultNullValue()}; } } else { return new Object[]{extractSingleValue(jsonValue, fieldSpec.getDataType())}; } } else { return new Object[]{fieldSpec.getDefaultNullValue()}; } } }
/** * Transform a single-value Avro value into an object in Pinot format. */ public static Object transformAvroValueToObject(Object avroValue, FieldSpec fieldSpec) { if (avroValue == null) { return fieldSpec.getDefaultNullValue(); } if (avroValue instanceof GenericData.Record) { return transformAvroValueToObject(((GenericData.Record) avroValue).get(0), fieldSpec); } if (fieldSpec.getDataType() == FieldSpec.DataType.STRING) { return avroValue.toString(); } else if (fieldSpec.getDataType() == FieldSpec.DataType.BYTES && avroValue instanceof ByteBuffer) { // Avro ByteBuffer maps to byte[]. ByteBuffer byteBuffer = (ByteBuffer) avroValue; // Assumes byte-buffer is ready to read. Also, avoid getting underlying array, as it may be over-sized. byte[] bytes = new byte[byteBuffer.remaining()]; byteBuffer.get(bytes); return bytes; } return avroValue; }
Object defaultNullValue = fieldSpec.getDefaultNullValue(); if (fieldSpec.isSingleValueField()) { value = defaultNullValue;
value = entry.toString(); } else { value = fieldSpec.getDefaultNullValue().toString();
/** * Transform an Avro array into an object array in Pinot format. */ public static Object[] transformAvroArrayToObjectArray(GenericData.Array avroArray, FieldSpec fieldSpec) { if (avroArray == null || avroArray.size() == 0) { return new Object[]{fieldSpec.getDefaultNullValue()}; } int numValues = avroArray.size(); Object[] objects = new Object[numValues]; for (int i = 0; i < numValues; i++) { objects[i] = transformAvroValueToObject(avroArray.get(i), fieldSpec); } return objects; } }
_numDimensions++; _dimensionNames.add(dimensionName); _dimensionStarValues.add(fieldSpec.getDefaultNullValue()); _dimensionDictionaries.add(HashBiMap.create());
boolean isSingleValueInSchema = fieldSpecInSchema.isSingleValueField(); String defaultValueInMetadata = columnMetadata.getDefaultNullValueString(); String defaultValueInSchema = fieldSpecInSchema.getDefaultNullValue().toString(); if (dataTypeInMetadata != dataTypeInSchema || isSingleValueInMetadata != isSingleValueInSchema || !defaultValueInSchema.equals(defaultValueInMetadata)) {
@Test public void testByteType() throws DecoderException, IOException { Schema expectedSchema = new Schema(); byte[] expectedEmptyDefault = new byte[0]; byte[] expectedNonEmptyDefault = Hex.decodeHex("abcd1234".toCharArray()); expectedSchema.setSchemaName("test"); expectedSchema.addField(new MetricFieldSpec("noDefault", FieldSpec.DataType.BYTES)); expectedSchema.addField(new MetricFieldSpec("emptyDefault", FieldSpec.DataType.BYTES, expectedEmptyDefault)); expectedSchema.addField(new MetricFieldSpec("nonEmptyDefault", FieldSpec.DataType.BYTES, expectedNonEmptyDefault)); // Ensure that schema can be serialized and de-serialized (ie byte[] converted to String and back). String jsonSchema = expectedSchema.getJSONSchema(); Schema actualSchema = Schema.fromString(jsonSchema); Assert.assertEquals(actualSchema.getFieldSpecFor("noDefault").getDefaultNullValue(), expectedEmptyDefault); Assert.assertEquals(actualSchema.getFieldSpecFor("emptyDefault").getDefaultNullValue(), expectedEmptyDefault); Assert.assertEquals(actualSchema.getFieldSpecFor("nonEmptyDefault").getDefaultNullValue(), expectedNonEmptyDefault); Assert.assertEquals(actualSchema, expectedSchema); Assert.assertEquals(actualSchema.hashCode(), expectedSchema.hashCode()); } }
/** * Complete the stats gathering process and store the stats information in indexCreationInfoMap. */ void buildIndexCreationInfo() throws Exception { for (FieldSpec spec : dataSchema.getAllFieldSpecs()) { String column = spec.getName(); // Skip adding virtual columns, so that they don't get an on-disk representation if (dataSchema.isVirtualColumn(column)) { continue; } ColumnStatistics columnProfile = segmentStats.getColumnProfileFor(column); indexCreationInfoMap.put(column, new ColumnIndexCreationInfo(columnProfile, true/*createDictionary*/, ForwardIndexType.FIXED_BIT_COMPRESSED, InvertedIndexType.ROARING_BITMAPS, false/*isAutoGenerated*/, dataSchema.getFieldSpecFor(column).getDefaultNullValue())); } segmentIndexCreationInfo.setTotalDocs(totalDocs); segmentIndexCreationInfo.setTotalRawDocs(totalRawDocs); segmentIndexCreationInfo.setTotalAggDocs(totalAggDocs); segmentIndexCreationInfo.setStarTreeEnabled(createStarTree); }
final int totalAggDocs = totalDocs - totalRawDocs; final FieldSpec.DataType dataType = fieldSpec.getDataType(); final Object defaultValue = fieldSpec.getDefaultNullValue(); final boolean isSingleValue = fieldSpec.isSingleValueField(); final int maxNumberOfMultiValueElements = isSingleValue ? 0 : 1;
Assert.assertEquals(fieldSpec1.toString(), fieldSpec2.toString()); Assert.assertEquals(fieldSpec1.hashCode(), fieldSpec2.hashCode()); Assert.assertEquals(fieldSpec1.getDefaultNullValue(), "false"); Assert.assertEquals(fieldSpec1.toString(), fieldSpec2.toString()); Assert.assertEquals(fieldSpec1.hashCode(), fieldSpec2.hashCode()); Assert.assertEquals(fieldSpec1.getDefaultNullValue(), Integer.MIN_VALUE); Assert.assertEquals(fieldSpec1.toString(), fieldSpec2.toString()); Assert.assertEquals(fieldSpec1.hashCode(), fieldSpec2.hashCode()); Assert.assertEquals(fieldSpec1.getDefaultNullValue(), -0.1F); Assert.assertEquals(fieldSpec1.toString(), fieldSpec2.toString()); Assert.assertEquals(fieldSpec1.hashCode(), fieldSpec2.hashCode()); Assert.assertEquals(fieldSpec1.getDefaultNullValue(), 1L);
defaultNullValue = fieldSpec.getDefaultNullValue();