public void setDatatype(String datatype) { //logger.info("setting datatype to " + datatype); this.datatype = datatype; type = DataType.getType(datatype); }
public List<String> getValidEncodings(DataType dataType) { Message msg = MsgPicker.getMsg(); if (dataType.isIntegerFamily()) { return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME); } else if (dataType.isNumberFamily()) { //numbers include integers return Lists.newArrayList(DictionaryDimEnc.ENCODING_NAME); } else if (dataType.isDateTimeFamily()) { return Lists.newArrayList(DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME); } else if (dataType.isStringFamily()) { return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, FixedLenDimEnc.ENCODING_NAME, // FixedLenHexDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME); } else { throw new BadRequestException(String.format(Locale.ROOT, msg.getVALID_ENCODING_NOT_AVAILABLE(), dataType)); } }
public static DataType getType(String type) { if (type == null) return null; DataType cached = CACHE.get(type); if (cached == null) { DataType dataType = new DataType(type); CACHE.put(type, dataType); cached = dataType; } return cached; }
public TopNCounterSerializer(DataType dataType) { this.precision = dataType.getPrecision(); this.scale = dataType.getScale(); if (scale < 0) { scale = DictionaryDimEnc.MAX_ENCODING_LENGTH; } }
static DataTypeOrder getInstance(DataType type) throws IllegalArgumentException { if (type.isStringFamily()) return STRING_ORDER; else if (type.isDateTimeFamily()) return DATETIME_ORDER; else if (type.isIntegerFamily()) return INTEGER_ORDER; else if (type.isFloat() || type.isDouble()) return DOUBLE_ORDER; else if (type.isDecimal()) return DECIMAL_ORDER; else if (type.isBoolean()) return BOOLEAN_ORDER; else throw new IllegalArgumentException("Unsupported data type " + type); }
public static String[] replaceEncodingArgs(String encoding, String[] encodingArgs, String encodingName, DataType type) { // https://issues.apache.org/jira/browse/KYLIN-2495 if (DateDimEnc.ENCODING_NAME.equals(encodingName)) { if (type.isIntegerFamily()) { if (encodingArgs.length != 0) { throw new IllegalArgumentException("Date encoding should not specify arguments: " + encoding); } return new String[] { type.toString() }; } } return encodingArgs; }
public static IFilterCodeSystem getFilterCodeSystem(DataType dataType) { if (dataType.isIntegerFamily()) { return codeSystemMap.get("integer"); } else if (dataType.isNumberFamily()) { return codeSystemMap.get("decimal"); } else if (dataType.isDateTimeFamily()) { return codeSystemMap.get("integer"); } else { return codeSystemMap.get("string"); } }
private String tsRangeToStr(long ts, PartitionDesc part) { String value; DataType partitionColType = part.getPartitionDateColumnRef().getType(); if (partitionColType.isDate()) { value = DateFormat.formatToDateStr(ts); } else if (partitionColType.isTimeFamily()) { value = DateFormat.formatToTimeWithoutMilliStr(ts); } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101 String partitionDateFormat = part.getPartitionDateFormat(); if (StringUtils.isEmpty(partitionDateFormat)) { value = "" + ts; } else { value = DateFormat.formatToDateStr(ts, partitionDateFormat); } } else { throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); } return value; }
DataType returnType = DataType.getType(measureDesc.getFunction().getReturnType()); DataType newReturnType = new DataType(returnType.getName(), returnType.getPrecision(), keyLength); measureDesc.getFunction().setReturnType(newReturnType.toString());
public static RelDataType createSqlType(RelDataTypeFactory typeFactory, DataType dataType, boolean isNullable) { SqlTypeName sqlTypeName = SQLTYPE_MAPPING.get(dataType.getName()); if (sqlTypeName == null) { for (String reg : REGEX_SQLTYPE_MAPPING.keySet()) { Pattern pattern = Pattern.compile(reg); if (pattern.matcher(dataType.getName()).matches()) { sqlTypeName = REGEX_SQLTYPE_MAPPING.get(reg); break; throw new IllegalArgumentException("Unrecognized data type " + dataType); int precision = dataType.getPrecision(); int scale = dataType.getScale(); String innerTypeName = dataType.getName().split("<|>")[1]; result = typeFactory.createArrayType(createSqlType(typeFactory, DataType.getType(innerTypeName), false), -1); } else if (precision >= 0 && scale >= 0)
@Override public void validate(FunctionDesc functionDesc) throws IllegalArgumentException { DataType rtype = dataType; if (funcName.equals(FunctionDesc.FUNC_SUM)) { if (rtype.isNumberFamily() == false) { throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY); } } else if (funcName.equals(FunctionDesc.FUNC_COUNT)) { if (rtype.isIntegerFamily() == false) { throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.INTEGER_FAMILY); } } else if (funcName.equals(FunctionDesc.FUNC_MAX) || funcName.equals(FunctionDesc.FUNC_MIN)) { if (rtype.isNumberFamily() == false) { throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY); } } else { KylinConfig config = KylinConfig.getInstanceFromEnv(); if (config.isQueryIgnoreUnknownFunction() == false) throw new IllegalArgumentException("Unrecognized function: [" + funcName + "]"); } }
@Override public MeasureIngester<?> newIngester() { if (dataType.isIntegerFamily()) return new LongIngester(); else if (dataType.isDecimal()) return new BigDecimalIngester(); else if (dataType.isNumberFamily()) return new DoubleIngester(); else throw new IllegalArgumentException("No ingester for aggregation type " + dataType); }
public void init(int index, CubeDesc cubeDesc) { bitIndex = index; colRef = cubeDesc.getModel().findColumn(column); column = colRef.getIdentity(); Preconditions.checkArgument(colRef != null, "Cannot find rowkey column %s in cube %s", column, cubeDesc); Preconditions.checkState(StringUtils.isNotEmpty(this.encoding)); Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding); encodingName = (String) encodingConf[0]; encodingArgs = (String[]) encodingConf[1]; if (!DimensionEncodingFactory.isValidEncoding(this.encodingName)) throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); // convert date/time dictionary on date/time column to DimensionEncoding implicitly // however date/time dictionary on varchar column is still required DataType type = colRef.getType(); if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) { if (type.isDate()) { encoding = encodingName = DateDimEnc.ENCODING_NAME; } if (type.isTimeFamily()) { encoding = encodingName = TimeDimEnc.ENCODING_NAME; } } encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type); if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily())) { logger.warn(colRef + " type is " + type + " and cannot apply fixed_length encoding"); } }
@Test public void testDataTypeOrder() { DataType intType = DataType.getType("integer"); DataTypeOrder dataTypeOrder = intType.getOrder(); Set<String> integers = Sets.newHashSet("100000", "2", "1000", "100", "77", "10", "9", "2000000", "-10000", "0"); Assert.assertEquals("2000000", dataTypeOrder.max(integers)); Assert.assertEquals("-10000", dataTypeOrder.min(integers)); DataType doubleType = DataType.getType("double"); dataTypeOrder = doubleType.getOrder(); Set<String> doubels = Sets.newHashSet("1.1", "-299.5", "100000", "1.000", "4.000000001", "0.00", "-1000000.231231", "8000000", "10", "10.00"); Assert.assertEquals("8000000", dataTypeOrder.max(doubels)); Assert.assertEquals("-1000000.231231", dataTypeOrder.min(doubels)); DataType datetimeType = DataType.getType("date"); dataTypeOrder = datetimeType.getOrder(); Set<String> datetimes = Sets.newHashSet("2010-01-02", "2888-08-09", "2018-05-26", "1527512082000", "2010-02-03 23:59:59", "2000-12-12 12:00:00", "1970-01-19 00:18:32", "1998-12-02", "2018-05-28 10:00:00.255", "1995-09-20 20:00:00.220"); Assert.assertEquals("2888-08-09", dataTypeOrder.max(datetimes)); Assert.assertEquals("1970-01-19 00:18:32", dataTypeOrder.min(datetimes)); DataType stringType = new DataType("varchar", 256, 10); dataTypeOrder = stringType.getOrder(); Set<String> strings = Sets.newHashSet(null, "", "ä¸å›½", "China No.1", "神兽麒麟", "Rocket", "Apache Kylin", "google", "NULL", "empty"); Assert.assertEquals("神兽麒麟", dataTypeOrder.max(strings)); Assert.assertEquals("", dataTypeOrder.min(strings)); } }
@Override protected void init() throws IOException { ColumnDesc[] cols = tableDesc.getColumns(); colIsDateTime = new boolean[cols.length]; colIsNumber = new boolean[cols.length]; for (int i = 0; i < cols.length; i++) { DataType t = cols[i].getType(); colIsDateTime[i] = t.isDateTimeFamily(); colIsNumber[i] = t.isNumberFamily(); } super.init(); }
public static IDictionaryBuilder newDictionaryBuilder(DataType dataType) { Preconditions.checkNotNull(dataType, "dataType cannot be null"); // build dict, case by data type IDictionaryBuilder builder; if (dataType.isDateTimeFamily()) { if (dataType.isDate()) builder = new DateDictBuilder(); else builder = new TimeDictBuilder(); } else { boolean useForest = KylinConfig.getInstanceFromEnv().isUseForestTrieDictionary(); if (dataType.isNumberFamily()) builder = useForest ? new NumberTrieDictForestBuilder() : new NumberTrieDictBuilder(); else builder = useForest ? new StringTrieDictForestBuilder() : new StringTrieDictBuilder(); } return builder; }
public int compare(String value1, String value2) { return getOrder().compare(value1, value2); }
@Test public void testComplexType() { DataType arrayInt = DataType.getType("array<int>"); DataType arrayString = DataType.getType("array<string>"); Assert.assertTrue(DataType.isComplexType(arrayInt)); Assert.assertTrue(DataType.isComplexType(arrayString)); }
public StringSerializer(DataType type) { this.type = type; // see serialize(): 2 byte length, rest is String.toBytes() this.maxLength = 2 + type.getPrecision(); }