private static DimensionEncoding[] getDimensionEncodings(FunctionDesc function, List<TblColRef> literalCols, Map<TblColRef, Dictionary<String>> dictionaryMap) { final DimensionEncoding[] dimensionEncodings = new DimensionEncoding[literalCols.size()]; for (int i = 0; i < literalCols.size(); i++) { TblColRef colRef = literalCols.get(i); Pair<String, String> topNEncoding = TopNMeasureType.getEncoding(function, colRef); String encoding = topNEncoding.getFirst(); String encodingVersionStr = topNEncoding.getSecond(); if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) { dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef)); } else { int encodingVersion = 1; if (!StringUtils.isEmpty(encodingVersionStr)) { try { encodingVersion = Integer.parseInt(encodingVersionStr); } catch (NumberFormatException e) { throw new RuntimeException(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName() + " has to be an integer"); } } Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding); String encodingName = (String) encodingConf[0]; String[] encodingArgs = (String[]) encodingConf[1]; encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, literalCols.get(i).getType()); dimensionEncodings[i] = DimensionEncodingFactory.create(encodingName, encodingArgs, encodingVersion); } } return dimensionEncodings; }
/** * Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs. */ public static DimensionEncoding create(String encodingName, String[] args, int version) { if (factoryMap == null) initFactoryMap(); DimensionEncodingFactory factory = factoryMap.get(Pair.newPair(encodingName, version)); if (factory == null) { throw new IllegalArgumentException("Unknown dimension encoding name " + encodingName // + " (note '" + DictionaryDimEnc.ENCODING_NAME + "' is not handled by factory)"); } return factory.createDimensionEncoding(encodingName, args); }
@RequestMapping(value = "validEncodings", method = { RequestMethod.GET }, produces = { "application/json" }) @ResponseBody public Map<String, Integer> getValidEncodings() { Map<String, Integer> encodings; try { encodings = DimensionEncodingFactory.getValidEncodings(); } catch (Exception e) { logger.error("Error when getting valid encodings", e); return Maps.newHashMap(); } return encodings; }
try { DimensionEncodingFactory factory = (DimensionEncodingFactory) ClassUtil.newInstance(clsName); map.put(Pair.newPair(factory.getSupportedEncodingName(), factory.getCurrentVersion()), factory); } catch (Exception ex) { logger.error("Failed to init dimension encoding factory " + clsName, ex);
public static boolean isValidEncoding(final String encodingName) { if (factoryMap == null) initFactoryMap(); // note dictionary is a special case return DictionaryDimEnc.ENCODING_NAME.equals(encodingName) || // Iterables.any(factoryMap.keySet(), new Predicate<Pair<String, Integer>>() { @Override public boolean apply(@Nullable Pair<String, Integer> input) { return input.getFirst().equals(encodingName); } }); }
public void init(int index, CubeDesc cubeDesc) { bitIndex = index; colRef = cubeDesc.getModel().findColumn(column); column = colRef.getIdentity(); Preconditions.checkArgument(colRef != null, "Cannot find rowkey column %s in cube %s", column, cubeDesc); Preconditions.checkState(StringUtils.isNotEmpty(this.encoding)); Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding); encodingName = (String) encodingConf[0]; encodingArgs = (String[]) encodingConf[1]; if (!DimensionEncodingFactory.isValidEncoding(this.encodingName)) throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); // convert date/time dictionary on date/time column to DimensionEncoding implicitly // however date/time dictionary on varchar column is still required DataType type = colRef.getType(); if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) { if (type.isDate()) { encoding = encodingName = DateDimEnc.ENCODING_NAME; } if (type.isTimeFamily()) { encoding = encodingName = TimeDimEnc.ENCODING_NAME; } } encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type); if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily())) { logger.warn(colRef + " type is " + type + " and cannot apply fixed_length encoding"); } }
try { DimensionEncodingFactory factory = (DimensionEncodingFactory) ClassUtil.newInstance(clsName); map.put(Pair.newPair(factory.getSupportedEncodingName(), factory.getCurrentVersion()), factory); } catch (Exception ex) { logger.error("Failed to init dimension encoding factory " + clsName, ex);
public static Map<String, Integer> getValidEncodings() { if (factoryMap == null) initFactoryMap(); Map<String, Integer> result = Maps.newTreeMap(); for (Pair<String, Integer> p : factoryMap.keySet()) { if (result.containsKey(p.getFirst())) { if (result.get(p.getFirst()) > p.getSecond()) { continue;//skip small versions } } result.put(p.getFirst(), p.getSecond()); } result.put(DictionaryDimEnc.ENCODING_NAME, 1); return result; }
public void init(int index, CubeDesc cubeDesc) { bitIndex = index; colRef = cubeDesc.getModel().findColumn(column); column = colRef.getIdentity(); Preconditions.checkArgument(colRef != null, "Cannot find rowkey column %s in cube %s", column, cubeDesc); Preconditions.checkState(StringUtils.isNotEmpty(this.encoding)); Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding); encodingName = (String) encodingConf[0]; encodingArgs = (String[]) encodingConf[1]; if (!DimensionEncodingFactory.isValidEncoding(this.encodingName)) throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'"); // convert date/time dictionary on date/time column to DimensionEncoding implicitly // however date/time dictionary on varchar column is still required DataType type = colRef.getType(); if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) { if (type.isDate()) { encoding = encodingName = DateDimEnc.ENCODING_NAME; } if (type.isTimeFamily()) { encoding = encodingName = TimeDimEnc.ENCODING_NAME; } } encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type); if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily())) { logger.warn(colRef + " type is " + type + " and cannot apply fixed_length encoding"); } }
@Override public DimensionEncoding get(TblColRef col) { DimensionEncoding result = encMap.get(col); if (result == null) { RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col); if (colDesc.isUsingDictionary()) { // special dictionary encoding Dictionary<String> dict = getDictionary(col); if (dict == null) { logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg); result = new FixedLenDimEnc(0); } else { result = new DictionaryDimEnc(dict); } } else { // normal case result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion()); } encMap.put(col, result); } return result; }
/** * Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs. */ public static DimensionEncoding create(String encodingName, String[] args, int version) { if (factoryMap == null) initFactoryMap(); DimensionEncodingFactory factory = factoryMap.get(Pair.newPair(encodingName, version)); if (factory == null) { throw new IllegalArgumentException("Unknown dimension encoding name " + encodingName // + " (note '" + DictionaryDimEnc.ENCODING_NAME + "' is not handled by factory)"); } return factory.createDimensionEncoding(encodingName, args); }
public static boolean isValidEncoding(final String encodingName) { if (factoryMap == null) initFactoryMap(); // note dictionary is a special case return DictionaryDimEnc.ENCODING_NAME.equals(encodingName) || // Iterables.any(factoryMap.keySet(), new Predicate<Pair<String, Integer>>() { @Override public boolean apply(@Nullable Pair<String, Integer> input) { return input.getFirst().equals(encodingName); } }); }
DimensionEncoding dimensionEncoding = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion); keyLength += dimensionEncoding.getLengthOfEncoding();
public static Map<String, Integer> getValidEncodings() { if (factoryMap == null) initFactoryMap(); Map<String, Integer> result = Maps.newTreeMap(); for (Pair<String, Integer> p : factoryMap.keySet()) { if (result.containsKey(p.getFirst())) { if (result.get(p.getFirst()) > p.getSecond()) { continue;//skip small versions } } result.put(p.getFirst(), p.getSecond()); } result.put(DictionaryDimEnc.ENCODING_NAME, 1); return result; }
private static DimensionEncoding[] getDimensionEncodings(FunctionDesc function, List<TblColRef> literalCols, Map<TblColRef, Dictionary<String>> dictionaryMap) { final DimensionEncoding[] dimensionEncodings = new DimensionEncoding[literalCols.size()]; for (int i = 0; i < literalCols.size(); i++) { TblColRef colRef = literalCols.get(i); Pair<String, String> topNEncoding = TopNMeasureType.getEncoding(function, colRef); String encoding = topNEncoding.getFirst(); String encodingVersionStr = topNEncoding.getSecond(); if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) { dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef)); } else { int encodingVersion = 1; if (!StringUtils.isEmpty(encodingVersionStr)) { try { encodingVersion = Integer.parseInt(encodingVersionStr); } catch (NumberFormatException e) { throw new RuntimeException(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName() + " has to be an integer"); } } Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding); String encodingName = (String) encodingConf[0]; String[] encodingArgs = (String[]) encodingConf[1]; encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, literalCols.get(i).getType()); dimensionEncodings[i] = DimensionEncodingFactory.create(encodingName, encodingArgs, encodingVersion); } } return dimensionEncodings; }
@Override public DimensionEncoding get(TblColRef col) { DimensionEncoding result = encMap.get(col); if (result == null) { RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col); if (colDesc.isUsingDictionary()) { // special dictionary encoding Dictionary<String> dict = getDictionary(col); if (dict == null) { logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg); result = new FixedLenDimEnc(0); } else { result = new DictionaryDimEnc(dict); } } else { // normal case result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion()); } encMap.put(col, result); } return result; }
DimensionEncoding dimensionEncoding = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion); keyLength += dimensionEncoding.getLengthOfEncoding();