public boolean isInnerColumn() { return InnerDataTypeEnum.contains(getDatatype()); }
@Override public boolean moveNext() { while (currentDict == null || !currentDict.hasNext()) { if (!iterator.hasNext()) { return false; } final Dictionary<String> dict = iterator.next(); currentDict = dict.enumeratorValues().iterator(); } current[dictColIdx] = Tuple.convertOptiqCellValue(currentDict.next(), dictCol.getDatatype()); return true; }
public static long getTs(ITuple row, TblColRef partitionCol) { //ts column type differentiate if (partitionCol.getDatatype().equals("date")) { return epicDaysToMillis(Integer.parseInt(row.getValue(partitionCol).toString())); } else { return Long.parseLong(row.getValue(partitionCol).toString()); } }
private DictionaryInfo createDictionaryInfo(TblColRef col, IReadableTable inpTable) throws IOException { TableSignature inputSig = inpTable.getSignature(); if (inputSig == null) // table does not exists throw new IllegalStateException("Input table does not exist: " + inpTable); DictionaryInfo dictInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype(), inputSig); return dictInfo; }
public static String generateCreateTableStatement(IJoinedFlatTableDesc flatDesc, String storageDfsDir, String storageFormat, String fieldDelimiter) { StringBuilder ddl = new StringBuilder(); ddl.append("CREATE EXTERNAL TABLE IF NOT EXISTS " + flatDesc.getTableName() + "\n"); ddl.append("(" + "\n"); for (int i = 0; i < flatDesc.getAllColumns().size(); i++) { TblColRef col = flatDesc.getAllColumns().get(i); if (i > 0) { ddl.append(","); } ddl.append(colName(col, flatDesc.useAlias()) + " " + getHiveDataType(col.getDatatype()) + "\n"); } ddl.append(")" + "\n"); if (TEXTFILE.equals(storageFormat)) { ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '" + fieldDelimiter + "'\n"); } ddl.append("STORED AS " + storageFormat + "\n"); ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n"); ddl.append("ALTER TABLE " + flatDesc.getTableName() + " SET TBLPROPERTIES('auto.purge'='true');\n"); return ddl.toString(); }
@SuppressWarnings("unchecked") public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { final TblColRef tblColRef = entry.getKey(); final Dictionary<String> dictionary = entry.getValue(); IReadableTable.TableSignature signature = new IReadableTable.TableSignature(); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath(String.format(Locale.ROOT, "streaming_%s_%s", startOffset, endOffset)); signature.setSize(endOffset - startOffset); DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature); logger.info("writing dictionary for TblColRef:" + tblColRef.toString()); DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig()); try { DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo); cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath()); realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject()); } catch (IOException e) { throw new RuntimeException("error save dictionary for column:" + tblColRef, e); } } return realDictMap; }
if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) { logger.warn("PK " + dimTable + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with FK " + fkTable + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
@Override public void serialize(CoprocessorRowType o, ByteBuffer out) { int n = o.columns.length; BytesUtil.writeVInt(o.columns.length, out); BytesUtil.writeVInt(o.bodyOffset, out); for (int i = 0; i < n; i++) { BytesUtil.writeAsciiString(o.columns[i].getTable(), out); BytesUtil.writeAsciiString(o.columns[i].getName(), out); BytesUtil.writeAsciiString(o.columns[i].getDatatype(), out); BytesUtil.writeVInt(o.columnSizes[i], out); } }
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); mos = new MultipleOutputs(context); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); CubeDesc cubeDesc = cube.getDescriptor(); List<TblColRef> uhcColumns = cubeDesc.getAllUHCColumns(); int taskId = context.getTaskAttemptID().getTaskID().getId(); col = uhcColumns.get(taskId); logger.info("column name: " + col.getIdentity()); if (cube.getDescriptor().getShardByColumns().contains(col)) { //for ShardByColumns builder = DictionaryGenerator.newDictionaryBuilder(col.getType()); builder.init(null, 0, null); } else { //for GlobalDictionaryColumns String hdfsDir = conf.get(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR); DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype()); String builderClass = cubeDesc.getDictionaryBuilderClass(col); builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass); builder.init(dictionaryInfo, 0, hdfsDir); } }
public String getDataType() { return colRef.getDatatype(); }
@Override public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { TableRef tableRef = columnRef.getTableRef(); if (tableRef == null) { // un-qualified column String table = columnRef.getTable(); BytesUtil.writeUTFString(table, buffer); String columnId = columnRef.getColumnDesc().getId(); BytesUtil.writeUTFString(columnId, buffer); String columnName = columnRef.getName(); BytesUtil.writeUTFString(columnName, buffer); String dataType = columnRef.getDatatype(); BytesUtil.writeUTFString(dataType, buffer); } else { // qualified column (from model) BytesUtil.writeUTFString(_QUALIFIED_, buffer); String model = tableRef.getModel().getName(); BytesUtil.writeUTFString(model, buffer); String alias = tableRef.getAlias(); BytesUtil.writeUTFString(alias, buffer); String col = columnRef.getName(); BytesUtil.writeUTFString(col, buffer); } }
public boolean isDerivedDataType() { return InnerDataTypeEnum.DERIVED.getDataType().equals(getDatatype()); }
@Override public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) { TableRef tableRef = columnRef.getTableRef(); if (tableRef == null) { // un-qualified column String table = columnRef.getTable(); BytesUtil.writeUTFString(table, buffer); String columnId = columnRef.getColumnDesc().getId(); BytesUtil.writeUTFString(columnId, buffer); String columnName = columnRef.getName(); BytesUtil.writeUTFString(columnName, buffer); String dataType = columnRef.getDatatype(); BytesUtil.writeUTFString(dataType, buffer); } else { // qualified column (from model) BytesUtil.writeUTFString(_QUALIFIED_, buffer); String model = tableRef.getModel().getName(); BytesUtil.writeUTFString(model, buffer); String alias = tableRef.getAlias(); BytesUtil.writeUTFString(alias, buffer); String col = columnRef.getName(); BytesUtil.writeUTFString(col, buffer); } }
public boolean isInnerColumn() { return InnerDataTypeEnum.contains(getDatatype()); }
@Override public byte[] serialize() { ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); String table = columnRef.getTable(); BytesUtil.writeUTFString(table, buffer); String columnName = columnRef.getName(); BytesUtil.writeUTFString(columnName, buffer); String dataType = columnRef.getDatatype(); BytesUtil.writeUTFString(dataType, buffer); byte[] result = new byte[buffer.position()]; System.arraycopy(buffer.array(), 0, result, 0, buffer.position()); return result; }
public DictionaryInfo buildDictionary(DataModelDesc model, String dict, TblColRef col, String factColumnsPath) throws IOException { logger.info("building dictionary for " + col); Object[] tmp = decideSourceData(model, dict, col, factColumnsPath); String srcTable = (String) tmp[0]; String srcCol = (String) tmp[1]; int srcColIdx = (Integer) tmp[2]; ReadableTable inpTable = (ReadableTable) tmp[3]; DictionaryInfo dictInfo = new DictionaryInfo(srcTable, srcCol, srcColIdx, col.getDatatype(), inpTable.getSignature(), inpTable.getColumnDelimeter()); String dupDict = checkDupByInfo(dictInfo); if (dupDict != null) { logger.info("Identical dictionary input " + dictInfo.getInput() + ", reuse existing dictionary at " + dupDict); return getDictionaryInfo(dupDict); } Dictionary<?> dictionary = DictionaryGenerator.buildDictionary(dictInfo, inpTable); return trySaveNewDict(dictionary, dictInfo); }
if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) { throw new IllegalStateException("Primary key " + lookup.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.getFactTable() + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
if (!fkCols[i].getDatatype().equals(pkCols[i].getDatatype())) { addError("Primary key " + dim.getTable() + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with Foreign key " + this.factTable + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
protected Collection<Tuple> generateTuple(int number, List<TblColRef> columns, int[] matches) { Collection<Tuple> tuples = new ArrayList<Tuple>(number); TupleInfo info = new TupleInfo(); for (int i = 0; i < columns.size(); i++) { TblColRef column = columns.get(i); info.setField(column.getName(), column, column.getDatatype(), i); } int allMatches = 0; Random rand = new Random(); for (int i = 0; i < number; i++) { Tuple t = new Tuple(info); boolean isFullMatch = true; for (int k = 0; k < columns.size(); k++) { TblColRef column = columns.get(k); int index = Math.abs(rand.nextInt()) % SAMPLE_DATA[k].length; t.setDimensionValue(column.getName(), SAMPLE_DATA[k][index]); if (index == 0) { matches[k]++; } else { isFullMatch = false; } } if (isFullMatch) { allMatches++; } tuples.add(t); } matches[2] = allMatches; return tuples; }
@Override public boolean moveNext() { while (currentDict == null || !currentDict.hasNext()) { if (!iterator.hasNext()) { return false; } final Dictionary<String> dict = iterator.next(); currentDict = dict.enumeratorValues().iterator(); } current[dictColIdx] = Tuple.convertOptiqCellValue(currentDict.next(), dictCol.getDatatype()); return true; }