private static void collect(String[] row, int[] pi, Set<Array<String>> satisfyingHostRecords) { // TODO when go beyond IN_THRESHOLD, only keep min/max is enough String[] rec = new String[pi.length]; for (int i = 0; i < pi.length; i++) { rec[i] = row[pi[i]]; } satisfyingHostRecords.add(new Array<String>(rec)); }
@Override public int compareTo(Array<T> other) { return compare(this.data, other.data, null); }
private static TableRecordInfoDigest createDigest(int nColumns, boolean[] isMetric, String[] dataTypes, Dictionary<?>[] dictionaries) { int[] dictMaxIds = new int[nColumns]; int[] lengths = new int[nColumns]; final boolean emptyDictionary = Array.isEmpty(dictionaries); for (int i = 0; i < nColumns; ++i) { if (isMetric[i]) {
lengths[i] = FixedLenMeasureCodec.get(DataType.getType(tblColRef.getColumnDesc().getDatatype())).getLength(); } else { if (Array.isEmpty(dictionaryMap)) { final DataType dataType = DataType.getType(tblColRef.getColumnDesc().getDatatype()); if (dataType.isNumberFamily()) {
@SuppressWarnings("unchecked") private void initRow(String[] cols, int[] keyIndex) { T[] value = convertRow(cols); T[] keyCols = (T[]) java.lang.reflect.Array.newInstance(getType(), keyIndex.length); for (int i = 0; i < keyCols.length; i++) keyCols[i] = value[keyIndex[i]]; Array<T> key = new Array<T>(keyCols); if (data.containsKey(key)) throw new IllegalStateException("The table: " + tableDesc.getName() + " Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value)); data.put(key, value); }
@Override public int compareTo(Array<T> other) { return compare(this.data, other.data, null); }
Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols); List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray); if (infoList == null) {
@Override public int compareTo(Array<T> other) { return compare(this.data, other.data, null); }
@Test public void testGet() throws Exception { int getNum = 3000; List<String[]> keys = Lists.newArrayList(); for (int i = 0; i < getNum; i++) { String[] keyi = new String[] { "keyyyyy" + random.nextInt(sourceRowNum) }; keys.add(keyi); } long start = System.currentTimeMillis(); for (int i = 0; i < getNum; i++) { String[] row = lookupTable.getRow(new Array<>(keys.get(i))); if (row == null) { System.out.println("null value for key:" + Arrays.toString(keys.get(i))); } } long take = System.currentTimeMillis() - start; System.out.println("muliti get " + getNum + " rows, take " + take + " ms"); }
@Override public int compareTo(Array<T> other) { return compare(this.data, other.data, null); }
ILookupTable lookupTable = getAndAddLookupTable(cubeSeg, deriveInfo.join); int[] derivedColIdx = initDerivedColIdx(); Array<String> lookupKey = new Array<String>(new String[hostTmpIdx.length]);
private static void collect(String[] row, int[] pi, Set<Array<String>> satisfyingHostRecords) { // TODO when go beyond IN_THRESHOLD, only keep min/max is enough String[] rec = new String[pi.length]; for (int i = 0; i < pi.length; i++) { rec[i] = row[pi[i]]; } satisfyingHostRecords.add(new Array<String>(rec)); }
private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) { if (hostCols.length == 0 || derivedCols.length == 0) throw new IllegalStateException("host/derived columns must not be empty"); Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols); List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray); if (infoList == null) { hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>()); } infoList.add(new DeriveInfo(type, dimension, derivedCols, false)); for (int i = 0; i < derivedCols.length; i++) { TblColRef derivedCol = derivedCols[i]; boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne)); } }
private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) { if (hostCols.length == 0 || derivedCols.length == 0) throw new IllegalStateException("host/derived columns must not be empty"); Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols); List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray); if (infoList == null) { hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>()); } infoList.add(new DeriveInfo(type, dimension, derivedCols, false)); for (int i = 0; i < derivedCols.length; i++) { TblColRef derivedCol = derivedCols[i]; boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne)); } }
@SuppressWarnings("unchecked") private void initRow(String[] cols, int[] keyIndex) { T[] value = convertRow(cols); T[] keyCols = (T[]) java.lang.reflect.Array.newInstance(value[0].getClass(), keyIndex.length); for (int i = 0; i < keyCols.length; i++) keyCols[i] = value[keyIndex[i]]; Array<T> key = new Array<T>(keyCols); if (data.containsKey(key)) throw new IllegalStateException("Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value)); data.put(key, value); }
public LookupFiller(int[] hostIndex, LookupStringTable lookupTable, DeriveInfo deriveInfo, String[] derivedFieldNames) { this.hostIndex = hostIndex; this.hostLen = hostIndex.length; this.lookupKey = new Array<String>(new String[hostLen]); this.lookupTable = lookupTable; this.derivedIndex = new int[deriveInfo.columns.length]; this.derivedLen = derivedIndex.length; this.derivedFieldNames = derivedFieldNames; for (int i = 0; i < derivedLen; i++) { derivedIndex[i] = deriveInfo.columns[i].getColumn().getZeroBasedIndex(); } }
rawKey[i] = new ByteArray(trimSplitBuffer(splitBuffers[tableJoin.fkIndice.get(i)])); Array<ByteArray> key = new Array<ByteArray>(rawKey); ByteArray[] dimRow = dimTbl.getRow(key); if (dimRow == null) {
private boolean matchAllCompositeKeys(TreeMap<String, String> lookupCol2FactTableCol, LinkedList<String> columnValues) { KylinConfig config = KylinConfig.getInstanceFromEnv(); for (String lookupTable : lookupTableKeys.keySet()) { if (lookupTableKeys.get(lookupTable).size() == 1) continue; String[] comboKey = new String[lookupTableKeys.get(lookupTable).size()]; int index = 0; for (String column : lookupTableKeys.get(lookupTable)) { String key = lookupTable + "/" + column; String factTableCol = lookupCol2FactTableCol.get(key); int cardinal = MetadataManager.getInstance(config).getTableDesc(factTableName).findColumnByName(factTableCol).getZeroBasedIndex(); comboKey[index] = columnValues.get(cardinal); index++; } Array<String> wrap = new Array<String>(comboKey); if (!lookupTableCompositeKeyValues.get(lookupTable).contains(wrap)) { // System.out.println("Try " + wrap + " Failed, continue..."); return false; } } return true; }
Array<String> wrap = new Array<String>(comboKeys); if (lookupTableCompositeKeyValues.get(lookupTableName).contains(wrap)) { throw new Exception("The composite key already exist in the lookup table");
@Test public void testBasic() throws Exception { TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES"); TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings"); LookupBytesTable lookup; System.out.println("============================================================================"); File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv"); lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new FileTable("file://" + f.getAbsolutePath(), 10)); lookup.dump(); System.out.println("============================================================================"); f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv"); lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id" }, new FileTable("file://" + f.getAbsolutePath(), 36)); lookup.dump(); System.out.println("============================================================================"); ByteArray k1 = new ByteArray(Bytes.toBytes("533")); ByteArray k2 = new ByteArray(Bytes.toBytes("0")); Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2 }); System.out.println(lookup.getRow(key)); } }