private GridTable newGridTableByCuboidID(long cuboidID) throws IOException { GTInfo info = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, cuboidID), new CubeDimEncMap(cubeDesc, dictionaryMap) ); // Below several store implementation are very similar in performance. The ConcurrentDiskStore is the simplest. // MemDiskStore store = new MemDiskStore(info, memBudget == null ? MemoryBudgetController.ZERO_BUDGET : memBudget); // MemDiskStore store = new MemDiskStore(info, MemoryBudgetController.ZERO_BUDGET); IGTStore store = new ConcurrentDiskStore(info); GridTable gridTable = new GridTable(info, store); return gridTable; }
static GTBuilder rebuild(GridTable table) throws IOException { GTBuilder builder = table.rebuild(); for (GTRecord rec : UnitTestSupport.mockupData(table.getInfo(), 10)) { builder.write(rec); } builder.close(); System.out.println("Written Row Count: " + builder.getWrittenRowCount()); return builder; }
protected void outputCuboid(long cuboidId, GridTable gridTable, ICuboidWriter output) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { output.write(cuboidId, record); } scanner.close(); logger.debug("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); }
private static GridTable newTable(GTInfo info) throws IOException { GTSimpleMemStore store = new GTSimpleMemStore(info); GridTable table = new GridTable(info, store); GTRecord record = new GTRecord(info); Random rand = new Random(); GTBuilder builder = table.rebuild(); expUserRet.clear(); Map<String, List<BigDecimal>> contents = Maps.newHashMap(); for (String date : dateList) { for (String user : userList) { List<BigDecimal> innerList = contents.get(user); if (innerList == null) { innerList = Lists.newArrayList(); contents.put(user, innerList); } BigDecimal value = priceList.get(rand.nextInt(priceList.size())); innerList.add(value); builder.write(record.setValues(date, user, value, new BigDecimal(0))); } } for (String user : contents.keySet()) { BigDecimal sum = new BigDecimal(0); for (BigDecimal innerValue : contents.get(user)) { sum = sum.add(innerValue); } expUserRet.put(user, sum); } builder.close(); return table; }
public static GridTable newTestTable() throws IOException { GTInfo info = newInfo(); GTSimpleMemStore store = new GTSimpleMemStore(info); GridTable table = new GridTable(info, store); GTRecord r = new GTRecord(table.getInfo()); GTBuilder builder = table.rebuild(); builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5"))); builder.close(); return table; }
private void verifyWriteAndRead(GridTable table) throws IOException { GTInfo info = table.getInfo(); GTBuilder builder = table.rebuild(); for (GTRecord r : data) { builder.write(r); } builder.close(); IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int i = 0; for (GTRecord r : scanner) { assertEquals(data.get(i++), r); } scanner.close(); } }
@Test public void verifyConvertFilterConstants1() { GTInfo info = table.getInfo(); TableDesc extTable = TableDesc.mockup("ext"); TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp"); TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer"); CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10"); LogicalTupleFilter filter = and(fComp1, fComp2); List<TblColRef> colMapping = Lists.newArrayList(); colMapping.add(extColA); colMapping.add(extColB); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString()); }
private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException { System.out.println(req); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) { System.out.println(r); if (verifyRows == null || i >= verifyRows.length) { Assert.fail(); } assertEquals(verifyRows[i], r.toString()); i++; } scanner.close(); }
static void rebuildViaAppend(GridTable table) throws IOException { List<GTRecord> data = UnitTestSupport.mockupData(table.getInfo(), 10); GTBuilder builder; int i = 0; builder = table.append(); builder.write(data.get(i++)); builder.write(data.get(i++)); builder.write(data.get(i++)); builder.write(data.get(i++)); builder.close(); System.out.println("Written Row Count: " + builder.getWrittenRowCount()); builder = table.append(); builder.write(data.get(i++)); builder.write(data.get(i++)); builder.write(data.get(i++)); builder.close(); System.out.println("Written Row Count: " + builder.getWrittenRowCount()); builder = table.append(); builder.write(data.get(i++)); builder.write(data.get(i++)); builder.close(); System.out.println("Written Row Count: " + builder.getWrittenRowCount()); builder = table.append(); builder.write(data.get(i++)); builder.close(); System.out.println("Written Row Count: " + builder.getWrittenRowCount()); }
public GTBuilder rebuild() throws IOException { return rebuild(-1); }
private void closeGirdTable(GridTable gridTable) { try { gridTable.close(); } catch (Throwable e) { logger.error("Error closing grid table " + gridTable, e); } } }
public GTBuilder append() throws IOException { return append(-1); }
@Test public void verifyConvertFilterConstants4() { GTInfo info = table.getInfo(); TableDesc extTable = TableDesc.mockup("ext"); TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp"); TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer"); CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15"); LogicalTupleFilter filter = and(fComp1, fComp2); List<TblColRef> colMapping = Lists.newArrayList(); colMapping.add(extColA); colMapping.add(extColB); // $1 in ("9", "10", "15") has only "10" left TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]", newFilter.toString()); }
@SuppressWarnings("unused") private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter) throws IOException { long start = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) { i++; } scanner.close(); long end = System.currentTimeMillis(); System.out.println( (end - start) + "ms with filter cache enabled=" + FilterResultCache.DEFAULT_OPTION + ", " + i + " rows"); }
public GTBuilder rebuild() throws IOException { return rebuild(-1); }
private void closeGirdTables(List<SplitThread> splits) { for (SplitThread split : splits) { if (split.buildResult != null) { for (CuboidResult r : split.buildResult.values()) { try { r.table.close(); } catch (Throwable e) { logger.error("Error closing grid table " + r.table, e); } } } } }
public GTBuilder append() throws IOException { return append(-1); }
@Override public void write(long cuboidId, GridTable gridTable) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { write(cuboidId, record); } scanner.close(); logger.info("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); } }