public GTBuilder rebuild() throws IOException { return rebuild(-1); }
private static GridTable newTable(GTInfo info) throws IOException { GTSimpleMemStore store = new GTSimpleMemStore(info); GridTable table = new GridTable(info, store); GTRecord record = new GTRecord(info); Random rand = new Random(); GTBuilder builder = table.rebuild(); expUserRet.clear(); Map<String, List<BigDecimal>> contents = Maps.newHashMap(); for (String date : dateList) { for (String user : userList) { List<BigDecimal> innerList = contents.get(user); if (innerList == null) { innerList = Lists.newArrayList(); contents.put(user, innerList); } BigDecimal value = priceList.get(rand.nextInt(priceList.size())); innerList.add(value); builder.write(record.setValues(date, user, value, new BigDecimal(0))); } } for (String user : contents.keySet()) { BigDecimal sum = new BigDecimal(0); for (BigDecimal innerValue : contents.get(user)) { sum = sum.add(innerValue); } expUserRet.put(user, sum); } builder.close(); return table; }
private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating cuboid {}", cuboidId); GTAggregateScanner scanner = prepareGTAggregationScanner(gridTable, parentId, cuboidId, aggregationColumns, measureColumns); GridTable newGridTable = newGridTableByCuboidID(cuboidId); GTBuilder builder = newGridTable.rebuild(); ImmutableBitSet allNeededColumns = aggregationColumns.or(measureColumns); GTRecord newRecord = new GTRecord(newGridTable.getInfo()); int count = 0; try { for (GTRecord record : scanner) { count++; for (int i = 0; i < allNeededColumns.trueBitCount(); i++) { int c = allNeededColumns.trueBitAt(i); newRecord.set(i, record.get(c)); } builder.write(newRecord); } } finally { scanner.close(); builder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", cuboidId, count, timeSpent); return updateCuboidResult(cuboidId, newGridTable, count, timeSpent, 0); }
GTBuilder builder = newGridTable.rebuild();
static GTBuilder rebuild(GridTable table) throws IOException { GTBuilder builder = table.rebuild(); for (GTRecord rec : UnitTestSupport.mockupData(table.getInfo(), 10)) { builder.write(rec); } builder.close(); System.out.println("Written Row Count: " + builder.getWrittenRowCount()); return builder; }
private void verifyWriteAndRead(final GridTable table, int readThreads) throws IOException, InterruptedException { GTBuilder builder = table.rebuild(); for (GTRecord r : data) { builder.write(r);
public static GridTable newTestTable() throws IOException { GTInfo info = newInfo(); GTSimpleMemStore store = new GTSimpleMemStore(info); GridTable table = new GridTable(info, store); GTRecord r = new GTRecord(table.getInfo()); GTBuilder builder = table.rebuild(); builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5"))); builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5"))); builder.close(); return table; }
GTBuilder builder = table.rebuild();
private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating base cuboid {}", baseCuboidId); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req); aggregationScanner.trackMemoryLevel(baseCuboidMemTracker); int count = 0; try { for (GTRecord r : aggregationScanner) { if (count == 0) { baseCuboidMemTracker.markHigh(); } baseBuilder.write(r); count++; } } finally { aggregationScanner.close(); baseBuilder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", baseCuboidId, count, timeSpent); int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB); logger.info("Wild estimate of base aggr cache is {} MB", mbEstimateBaseAggrCache); return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0, input.inputConverterUnit.ifChange()); }
sw.start(); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter<>(baseCuboid.getInfo(), input);
private void verifyWriteAndRead(GridTable table) throws IOException { GTInfo info = table.getInfo(); GTBuilder builder = table.rebuild(); for (GTRecord r : data) { builder.write(r); } builder.close(); IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int i = 0; for (GTRecord r : scanner) { assertEquals(data.get(i++), r); } scanner.close(); } }
public GTBuilder rebuild() throws IOException { return rebuild(-1); }
private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating cuboid {}", cuboidId); GTAggregateScanner scanner = prepareGTAggregationScanner(gridTable, parentId, cuboidId, aggregationColumns, measureColumns); GridTable newGridTable = newGridTableByCuboidID(cuboidId); GTBuilder builder = newGridTable.rebuild(); ImmutableBitSet allNeededColumns = aggregationColumns.or(measureColumns); GTRecord newRecord = new GTRecord(newGridTable.getInfo()); int count = 0; try { for (GTRecord record : scanner) { count++; for (int i = 0; i < allNeededColumns.trueBitCount(); i++) { int c = allNeededColumns.trueBitAt(i); newRecord.set(i, record.get(c)); } builder.write(newRecord); } } finally { scanner.close(); builder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", cuboidId, count, timeSpent); return updateCuboidResult(cuboidId, newGridTable, count, timeSpent, 0); }
GTBuilder builder = newGridTable.rebuild();
private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating base cuboid {}", baseCuboidId); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input); Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount); GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req); aggregationScanner.trackMemoryLevel(baseCuboidMemTracker); int count = 0; try { for (GTRecord r : aggregationScanner) { if (count == 0) { baseCuboidMemTracker.markHigh(); } baseBuilder.write(r); count++; } } finally { aggregationScanner.close(); baseBuilder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", baseCuboidId, count, timeSpent); int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB); logger.info("Wild estimate of base aggr cache is {} MB", mbEstimateBaseAggrCache); return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, 0, input.inputConverterUnit.ifChange()); }
sw.start(); GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId); GTBuilder baseBuilder = baseCuboid.rebuild(); IGTScanner baseInput = new InputConverter<>(baseCuboid.getInfo(), input);