protected void outputCuboid(long cuboidId, GridTable gridTable, ICuboidWriter output) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { output.write(cuboidId, record); } scanner.close(); logger.debug("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); }
@Override public void write(long cuboidId, GridTable gridTable) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { write(cuboidId, record); } scanner.close(); logger.info("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); } }
public boolean fetchNext() throws IOException { if (recordIterator == null) { if (cuboidIterator.hasNext()) { CuboidResult cuboid = cuboidIterator.next(); currentCuboidId = cuboid.cuboidId; scanner = cuboid.table.scan(new GTScanRequestBuilder().setInfo(cuboid.table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); recordIterator = scanner.iterator(); } else { return false; } } if (recordIterator.hasNext()) { currentRecord = recordIterator.next(); return true; } else { scanner.close(); recordIterator = null; return fetchNext(); } }
public boolean fetchNext() throws IOException { if (recordIterator == null) { currentCuboidId = splitResult.cuboidId; scanner = splitResult.table.scan(new GTScanRequestBuilder().setInfo(splitResult.table.getInfo()) .setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); recordIterator = scanner.iterator(); } if (recordIterator.hasNext()) { currentRecord = recordIterator.next(); return true; } else { scanner.close(); recordIterator = null; return false; } }
private void doScanAndVerify(GridTable table, GTScanRequest req, String... verifyRows) throws IOException { System.out.println(req); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) { System.out.println(r); if (verifyRows == null || i >= verifyRows.length) { Assert.fail(); } assertEquals(verifyRows[i], r.toString()); i++; } scanner.close(); }
@SuppressWarnings("unused") private void testFilterScannerPerfInner(GridTable table, GTInfo info, LogicalTupleFilter filter) throws IOException { long start = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setFilterPushDown(filter).createGTScanRequest(); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) { i++; } scanner.close(); long end = System.currentTimeMillis(); System.out.println( (end - start) + "ms with filter cache enabled=" + FilterResultCache.DEFAULT_OPTION + ", " + i + " rows"); }
private IGTScanner scan(GridTable table) throws IOException { GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = table.scan(req); for (GTRecord r : scanner) { Object[] v = r.getValues(); assertTrue(((String) v[0]).startsWith("2015-")); assertTrue(((String) v[2]).equals("Food")); assertTrue(((Long) v[3]).longValue() == 10); assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5); System.out.println(r); } scanner.close(); return scanner; }
public void run() { try { IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int i = 0; for (GTRecord r : scanner) { assertEquals(data.get(i++), r); } scanner.close(); } catch (Exception ex) { ex.printStackTrace(); } } };
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs) .setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }
private IGTScanner scanAndAggregate(GridTable table) throws IOException { GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[] { "count", "sum" }).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = table.scan(req); int i = 0; for (GTRecord r : scanner) {
private void verifyWriteAndRead(GridTable table) throws IOException { GTInfo info = table.getInfo(); GTBuilder builder = table.rebuild(); for (GTRecord r : data) { builder.write(r); } builder.close(); IGTScanner scanner = table.scan(new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); int i = 0; for (GTRecord r : scanner) { assertEquals(data.get(i++), r); } scanner.close(); } }
public static void prepareTestData() throws Exception { try { util.getHBaseAdmin().disableTable(TABLE); util.getHBaseAdmin().deleteTable(TABLE); } catch (Exception e) { // ignore table not found } Table table = util.createTable(TABLE, FAM); HRegionInfo hRegionInfo = new HRegionInfo(table.getName()); region = util.createLocalHRegion(hRegionInfo, table.getTableDescriptor()); gtInfo = newInfo(); GridTable gridTable = newTable(gtInfo); IGTScanner scanner = gridTable.scan(new GTScanRequestBuilder().setInfo(gtInfo).setRanges(null) .setDimensions(null).setFilterPushDown(null).createGTScanRequest()); for (GTRecord record : scanner) { byte[] value = record.exportColumns(gtInfo.getPrimaryKey()).toBytes(); byte[] key = new byte[RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN + value.length]; System.arraycopy(Bytes.toBytes(baseCuboid), 0, key, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); System.arraycopy(value, 0, key, RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN, value.length); Put put = new Put(key); put.addColumn(FAM, COL_M, record.exportColumns(gtInfo.getColumnBlock(1)).toBytes()); region.put(put); } }
protected void outputCuboid(long cuboidId, GridTable gridTable, ICuboidWriter output) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { output.write(cuboidId, record); } scanner.close(); logger.debug("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); }
@Override public void write(long cuboidId, GridTable gridTable) throws IOException { long startTime = System.currentTimeMillis(); GTScanRequest req = new GTScanRequestBuilder().setInfo(gridTable.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest(); IGTScanner scanner = gridTable.scan(req); for (GTRecord record : scanner) { write(cuboidId, record); } scanner.close(); logger.info("Cuboid " + cuboidId + " output takes " + (System.currentTimeMillis() - startTime) + "ms"); } }
public boolean fetchNext() throws IOException { if (recordIterator == null) { if (cuboidIterator.hasNext()) { CuboidResult cuboid = cuboidIterator.next(); currentCuboidId = cuboid.cuboidId; scanner = cuboid.table.scan(new GTScanRequestBuilder().setInfo(cuboid.table.getInfo()).setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); recordIterator = scanner.iterator(); } else { return false; } } if (recordIterator.hasNext()) { currentRecord = recordIterator.next(); return true; } else { scanner.close(); recordIterator = null; return fetchNext(); } }
public boolean fetchNext() throws IOException { if (recordIterator == null) { currentCuboidId = splitResult.cuboidId; scanner = splitResult.table.scan(new GTScanRequestBuilder().setInfo(splitResult.table.getInfo()) .setRanges(null).setDimensions(null).setFilterPushDown(null).createGTScanRequest()); recordIterator = scanner.iterator(); } if (recordIterator.hasNext()) { currentRecord = recordIterator.next(); return true; } else { scanner.close(); recordIterator = null; return false; } }
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null).setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }
private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { GTInfo info = gridTable.getInfo(); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(aggregationColumns).setAggrMetrics(measureColumns).setAggrMetricsFuncs(metricsAggrFuncs) .setFilterPushDown(null).createGTScanRequest(); GTAggregateScanner scanner = (GTAggregateScanner) gridTable.scan(req); // for child cuboid, some measures don't need aggregation. if (parentId != cuboidId) { boolean[] aggrMask = new boolean[measureDescs.length]; for (int i = 0; i < measureDescs.length; i++) { aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid(); if (!aggrMask[i]) { logger.info("{} doesn't need aggregation.", measureDescs[i]); } } scanner.setAggrMask(aggrMask); } return scanner; }