private ResultScanner buildScanner(String keyPrefix, String value, Table ht) throws IOException { // OurFilterList allFilters = new OurFilterList(); FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes .toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); // allFilters.addFilter(new // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return ht.getScanner(scan); }
private FilterList generateTimeFilterList(VisitFilter visitFilter) { FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); if (visitFilter.lastModStart >= 0) { // NOTE: Negative value does not work in its binary form SingleColumnValueFilter timeStartFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.GREATER_OR_EQUAL, Bytes.toBytes(visitFilter.lastModStart)); filterList.addFilter(timeStartFilter); } if (visitFilter.lastModEndExclusive != Long.MAX_VALUE) { SingleColumnValueFilter timeEndFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.LESS, Bytes.toBytes(visitFilter.lastModEndExclusive)); filterList.addFilter(timeEndFilter); } return filterList.getFilters().size() == 0 ? null : filterList; }
} break; case FilterList: this.op = ((FilterList)filter).getOperator().toString(); this.filters = new ArrayList<>(); for (Filter child: ((FilterList)filter).getFilters()) { this.filters.add(new FilterModel(child)); ((PrefixFilter)filter).getPrefix())); break; case FamilyFilter: case SingleColumnValueFilter: { SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter; this.family = Bytes.toString(Base64.getEncoder().encode(scvf.getFamily())); byte[] qualifier = scvf.getQualifier(); this.op = scvf.getOperator().toString();
@Override public int hashCode() { return Objects.hash(getOperator(), getFilters()); } }
private static void setRowPrefixFilter(Scan scan, String rowPrefixes) { if (rowPrefixes != null && !rowPrefixes.isEmpty()) { String[] rowPrefixArray = rowPrefixes.split(","); Arrays.sort(rowPrefixArray); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); for (String prefix : rowPrefixArray) { Filter filter = new PrefixFilter(Bytes.toBytes(prefix)); filterList.addFilter(filter); } scan.setFilter(filterList); byte[] startPrefixRow = Bytes.toBytes(rowPrefixArray[0]); byte[] lastPrefixRow = Bytes.toBytes(rowPrefixArray[rowPrefixArray.length -1]); setStartAndStopRows(scan, startPrefixRow, lastPrefixRow); } }
@Test public void testMultiRowRangeWithFilterListAndOperator() throws IOException { tableName = TableName.valueOf(name.getMethodName()); Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); generateRows(numRows, ht, family, qf, value); Scan scan = new Scan(); scan.setMaxVersions(); List<RowRange> ranges1 = new ArrayList<>(); ranges1.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(20), false)); ranges1.add(new RowRange(Bytes.toBytes(30), true, Bytes.toBytes(40), false)); ranges1.add(new RowRange(Bytes.toBytes(60), true, Bytes.toBytes(70), false)); MultiRowRangeFilter filter1 = new MultiRowRangeFilter(ranges1); List<RowRange> ranges2 = new ArrayList<>(); ranges2.add(new RowRange(Bytes.toBytes(20), true, Bytes.toBytes(40), false)); ranges2.add(new RowRange(Bytes.toBytes(80), true, Bytes.toBytes(90), false)); MultiRowRangeFilter filter2 = new MultiRowRangeFilter(ranges2); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); filterList.addFilter(filter1); filterList.addFilter(filter2); scan.setFilter(filterList); int resultsSize = getResultsSize(ht, scan); LOG.info("found " + resultsSize + " results"); List<Cell> results1 = getScanResult(Bytes.toBytes(30), Bytes.toBytes(40), ht); assertEquals(results1.size(), resultsSize); ht.close(); }
@Override boolean testRow(final int i) throws IOException { Scan scan = new Scan().withStartRow(getRandomRow(this.rand, opts.totalRows)) .setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) .setScanMetricsEnabled(true); FilterList list = new FilterList(); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(familyName, qualifier); list.addFilter(new FilterAllFilter()); list.addFilter(new WhileMatchFilter(new PageFilter(120))); scan.setFilter(list); ResultScanner s = this.table.getScanner(scan); try { for (Result rr; (rr = s.next()) != null;) { updateValueSize(rr); updateScanMetrics(s.getScanMetrics()); s.close();
tableScan.setBatch(batchSize); tableScan.setMaxVersions(maxVersions); tableScan.setTimeRange(startTime, endTime); if (!startRow.isEmpty()) { tableScan.setStartRow(Bytes.toBytes(startRow)); tableScan.setStopRow(Bytes.toBytes(endRow)); FilterList filterList = new FilterList(); if (StringUtils.isNotEmpty(paramFilter)) { ParseFilter pf = new ParseFilter(); Filter parsedParamFilter = pf.parseFilterString(paramFilter); if (parsedParamFilter != null) { filterList.addFilter(parsedParamFilter); filterList.addFilter(prefixFilter); if (filterList.size() > 0) { tableScan.setFilter(filterList); tableScan.setReversed(reversed); tableScan.setCacheBlocks(cacheBlocks); return new TableScanResource(hTable.getScanner(tableScan), userRequestedLimit); } catch (IOException exp) { servlet.getMetrics().incrementFailedScanRequests(1);
Table table = connection.getTable(TableName.valueOf("testtable")); Filter filter1 = new CustomFilter(Bytes.toBytes("val-05.05")); filters.add(filter1); Filter filter2 = new CustomFilter(Bytes.toBytes("val-02.07")); filters.add(filter2); Filter filter3 = new CustomFilter(Bytes.toBytes("val-09.01")); filters.add(filter3); FilterList filterList = new FilterList( FilterList.Operator.MUST_PASS_ONE, filters); Scan scan = new Scan(); scan.setFilter(filterList); ResultScanner scanner = table.getScanner(scan); scanner.close();
Table table = connection.getTable(TableName.valueOf("testtable")); Put put = new Put(Bytes.toBytes("row-5")) .addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col-6"), Bytes.toBytes("val-6")); table.put(put); put = new Put(Bytes.toBytes("row-8")) .addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col-6"), Bytes.toBytes("val-6")); table.put(put); Delete delete = new Delete(Bytes.toBytes("row-5")) .addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("col-3")); table.delete(delete); FilterList list = new FilterList(); list.addFilter(filter); list.addFilter(filter2); Scan scan = new Scan(); scan.setFilter(list); ResultScanner scanner = table.getScanner(scan); scanner.close(); table.close();
private void runScanner(Table hTable, int expectedSize, Filter... filters) throws IOException { String cf = "f"; Scan scan = new Scan(); scan.addFamily(cf.getBytes()); FilterList filterList = new FilterList(filters); scan.setFilter(filterList); ResultScanner scanner = hTable.getScanner(scan); List<Cell> results = new ArrayList<>(); Result result; long timeBeforeScan = System.currentTimeMillis(); while ((result = scanner.next()) != null) { for (Cell kv : result.listCells()) { LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: " + Bytes.toStringBinary(CellUtil.cloneQualifier(kv))); results.add(kv); } } long scanTime = System.currentTimeMillis() - timeBeforeScan; scanner.close(); LOG.info("scan time = " + scanTime + "ms"); LOG.info("found " + results.size() + " results"); assertEquals(expectedSize, results.size()); } }
protected Scan constructScan(byte[] valuePrefix) throws IOException { FilterList list = new FilterList(); Filter filter = new SingleColumnValueFilter(FAMILY_ZERO, COLUMN_ZERO, CompareOperator.EQUAL, new BinaryComparator(valuePrefix)); list.addFilter(filter); if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) .setScanMetricsEnabled(true); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(FAMILY_ZERO, qualifier); } } else { scan.addFamily(FAMILY_ZERO); } scan.setFilter(list); return scan; } }
@Test public void testFiltersWithOR() throws Exception { TableName tn = TableName.valueOf(name.getMethodName()); Table table = TEST_UTIL.createTable(tn, new String[] { "cf1", "cf2" }); byte[] CF1 = Bytes.toBytes("cf1"); byte[] CF2 = Bytes.toBytes("cf2"); Put put1 = new Put(Bytes.toBytes("0")); put1.addColumn(CF1, Bytes.toBytes("col_a"), Bytes.toBytes(0)); table.put(put1); Put put2 = new Put(Bytes.toBytes("0")); put2.addColumn(CF2, Bytes.toBytes("col_b"), Bytes.toBytes(0)); table.put(put2); FamilyFilter filterCF1 = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(CF1)); FamilyFilter filterCF2 = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(CF2)); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.addFilter(filterCF1); filterList.addFilter(filterCF2); Scan scan = new Scan(); scan.setFilter(filterList); ResultScanner scanner = table.getScanner(scan); LOG.info("Filter list: " + filterList); for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { Assert.assertEquals(2, rr.size()); } }
@Test public void testEmptyFilterList() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); Table table = TEST_UTIL.createTable(tableName, FAMILY); Put put = new Put(Bytes.toBytes("row")); put.addColumn(FAMILY, QUALIFIER, VALUE); table.put(put); Scan scan = new Scan(); scan.setFilter(new FilterList()); try (ResultScanner scanner = table.getScanner(scan)) { for (Result r : scanner) { scanResults.add(r); Get g = new Get(Bytes.toBytes("row")); g.setFilter(new FilterList()); Result getResult = table.get(g); Result scanResult = scanResults.get(0); assertEquals(scanResult.rawCells().length, getResult.rawCells().length); Cell scanCell = scanResult.rawCells()[i]; Cell getCell = getResult.rawCells()[i]; assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell), CellUtil.cloneRow(getCell))); assertEquals(0, Bytes.compareTo(CellUtil.cloneFamily(scanCell), CellUtil.cloneFamily(getCell))); assertEquals(0, Bytes.compareTo(CellUtil.cloneQualifier(scanCell), CellUtil.cloneQualifier(getCell)));
TableName tn = TableName.valueOf(name.getMethodName()); byte[] cf1 = Bytes.toBytes("f1"); byte[] row = Bytes.toBytes("row"); byte[] value = Bytes.toBytes("value"); String[] columns = new String[]{ "1544768273917010001_lt", for (int i = 0; i < columns.length; i++) { Put put = new Put(row).addColumn(cf1, Bytes.toBytes(columns[i]), value); table.put(put); Scan scan = new Scan(); scan.withStartRow(row).withStopRow(row, true) .setFilter(new FilterList(Operator.MUST_PASS_ONE, new ColumnPrefixFilter(Bytes.toBytes("1544770422942010001_")), new ColumnPrefixFilter(Bytes.toBytes("1544769883529010001_")))); ResultScanner scanner = table.getScanner(scan); Result result; int resultCount = 0; int cellCount = 0; while ((result = scanner.next()) != null) { cellCount += result.listCells().size(); resultCount++;
Scan scan = new Scan(); InternalScanner scanner = region.getScanner(scan); verifyCountAndOrder(scanner, ROWSIZE * QUALSIZE * 2, ROWSIZE, true); scan.setReversed(true); scanner = region.getScanner(scan); verifyCountAndOrder(scanner, ROWSIZE * QUALSIZE * 2, ROWSIZE, false); scan = new Scan(); scan.setReversed(true); scan.addFamily(FAMILYNAME); Filter filter = new SingleColumnValueFilter(FAMILYNAME, specifiedQualifiers[0], CompareOp.EQUAL, VALUES[valueNum]); scan.setFilter(filter); filter = new FilterList(Operator.MUST_PASS_ONE, scvFilter1, scvFilter2); scan.setFilter(filter); scanner = region.getScanner(scan); filter = new FilterList(Operator.MUST_PASS_ALL, scvFilter1, scvFilter2); expectedRowNum = 0; scan.setFilter(filter);
@Test public void testEmptyFilterListTransformCell() throws IOException { KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"), 1, Bytes.toBytes("value")); FilterList filterList = new FilterList(Operator.MUST_PASS_ALL); assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv)); assertEquals(kv, filterList.transformCell(kv)); filterList = new FilterList(Operator.MUST_PASS_ONE); assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv)); assertEquals(kv, filterList.transformCell(kv)); }
@Test public void testFilterListWithPrefixFilter() throws IOException { byte[] family = Bytes.toBytes("f1"); byte[] qualifier = Bytes.toBytes("q1"); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); htd.addFamily(new HColumnDescriptor(family)); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); Put p = new Put(Bytes.toBytes((char)('a'+i) + "row")); p.setDurability(Durability.SKIP_WAL); p.addColumn(family, qualifier, Bytes.toBytes(String.valueOf(111 + i))); PrefixFilter pf = new PrefixFilter(new byte[] {'b'}) ; SingleColumnValueFilter scvf = new SingleColumnValueFilter( family, qualifier, CompareOperator.EQUAL, Bytes.toBytes("113")); FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, pf, scvf); Scan s1 = new Scan(); s1.setFilter(filterList); InternalScanner scanner = testRegion.getScanner(s1); List<Cell> results = new ArrayList<>();
@Test public void testIndexesScanWithOneDeletedRow() throws IOException { byte[] family = Bytes.toBytes("family"); // Setting up region this.region = initHRegion(tableName, method, CONF, family); Put put = new Put(Bytes.toBytes(1L)); put.addColumn(family, qual1, 1L, Bytes.toBytes(1L)); region.put(put); region.flush(true); Delete delete = new Delete(Bytes.toBytes(1L), 1L); region.delete(delete); put = new Put(Bytes.toBytes(2L)); put.addColumn(family, qual1, 2L, Bytes.toBytes(2L)); region.put(put); Scan idxScan = new Scan(); idxScan.addFamily(family); idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList( new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL, new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L)))))); InternalScanner scanner = region.getScanner(idxScan); List<Cell> res = new ArrayList<>(); while (scanner.next(res)) { // Ignore res value. } assertEquals(1L, res.size()); }
private void testScanWithFilters(Connection connection, String tableName) throws IOException { createTable(thriftAdmin, tableName); try (Table table = connection.getTable(TableName.valueOf(tableName))){ FilterList filterList = new FilterList(); PrefixFilter prefixFilter = new PrefixFilter(Bytes.toBytes("testrow")); ColumnValueFilter columnValueFilter = new ColumnValueFilter(FAMILYA, QUALIFIER_1, CompareOperator.EQUAL, VALUE_1); filterList.addFilter(prefixFilter); filterList.addFilter(columnValueFilter); Scan scan = new Scan(); scan.setMaxVersions(2); ResultScanner scanner = table.getScanner(scan); Iterator<Result> iterator = scanner.iterator(); assertTrue(iterator.hasNext()); int counter = 0; while (iterator.hasNext()) { Result result = iterator.next(); counter += result.size(); } assertEquals(2, counter); } }