@Override protected void initialize(JobContext job) throws IOException { Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create( job.getConfiguration())); TableName tableName = TableName.valueOf("exampleTable"); // mandatory initializeTable(connection, tableName); byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"), Bytes.toBytes("columnB") }; //optional Scan scan = new Scan(); for (byte[] family : inputColumns) { scan.addFamily(family); } Filter exampleFilter = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("aa.*")); scan.setFilter(exampleFilter); setScan(scan); }
@Test public void testRowFilter() throws IOException { String filterString = "RowFilter ( =, 'binary:regionse')"; RowFilter rowFilter = doTestFilter(filterString, RowFilter.class); assertEquals(CompareOperator.EQUAL, rowFilter.getCompareOperator()); assertTrue(rowFilter.getComparator() instanceof BinaryComparator); BinaryComparator binaryComparator = (BinaryComparator) rowFilter.getComparator(); assertEquals("regionse", new String(binaryComparator.getValue(), StandardCharsets.UTF_8)); }
public HBaseScanSpec parseTree() { HBaseScanSpec parsedSpec = le.accept(this, null); if (parsedSpec != null) { parsedSpec = mergeScanSpecs("booleanAnd", this.groupScan.getHBaseScanSpec(), parsedSpec); /* * If RowFilter is THE filter attached to the scan specification, * remove it since its effect is also achieved through startRow and stopRow. */ if (parsedSpec.filter instanceof RowFilter && ((RowFilter)parsedSpec.filter).getOperator() != CompareOp.NOT_EQUAL && ((RowFilter)parsedSpec.filter).getComparator() instanceof BinaryComparator) { parsedSpec.filter = null; } } return parsedSpec; }
@Override public boolean filterRowKey(Cell firstRowCell) { if (compareRow(getCompareOperator(), this.comparator, firstRowCell)) { this.filterOutRow = true; } return this.filterOutRow; }
@Test public void testCompareFilter() throws Exception { Filter f = new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); byte [] bytes = f.toByteArray(); Filter ff = RowFilter.parseFrom(bytes); assertNotNull(ff); }
private Scan createScanWithRowFilter(final byte [] key, final byte [] startRow, CompareOperator op) { // Make sure key is of some substance... non-null and > than first key. assertTrue(key != null && key.length > 0 && Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0); LOG.info("Key=" + Bytes.toString(key)); Scan s = startRow == null? new Scan(): new Scan(startRow); Filter f = new RowFilter(op, new BinaryComparator(key)); f = new WhileMatchFilter(f); s.setFilter(f); return s; }
Filter f = new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); Scan s = new Scan(); s.setFilter(f); verifyScanNoEarlyOut(s, expectedRows, expectedKeys); f = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("testRow.+-2")); s = new Scan(); f = new RowFilter(CompareOperator.LESS, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); f = new RowFilter(CompareOperator.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); f = new RowFilter(CompareOperator.NOT_EQUAL, f = new RowFilter(CompareOperator.GREATER_OR_EQUAL, f = new RowFilter(CompareOperator.GREATER, f = new RowFilter(CompareOperator.NOT_EQUAL, f = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"));
RegexStringComparator regexStringComparator = new RegexStringComparator(rowkeyRegex); regexStringComparator.setCharset(Charset.forName("ISO-8859-1")); RowFilter filter = new RowFilter(CompareOp.EQUAL, regexStringComparator); FilterList filterList = new FilterList(); filterList.addFilter(filter); Scan s1 = new Scan(); s1.setStartRow(stopKey); s1.setStopRow(startKey); s1.setFilter(filterList);
@Test public void testJira6912() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10); List<Put> puts = new ArrayList<Put>(); for (int i=0;i !=100; i++){ Put put = new Put(Bytes.toBytes(i)); put.addColumn(FAMILY, FAMILY, Bytes.toBytes(i)); puts.add(put); } foo.put(puts); // If i comment this out it works TEST_UTIL.flush(); Scan scan = new Scan(); scan.setStartRow(Bytes.toBytes(1)); scan.setStopRow(Bytes.toBytes(3)); scan.addColumn(FAMILY, FAMILY); scan.setFilter(new RowFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1)))); ResultScanner scanner = foo.getScanner(scan); Result[] bar = scanner.next(100); assertEquals(1, bar.length); }
@Test @Ignore("TODO: intentionally disabled?") public void testNestedFilterListWithSCVF() throws IOException { byte[] columnStatus = Bytes.toBytes("S"); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); htd.addFamily(new HColumnDescriptor(FAMILIES[0])); Filter rowFilter = new RowFilter(CompareOperator.GREATER,new BinaryComparator(Bytes.toBytes("row4"))); Scan s1 = new Scan(); s1.setFilter(rowFilter); InternalScanner scanner = testRegion.getScanner(s1); List<Cell> results = new ArrayList<>(); FilterList subFilterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); Filter subFilter1 = new RowFilter(CompareOperator.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes("row4"))); subFilterList.addFilter(subFilter1); Filter subFilter2 = new SingleColumnValueFilter(FAMILIES[0], columnStatus, CompareOperator.EQUAL, Bytes.toBytes(0)); subFilterList.addFilter(subFilter2); s1 = new Scan(); s1.setFilter(subFilterList); scanner = testRegion.getScanner(s1);
Scan scan = new Scan().setFilter( new ColumnValueFilter(FAMILIES[0], QUALIFIERS_FIVE[0], CompareOperator.EQUAL, VALUES[1])); KeyValue[] expectedEquals = verifyScanFull(scan, expectedEquals); scan.setFilter( new ColumnValueFilter(FAMILIES[0], QUALIFIERS_FIVE[0], CompareOperator.GREATER, VALUES[0])); KeyValue[] expectedGreater = andFilters.add( new ColumnValueFilter(FAMILIES[1], QUALIFIERS_FIVE[1], CompareOperator.NOT_EQUAL, VALUES[1])); andFilters.add(new RowFilter(CompareOperator.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("testRow")))); scan.setFilter(new FilterList(Operator.MUST_PASS_ALL, andFilters)); KeyValue[] expectedMustPassAll = { new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]), Scan anotherScan = new Scan().addColumn(FAMILIES[1], QUALIFIERS_FIVE[1]) .setFilter(new FilterList(Operator.MUST_PASS_ONE, orFilters)); KeyValue[] expectedValues = { new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
@Test public void testFilterList() throws IOException { // Test getting a single row, single key using Row, Qualifier, and Value // regular expression and substring filters // Use must pass all List<Filter> filters = new ArrayList<>(); filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One"))); Filter f = new FilterList(Operator.MUST_PASS_ALL, filters); Scan s = new Scan(); s.addFamily(FAMILIES[0]); s.setFilter(f); KeyValue [] kvs = { new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) }; verifyScanFull(s, kvs); // Test getting everything with a MUST_PASS_ONE filter including row, qf, val // regular expression and substring filters filters.clear(); filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+Two.+"))); filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2"))); filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One"))); f = new FilterList(Operator.MUST_PASS_ONE, filters); s = new Scan(); s.setFilter(f); verifyScanNoEarlyOut(s, this.numRows, this.colsPerRow); }
protected void initialize(JobConf job, String table) throws IOException { Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(job)); TableName tableName = TableName.valueOf(table); // mandatory initializeTable(connection, tableName); byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"), Bytes.toBytes("columnB") }; // mandatory setInputColumns(inputColumns); Filter exampleFilter = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("aa.*")); // optional setRowFilter(exampleFilter); }
Filter filter1 = new RowFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryComparator(Bytes.toBytes("row-03"))); filters.add(filter1); Filter filter2 = new RowFilter(CompareFilter.CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes("row-06"))); filters.add(filter2); new RegexStringComparator("col-0[03]")); filters.add(filter3); FilterList filterList1 = new FilterList(filters); Scan scan = new Scan(); scan.setFilter(filterList1); ResultScanner scanner1 = table.getScanner(scan); for (Cell cell : result.rawCells()) { System.out.println("Cell: " + cell + ", Value: " + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); FilterList filterList2 = new FilterList( FilterList.Operator.MUST_PASS_ONE, filters); scan.setFilter(filterList2); ResultScanner scanner2 = table.getScanner(scan);
public void testRowsSeenMetric(Scan baseScan) throws Exception { Scan scan; scan = new Scan(baseScan); testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, NUM_ROWS); scan = new Scan(baseScan); scan.withStartRow(ROWS[0]); scan.withStopRow(ROWS[i + 1]); testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, i + 1); new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("xyz"))); scan = new Scan(baseScan); scan.setFilter(filter);
byte[] schemaName = table.getSchemaName().getBytes(); byte[] tableName = table.getTableName().getBytes(); Scan scan = new Scan(); byte[] startRow = ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY); byte[] stopRow = ByteUtil.nextKey(startRow); scan.setStartRow(startRow); scan.setStopRow(stopRow); SingleColumnValueFilter filter2 = new SingleColumnValueFilter(TABLE_FAMILY_BYTES, BASE_TABLE_NAME_BYTES, EQUAL, tableName); filter2.setFilterIfMissing(true); BinaryComparator comparator = new BinaryComparator(ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY, schemaName, QueryConstants.SEPARATOR_BYTE_ARRAY, tableName)); RowFilter filter3 = new RowFilter(CompareOp.NOT_EQUAL,comparator); Filter filter = new FilterList(filter1,filter2,filter3); scan.setFilter(filter); RegionScanner scanner = region.getScanner(scan);
new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("xyz"))); testRowsFilteredMetric(baseScan, filter, ROWS.length); filter = new ColumnPrefixFilter(Bytes.toBytes("xyz")); testRowsFilteredMetric(baseScan, filter, ROWS.length); filters.add(new RowFilter(CompareOperator.EQUAL, new BinaryComparator(ROWS[0]))); filters.add(new RowFilter(CompareOperator.EQUAL, new BinaryComparator(ROWS[3]))); int numberOfMatchingRowFilters = filters.size(); filter = new FilterList(Operator.MUST_PASS_ONE, filters); testRowsFilteredMetric(baseScan, filter, ROWS.length - numberOfMatchingRowFilters); filters.clear(); filter = new FilterList(Operator.MUST_PASS_ONE, filters); testRowsFilteredMetric(baseScan, filter, ROWS.length);
Scan scan = new Scan(); scan.setRowPrefixFilter(getVariantFileMetadataRowKeyPrefix(studyId)); if (fileIds != null && !fileIds.isEmpty()) { FilterList filterList = new FilterList(); for (Integer fileId : fileIds) { filterList.addFilter(new RowFilter(CompareOp.EQUAL, new BinaryComparator(getVariantFileMetadataRowKey(studyId, fileId)))); scan.setFilter(filterList); } else { scan.setFilter(
FilterList nsFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL); nsFilters.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); nsFilters.addFilter(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator( getSettingsQualifierRegexForUserNamespace(filter.getNamespaceFilter()), 0))); userFilters.addFilter(nsFilters); FilterList tableFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL); tableFilters.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); tableFilters.addFilter(new QualifierFilter(CompareOperator.EQUAL, userFilters.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); filterList.addFilter(userFilters); } else if (StringUtils.isNotEmpty(filter.getTableFilter())) { filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getTableRowKeyRegex(filter.getTableFilter()), 0))); } else if (StringUtils.isNotEmpty(filter.getNamespaceFilter())) { filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getNamespaceRowKeyRegex(filter.getNamespaceFilter()), 0))); } else if (StringUtils.isNotEmpty(filter.getRegionServerFilter())) { filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator( getRegionServerRowKeyRegex(filter.getRegionServerFilter()), 0)));