static void setupTimeRange(Scan scan, List<IndexSearchCondition> conditions) throws IOException { long start = 0; long end = Long.MAX_VALUE; for (IndexSearchCondition sc : conditions) { long timestamp = getTimestampVal(sc); String comparisonOp = sc.getComparisonOp(); if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual".equals(comparisonOp)) { start = timestamp; end = timestamp + 1; } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan".equals(comparisonOp)) { end = timestamp; } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan" .equals(comparisonOp)) { start = timestamp; } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan" .equals(comparisonOp)) { start = timestamp + 1; } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan" .equals(comparisonOp)) { end = timestamp + 1; } else { throw new IOException(comparisonOp + " is not a supported comparison operator"); } } scan.setTimeRange(start, end); }
private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException { // create scan with any properties set from TableInputFormat Scan s = TableInputFormat.createScanFromConfiguration(conf); // Set Scan Versions if (conf.get(TableInputFormat.SCAN_MAXVERSIONS) == null) { // default to all versions unless explicitly set s.setMaxVersions(Integer.MAX_VALUE); } s.setCacheBlocks(false); // Set RowFilter or Prefix Filter if applicable. Filter rowFilter = getRowFilter(args); if (rowFilter!= null) { LOG.info("Setting Row Filter for counter."); s.setFilter(rowFilter); } // Set TimeRange if defined long timeRange[] = getTimeRange(args); if (timeRange != null) { LOG.info("Setting TimeRange for counter."); s.setTimeRange(timeRange[0], timeRange[1]); } return s; }
private ResultScanner scan(Table ht, byte[] cf, Integer[] rowIndexes, Integer[] columnIndexes, Long[] versions, int maxVersions) throws IOException { Arrays.asList(rowIndexes); byte startRow[] = Bytes.toBytes("row:" + Collections.min( Arrays.asList(rowIndexes))); byte endRow[] = Bytes.toBytes("row:" + Collections.max( Arrays.asList(rowIndexes))+1); Scan scan = new Scan(startRow, endRow); for (Integer colIdx: columnIndexes) { byte column[] = Bytes.toBytes("column:" + colIdx); scan.addColumn(cf, column); } scan.setMaxVersions(maxVersions); scan.setTimeRange(Collections.min(Arrays.asList(versions)), Collections.max(Arrays.asList(versions))+1); ResultScanner scanner = ht.getScanner(scan); return scanner; }
@Override public int scannerOpenWithStopTs(ByteBuffer tableName, ByteBuffer startRow, ByteBuffer stopRow, List<ByteBuffer> columns, long timestamp, Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException { Table table = null; try { table = getTable(tableName); Scan scan = new Scan(getBytes(startRow), getBytes(stopRow)); addAttributes(scan, attributes); scan.setTimeRange(0, timestamp); if (columns != null && !columns.isEmpty()) { for (ByteBuffer column : columns) { byte [][] famQf = CellUtil.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } scan.setTimeRange(0, timestamp); return addScanner(table.getScanner(scan), false); } catch (IOException e) { LOG.warn(e.getMessage(), e); throw getIOError(e); } finally{ closeTable(table); } }
Scan initScan() throws IOException { Scan scan = new Scan(); scan.setCacheBlocks(false); if (startTime != 0 || endTime != 0) { scan.setTimeRange(startTime, endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime); } if (scanBatch > 0) { scan.setBatch(scanBatch); } if (versions >= 0) { scan.setMaxVersions(versions); } if (!isTableStartRow(startRow)) { scan.setStartRow(startRow); } if (!isTableEndRow(stopRow)) { scan.setStopRow(stopRow); } if(families != null) { for(String fam : families.split(",")) { scan.addFamily(Bytes.toBytes(fam)); } } return scan; }
private KeyIterator executeKeySliceQuery(@Nullable byte[] startKey, @Nullable byte[] endKey, FilterList filters, @Nullable SliceQuery columnSlice) throws BackendException { Scan scan = new Scan().addFamily(columnFamilyBytes); try { scan.setTimeRange(0, Long.MAX_VALUE); } catch (IOException e) { throw new PermanentBackendException(e); } if (startKey != null) scan.setStartRow(startKey); if (endKey != null) scan.setStopRow(endKey); if (columnSlice != null) { filters.addFilter(getFilter(columnSlice)); } TableMask table = null; try { table = cnx.getTable(tableName); return new RowIterator(table, table.getScanner(scan.setFilter(filters)), columnFamilyBytes); } catch (IOException e) { IOUtils.closeQuietly(table); throw new PermanentBackendException(e); } }
private void runTestOnTable() throws IOException, InterruptedException, ClassNotFoundException { Job job = null; try { job = new Job(UTIL.getConfiguration(), "test123"); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); Scan scan = new Scan(); scan.addColumn(FAMILY_NAME, COLUMN_NAME); scan.setTimeRange(MINSTAMP, MAXSTAMP); scan.setMaxVersions(); TableMapReduceUtil.initTableMapperJob(TABLE_NAME, scan, ProcessTimeRangeMapper.class, Text.class, Text.class, job); job.waitForCompletion(true); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (job != null) { FileUtil.fullyDelete( new File(job.getConfiguration().get("hadoop.tmp.dir"))); } } }
@Override public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow, List<ByteBuffer> columns, long timestamp, Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException { Table table = null; try { table = getTable(tableName); Scan scan = new Scan(getBytes(startRow)); addAttributes(scan, attributes); scan.setTimeRange(0, timestamp); if (columns != null && !columns.isEmpty()) { for (ByteBuffer column : columns) { byte [][] famQf = CellUtil.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]); } } } return addScanner(table.getScanner(scan), false); } catch (IOException e) { LOG.warn(e.getMessage(), e); throw getIOError(e); } finally{ closeTable(table); } }
private void scanVersionRangeAndVerify(Table ht, byte [] row, byte [] family, byte [] qualifier, long [] stamps, byte [][] values, int start, int end) throws IOException { Scan scan = new Scan(row); scan.addColumn(family, qualifier); scan.setMaxVersions(Integer.MAX_VALUE); scan.setTimeRange(stamps[start], stamps[end]+1); Result result = getSingleScanResult(ht, scan); assertNResult(result, row, family, qualifier, stamps, values, start, end); }
scan.setCacheBlocks(false); scan.setCaching(2000); scan.setTimeRange(0, Long.MAX_VALUE); scan.setMaxVersions(1);
scan.setCacheBlocks(false); scan.setCaching(2000); scan.setTimeRange(0, Long.MAX_VALUE); scan.setMaxVersions(1);
scan.setCacheBlocks(false); scan.setCaching(2000); scan.setTimeRange(0, Long.MAX_VALUE); scan.setMaxVersions(1);
private void scanVersionRangeAndVerifyGreaterThan(Table ht, byte [] row, byte [] family, byte [] qualifier, long [] stamps, byte [][] values, int start, int end) throws IOException { Scan scan = new Scan(row); scan.addColumn(family, qualifier); scan.setMaxVersions(Integer.MAX_VALUE); scan.setTimeRange(stamps[start+1], Long.MAX_VALUE); Result result = getSingleScanResult(ht, scan); assertNResult(result, row, family, qualifier, stamps, values, start+1, end); }
public static int assertScanContentTimestamp(final Table in, final long ts) throws IOException { Scan scan = new Scan(HConstants.EMPTY_START_ROW); scan.addFamily(FAMILY_NAME); scan.setTimeRange(0, ts); ResultScanner scanner = in.getScanner(scan); int count = 0; try { // TODO FIX // HStoreKey key = new HStoreKey(); // TreeMap<byte [], Cell>value = // new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR); // while (scanner.next(key, value)) { // assertTrue(key.getTimestamp() <= ts); // // Content matches the key or HConstants.LATEST_TIMESTAMP. // // (Key does not match content if we 'put' with LATEST_TIMESTAMP). // long l = Bytes.toLong(value.get(COLUMN).getValue()); // assertTrue(key.getTimestamp() == l || // HConstants.LATEST_TIMESTAMP == l); // count++; // value.clear(); // } } finally { scanner.close(); } return count; }
protected ResultScanner getResults(final Table table, final Collection<Column> columns, final Filter filter, final long minTime, List<String> authorizations) throws IOException { // Create a new scan. We will set the min timerange as the latest timestamp that // we have seen so far. The minimum timestamp is inclusive, so we will get duplicates. // We will record any cells that have the latest timestamp, so that when we scan again, // we know to throw away those duplicates. final Scan scan = new Scan(); scan.setTimeRange(minTime, Long.MAX_VALUE); if (authorizations != null && authorizations.size() > 0) { scan.setAuthorizations(new Authorizations(authorizations)); } if (filter != null) { scan.setFilter(filter); } if (columns != null) { for (Column col : columns) { if (col.getQualifier() == null) { scan.addFamily(col.getFamily()); } else { scan.addColumn(col.getFamily(), col.getQualifier()); } } } return table.getScanner(scan); }
protected ResultScanner getResults(final Table table, final Collection<Column> columns, final Filter filter, final long minTime, List<String> authorizations) throws IOException { // Create a new scan. We will set the min timerange as the latest timestamp that // we have seen so far. The minimum timestamp is inclusive, so we will get duplicates. // We will record any cells that have the latest timestamp, so that when we scan again, // we know to throw away those duplicates. final Scan scan = new Scan(); scan.setTimeRange(minTime, Long.MAX_VALUE); if (authorizations != null && authorizations.size() > 0) { scan.setAuthorizations(new Authorizations(authorizations)); } if (filter != null) { scan.setFilter(filter); } if (columns != null) { for (Column col : columns) { if (col.getQualifier() == null) { scan.addFamily(col.getFamily()); } else { scan.addColumn(col.getFamily(), col.getQualifier()); } } } return table.getScanner(scan); }
scan.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime()); scan.setMaxVersions(rowspec.getMaxVersions()); if (filter != null) {
@Override public void preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan) throws IOException { Region region = c.getEnvironment().getRegion(); TableName tableName = region.getTableDescriptor().getTableName(); Long ttl = this.ttls.get(tableName); if (ttl != null) { scan.setTimeRange(EnvironmentEdgeManager.currentTime() - ttl, scan.getTimeRange().getMax()); } Integer version = this.versions.get(tableName); if (version != null) { scan.readVersions(version); } } }
s.setTimeRange(0, ts+1); InternalScanner scanner = region.getScanner(s); List<Cell> kvs = new ArrayList<>(); s.setTimeRange(0, ts+2); scanner = region.getScanner(s); kvs = new ArrayList<>();
scan.setTimeRange(ts, ts+1); scan.setMaxVersions(maxVersions);