private static int listSpans(Opts opts, ScannerOpts scanOpts) throws Exception { PrintStream out = System.out; long endTime = System.currentTimeMillis(); long startTime = endTime - opts.length; try (AccumuloClient client = opts.createClient()) { Scanner scanner = client.createScanner(opts.getTableName(), opts.auths); scanner.setBatchSize(scanOpts.scanBatchSize); Range range = new Range(new Text("start:" + Long.toHexString(startTime)), new Text("start:" + Long.toHexString(endTime))); scanner.setRange(range); out.println("Trace Day/Time (ms) Start"); for (Entry<Key,Value> entry : scanner) { RemoteSpan span = TraceFormatter.getRemoteSpan(entry); out.println(String.format("%016x %s %5d %s", span.traceId, TraceFormatter.formatDate(new Date(span.getStart())), span.stop - span.start, span.description)); } } return 0; }
@Override public Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); checkArgument(authorizations != null, "authorizations is null"); ensureOpen(); Scanner scanner = new ScannerImpl(this, getTableId(tableName), authorizations); Integer batchSize = ClientProperty.SCANNER_BATCH_SIZE.getInteger(getProperties()); if (batchSize != null) { scanner.setBatchSize(batchSize); } return scanner; }
public static Text findMax(Scanner scanner, Text start, boolean is, Text end, boolean ie) throws TableNotFoundException { scanner.setBatchSize(12); IteratorSetting cfg = new IteratorSetting(Integer.MAX_VALUE, SortedKeyIterator.class); scanner.addScanIterator(cfg); if (start == null) { start = new Text(); is = true; } if (end == null) end = findInitialEnd(scanner); return _findMax(scanner, start, is, end, ie); } }
private static int dumpTrace(Opts opts, ScannerOpts scanOpts) throws Exception { final PrintStream out = System.out; int count = 0; try (AccumuloClient client = opts.createClient()) { for (String traceId : opts.traceIds) { Scanner scanner = client.createScanner(opts.getTableName(), opts.auths); scanner.setBatchSize(scanOpts.scanBatchSize); Range range = new Range(new Text(traceId)); scanner.setRange(range); count = printTrace(scanner, out::println); } } return count > 0 ? 0 : 1; }
private Iterator<Entry<Key,Value>> newIterator(Range r) { synchronized (scanner) { scanner.enableIsolation(); scanner.setBatchSize(batchSize); scanner.setTimeout(timeout, TimeUnit.MILLISECONDS); scanner.setRange(r); scanner.setReadaheadThreshold(readaheadThreshold); setOptions((ScannerOptions) scanner, opts); return scanner.iterator(); // return new FaultyIterator(scanner.iterator()); } }
@Override public Iterator<Entry<Key,Value>> iterator() { smi.scanner.setBatchSize(size); smi.scanner.setTimeout(timeOut, TimeUnit.MILLISECONDS); smi.scanner.setBatchTimeout(batchTimeOut, TimeUnit.MILLISECONDS);
private int findMaxDepth(Scanner scanner) { // do binary search to find max depth int origBatchSize = scanner.getBatchSize(); scanner.setBatchSize(100); int depth = findMaxDepth(scanner, 0, 64, 999); scanner.setBatchSize(origBatchSize); return depth; }
/** * @param dataStore * @param query * @param scanner */ public AccumuloResult(DataStore<K,T> dataStore, Query<K,T> query, Scanner scanner) { super(dataStore, query); if (this.limit > 0) { scanner.setBatchSize((int) this.limit); } iterator = new RowIterator(scanner.iterator()); }
private static String findAStartRow(long min, long max, Scanner scanner, Random r) { byte[] scanStart = ContinuousIngest.genRow(min, max, r); scanner.setRange(new Range(new Text(scanStart), null)); scanner.setBatchSize(100); int count = 0; String pr = null; long t1 = System.currentTimeMillis(); for (Entry<Key,Value> entry : scanner) { validate(entry.getKey(), entry.getValue()); pr = getPrevRow(entry.getValue()); count++; if (pr != null) break; } long t2 = System.currentTimeMillis(); System.out.printf("FSR %d %s %d %d%n", t1, new String(scanStart, UTF_8), (t2 - t1), count); return pr; }
static Entry<Key,Value> findRandomDocument(State state, Environment env, String dataTableName, Random rand) throws Exception { Scanner scanner = env.getConnector().createScanner(dataTableName, Authorizations.EMPTY); scanner.setBatchSize(1); scanner.setRange(new Range(Integer.toString(rand.nextInt(0xfffffff), 16), null)); Iterator<Entry<Key,Value>> iter = scanner.iterator(); if (!iter.hasNext()) return null; return iter.next(); }
private static int listSpans(Opts opts, ScannerOpts scanOpts) throws Exception { PrintStream out = System.out; long endTime = System.currentTimeMillis(); long startTime = endTime - opts.length; Connector conn = opts.getConnector(); Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths); scanner.setBatchSize(scanOpts.scanBatchSize); Range range = new Range(new Text("start:" + Long.toHexString(startTime)), new Text("start:" + Long.toHexString(endTime))); scanner.setRange(range); out.println("Trace Day/Time (ms) Start"); for (Entry<Key,Value> entry : scanner) { RemoteSpan span = TraceFormatter.getRemoteSpan(entry); out.println(String.format("%016x %s %5d %s", span.traceId, TraceFormatter.formatDate(new Date(span.getStart())), span.stop - span.start, span.description)); } return 0; }
private static void readBack(Opts opts, ScannerOpts scanOpts, Connector conn, List<String> tableNames) throws Exception { int i = 0; for (String table : tableNames) { // wait for table to exist while (!conn.tableOperations().exists(table)) UtilWaitThread.sleep(100); Scanner scanner = conn.createScanner(table, opts.auths); scanner.setBatchSize(scanOpts.scanBatchSize); int count = i; for (Entry<Key,Value> elt : scanner) { String expected = String.format("%06d", count); if (!elt.getKey().getRow().toString().equals(expected)) throw new RuntimeException( "entry " + elt + " does not match expected " + expected + " in table " + table); count += tableNames.size(); } i++; } }
public static Text findMax(Scanner scanner, Text start, boolean is, Text end, boolean ie) throws TableNotFoundException { scanner.setBatchSize(12); IteratorSetting cfg = new IteratorSetting(Integer.MAX_VALUE, SortedKeyIterator.class); scanner.addScanIterator(cfg); if (start == null) { start = new Text(); is = true; } if (end == null) end = findInitialEnd(scanner); return _findMax(scanner, start, is, end, ie); } }
private static int scanTablet(Connector conn, String table, Authorizations auths, int batchSize, Text prevEndRow, Text endRow, String[] columns) throws Exception { Scanner scanner = conn.createScanner(table, auths); scanner.setBatchSize(batchSize); scanner.setRange(new Range(prevEndRow, false, endRow, true)); for (String c : columns) { scanner.fetchColumnFamily(new Text(c)); } int count = 0; for (Entry<Key,Value> entry : scanner) { if (entry != null) count++; } return count; }
private static int dumpTrace(Opts opts, ScannerOpts scanOpts) throws Exception { final PrintStream out = System.out; Connector conn = opts.getConnector(); int count = 0; for (String traceId : opts.traceIds) { Scanner scanner = conn.createScanner(opts.getTableName(), opts.auths); scanner.setBatchSize(scanOpts.scanBatchSize); Range range = new Range(new Text(traceId.toString())); scanner.setRange(range); count = printTrace(scanner, new Printer() { @Override public void print(String line) { out.println(line); } }); } return count > 0 ? 0 : 1; }
private Iterator<Entry<Key,Value>> newIterator(Range r) { synchronized (scanner) { scanner.enableIsolation(); scanner.setBatchSize(batchSize); scanner.setTimeout(timeout, TimeUnit.MILLISECONDS); scanner.setRange(r); scanner.setReadaheadThreshold(readaheadThreshold); setOptions((ScannerOptions) scanner, opts); return scanner.iterator(); // return new FaultyIterator(scanner.iterator()); } }
public ChunkInputStream getData(String hash) throws IOException { scanner.setRange(new Range(hash)); scanner.setBatchSize(1); lastRefs.clear(); PeekingIterator<Entry<Key,Value>> pi = new PeekingIterator<Entry<Key,Value>>(scanner.iterator()); if (pi.hasNext()) { while (!pi.peek().getKey().getColumnFamily().equals(FileDataIngest.CHUNK_CF)) { lastRefs.add(pi.peek()); pi.next(); } } cis.clear(); cis.setSource(pi); return cis; }
private static TreeSet<RowColumn> scanAll(ClientOnDefaultTable opts, ScannerOpts scanOpts, String tableName) throws Exception { TreeSet<RowColumn> result = new TreeSet<>(); Connector conn = opts.getConnector(); Scanner scanner = conn.createScanner(tableName, auths); scanner.setBatchSize(scanOpts.scanBatchSize); for (Entry<Key,Value> entry : scanner) { Key key = entry.getKey(); Column column = new Column(TextUtil.getBytes(key.getColumnFamily()), TextUtil.getBytes(key.getColumnQualifier()), TextUtil.getBytes(key.getColumnVisibility())); result.add(new RowColumn(key.getRow(), column, key.getTimestamp())); } return result; }
@Test public void run() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 0; i < 100000; i++) { Mutation m = new Mutation(new Text(String.format("%08d", i))); for (int j = 0; j < 3; j++) m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8))); bw.addMutation(m); } bw.close(); Scanner scanner = c.createScanner(tableName, new Authorizations()); scanner.setBatchSize(1000); Iterator<Entry<Key,Value>> iter = scanner.iterator(); verify(iter, 0, 200); // sleep three times the session timeout sleepUninterruptibly(9, TimeUnit.SECONDS); verify(iter, 200, 100000); }
private String getTabletStats() throws Exception { Connector conn = opts.getConnector(); Scanner scanner = conn.createScanner(MetadataTable.NAME, opts.auths); scanner.setBatchSize(scanBatchSize); scanner.fetchColumnFamily(DataFileColumnFamily.NAME); scanner .addScanIterator(new IteratorSetting(1000, "cfc", ColumnFamilyCounter.class.getName())); scanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange()); Stat s = new Stat(); int count = 0; for (Entry<Key,Value> entry : scanner) { count++; s.addStat(Long.parseLong(entry.getValue().toString())); } if (count > 0) return String.format("%d %d %.3f %.3f", s.getMin(), s.getMax(), s.getAverage(), s.getStdDev()); else return "0 0 0 0"; }