/** * Gets a scanner from Accumulo over one row. * * @param row the row to scan * @param fields the set of columns to scan * @return an Accumulo {@link Scanner} bound to the given row and columns */ private Scanner getRow(String table, Text row, Set<String> fields) throws TableNotFoundException { Scanner scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(row)); if (fields != null) { for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } return scanner; }
private long getNumRowsInTable(String metricsTable, Authorizations auths) throws TableNotFoundException { // Create scanner against the metrics table, pulling the special column and the rows column Scanner scanner = connector.createScanner(metricsTable, auths); scanner.setRange(METRICS_TABLE_ROWID_RANGE); scanner.fetchColumn(METRICS_TABLE_ROWS_CF_AS_TEXT, CARDINALITY_CQ_AS_TEXT); // Scan the entry and get the number of rows long numRows = -1; for (Entry<Key, Value> entry : scanner) { if (numRows > 0) { throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "Should have received only one entry when scanning for number of rows in metrics table"); } numRows = Long.parseLong(entry.getValue().toString()); } scanner.close(); LOG.debug("Number of rows in table is %d", numRows); return numRows; }
private Optional<String> getDefaultTabletLocation(String fulltable) { try { String tableId = connector.tableOperations().tableIdMap().get(fulltable); // Create a scanner over the metadata table, fetching the 'loc' column of the default tablet row Scanner scan = connector.createScanner("accumulo.metadata", connector.securityOperations().getUserAuthorizations(username)); scan.fetchColumnFamily(new Text("loc")); scan.setRange(new Range(tableId + '<')); // scan the entry Optional<String> location = Optional.empty(); for (Entry<Key, Value> entry : scan) { if (location.isPresent()) { throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "Scan for default tablet returned more than one entry"); } location = Optional.of(entry.getValue().toString()); } scan.close(); return location; } catch (Exception e) { // Swallow this exception so the query does not fail due to being unable to locate the tablet server for the default tablet. // This is purely an optimization, but we will want to log the error. LOG.error("Failed to get tablet location, returning dummy location", e); return Optional.empty(); } }
conn.tableOperations().create(table.getFullTableName()); conn.tableOperations().create(table.getIndexTableName()); conn.tableOperations().create(table.getMetricsTableName()); indexer.flush(); Scanner scan = conn.createScanner(table.getIndexTableName(), new Authorizations()); scan.setRange(new Range()); scan = conn.createScanner(table.getMetricsTableName(), new Authorizations()); scan.setRange(new Range()); indexer.close(); scan = conn.createScanner(table.getIndexTableName(), new Authorizations()); scan.setRange(new Range()); iter = scan.iterator(); assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", ""); scan = conn.createScanner(table.getMetricsTableName(), new Authorizations()); scan.setRange(new Range());
private void verifyData(String table, int s, int e) throws Exception { Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY); Iterator<Entry<Key,Value>> iter = scanner.iterator(); for (int i = s; i <= e; i++) { if (!iter.hasNext()) throw new Exception("row " + i + " not found"); Entry<Key,Value> entry = iter.next(); String row = String.format("%04d", i); if (!entry.getKey().getRow().equals(new Text(row))) throw new Exception("unexpected row " + entry.getKey() + " " + i); if (Integer.parseInt(entry.getValue().toString()) != i) throw new Exception("unexpected value " + entry + " " + i); } if (iter.hasNext()) throw new Exception("found more than expected " + iter.next()); }
public void writeAndReadData(Connector connector, String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { // Write some data to the table BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig()); for (String s : rows) { Mutation m = new Mutation(new Text(s)); m.put(EMPTY, EMPTY, EMPTY_VALUE); bw.addMutation(m); } bw.close(); // Write the data to disk, read it back connector.tableOperations().flush(tableName, null, null, true); Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY); int i = 0; for (Entry<Key,Value> entry : scanner) { assertEquals("Data read is not data written", rows[i++], entry.getKey().getRow().toString()); } }
/** * Fetch all of the rfiles referenced by tablets in the metadata table for this table */ private Set<String> getFilesForTable(String tableName) throws Exception { final Connector conn = getConnector(); final String tableId = conn.tableOperations().tableIdMap().get(tableName); Assert.assertNotNull("Could not determine table ID for " + tableName, tableId); Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY); Range r = MetadataSchema.TabletsSection.getRange(tableId); s.setRange(r); s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME); Set<String> rfiles = new HashSet<>(); for (Entry<Key,Value> entry : s) { log.debug("Reading RFiles: {}={}", entry.getKey().toStringNoTruncate(), entry.getValue()); // uri://path/to/wal String cq = entry.getKey().getColumnQualifier().toString(); String path = new Path(cq).toString(); log.debug("Normalize path to rfile: {}", path); rfiles.add(path); } return rfiles; }
/** * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure * that the system user exists (since the master does an RPC to the tserver which will create the * system user if it doesn't already exist). */ private void createReadWriteDrop(Connector conn) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException { final String table = testName.getMethodName() + "_table"; conn.tableOperations().create(table); BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); Mutation m = new Mutation("a"); m.put("b", "c", "d"); bw.addMutation(m); bw.close(); conn.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true)); Scanner s = conn.createScanner(table, Authorizations.EMPTY); Entry<Key,Value> entry = Iterables.getOnlyElement(s); assertEquals("Did not find the expected key", 0, new Key("a", "b", "c").compareTo(entry.getKey(), PartialKey.ROW_COLFAM_COLQUAL)); assertEquals("d", entry.getValue().toString()); conn.tableOperations().delete(table); } }
public boolean checkTimeType(Connector connector, String tableName, TimeType expectedTimeType) throws TableNotFoundException { final Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY); String tableID = connector.tableOperations().tableIdMap().get(tableName) + "<"; for (Entry<Key,Value> entry : scanner) { Key k = entry.getKey(); if (k.getRow().toString().equals(tableID) && k.getColumnQualifier().toString() .equals(ServerColumnFamily.TIME_COLUMN.getColumnQualifier().toString())) { if (expectedTimeType == TimeType.MILLIS && entry.getValue().toString().charAt(0) == 'M') return true; if (expectedTimeType == TimeType.LOGICAL && entry.getValue().toString().charAt(0) == 'L') return true; } } return false; }
private Set<String> getRows(Connector c, String tableName) throws TableNotFoundException { Set<String> rows = new HashSet<>(); Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY); for (Entry<Key,Value> entry : scanner) rows.add(entry.getKey().getRowData().toString()); return rows; }
conn.tableOperations().create(table.getFullTableName()); conn.tableOperations().create(table.getIndexTableName()); conn.tableOperations().create(table.getMetricsTableName()); indexer.flush(); Scanner scan = conn.createScanner(table.getIndexTableName(), new Authorizations("private", "moreprivate")); scan.setRange(new Range()); scan = conn.createScanner(table.getMetricsTableName(), new Authorizations("private", "moreprivate")); scan.setRange(new Range()); indexer.close(); scan = conn.createScanner(table.getIndexTableName(), new Authorizations("private", "moreprivate")); scan.setRange(new Range()); iter = scan.iterator(); assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", ""); scan = conn.createScanner(table.getMetricsTableName(), new Authorizations("private", "moreprivate")); scan.setRange(new Range());
/** * Gets a scanner from Accumulo over one row. * * @param row the row to scan * @param fields the set of columns to scan * @return an Accumulo {@link Scanner} bound to the given row and columns */ private Scanner getRow(String table, Text row, Set<String> fields) throws TableNotFoundException { Scanner scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(row)); if (fields != null) { for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } return scanner; }
String tableId = connector.tableOperations().tableIdMap().get(table); Scanner scanner = connector.createScanner("accumulo.metadata", auths); scanner.fetchColumnFamily(new Text("loc")); Key start = new Key(tableId); Key end = defaultTabletRow.followingKey(PartialKey.ROW); scanner.setRange(new Range(start, end)); location = Optional.of(iter.next().getValue().toString()); Text splitCompareKey = new Text(); key.getRow(splitCompareKey); Text scannedCompareKey = new Text(); byte[] keyBytes = entry.getKey().getRow().copyBytes(); location = Optional.of(entry.getValue().toString()); break; int compareTo = splitCompareKey.compareTo(scannedCompareKey); if (compareTo <= 0) { location = Optional.of(entry.getValue().toString());
private static long scan(Connector conn, ArrayList<byte[]> cfset, String table, boolean cq) throws TableNotFoundException { Scanner scanner = conn.createScanner(table, Authorizations.EMPTY); if (!cq) scanner.fetchColumnFamily(new Text(cfset.get(15))); else scanner.fetchColumn(new Text(cfset.get(15)), new Text(cfset.get(15))); long t1 = System.currentTimeMillis(); Iterators.size(scanner.iterator()); long t2 = System.currentTimeMillis(); return t2 - t1; }
private int countLogs(String tableName, Connector conn) throws TableNotFoundException { Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LogColumnFamily.NAME); scanner.setRange(MetadataSchema.TabletsSection.getRange()); int count = 0; for (Entry<Key,Value> entry : scanner) { log.debug("Saw " + entry.getKey() + "=" + entry.getValue()); count++; } return count; }
public static int countRFiles(Connector c, String tableName) throws Exception { Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY); String tableId = c.tableOperations().tableIdMap().get(tableName); scanner.setRange(MetadataSchema.TabletsSection.getRange(tableId)); scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME); return Iterators.size(scanner.iterator()); }
public static void dumpTable(Authorizations auths) throws TableNotFoundException { TableOperations tops = connector.tableOperations(); org.apache.accumulo.core.client.Scanner scanner = connector.createScanner(DATE_INDEX_TABLE_NAME, auths); Iterator<Map.Entry<Key,Value>> iterator = scanner.iterator(); System.out.println("*************** " + DATE_INDEX_TABLE_NAME + " ********************"); while (iterator.hasNext()) { Map.Entry<Key,Value> entry = iterator.next(); System.out.println(entry); } System.out.println("*******************************************************************"); scanner.close(); }
private void checkData(String table2, Connector c) throws TableNotFoundException { Scanner scanner = c.createScanner(table2, Authorizations.EMPTY); HashMap<String,String> expected = new HashMap<>(); expected.put("001:x", "9"); expected.put("001:y", "7"); expected.put("008:x", "3"); expected.put("008:y", "4"); HashMap<String,String> actual = new HashMap<>(); for (Entry<Key,Value> entry : scanner) actual.put(entry.getKey().getRowData().toString() + ":" + entry.getKey().getColumnQualifierData().toString(), entry.getValue().toString()); Assert.assertEquals(expected, actual); }
@Test public void run() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); for (int i = 0; i < 100000; i++) { c.createScanner(tableName, Authorizations.EMPTY); } }
/** * Gets a scanner from Accumulo over one row. * * @param row the row to scan * @param fields the set of columns to scan * @return an Accumulo {@link Scanner} bound to the given row and columns */ private Scanner getRow(String table, Text row, Set<String> fields) throws TableNotFoundException { Scanner scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(row)); if (fields != null) { for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } return scanner; }