/** * If a migrating tablet splits, and the tablet dies before sending the master a message, the * migration will refer to a non-existing tablet, so it can never complete. Periodically scan * the metadata table and remove any migrating tablets that no longer exist. */ private void cleanupNonexistentMigrations(final AccumuloClient accumuloClient) throws TableNotFoundException { Scanner scanner = accumuloClient.createScanner(MetadataTable.NAME, Authorizations.EMPTY); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); Set<KeyExtent> found = new HashSet<>(); for (Entry<Key,Value> entry : scanner) { KeyExtent extent = new KeyExtent(entry.getKey().getRow(), entry.getValue()); if (migrations.containsKey(extent)) { found.add(extent); } } migrations.keySet().retainAll(found); }
TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME); scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LogColumnFamily.NAME); MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
@Override public TabletsMetadata build(AccumuloClient client) { try { Scanner scanner = new IsolatedScanner(client.createScanner(table, Authorizations.EMPTY)); scanner.setRange(range); if (checkConsistency && !fetchedCols.contains(FetchedColumns.PREV_ROW)) { fetchPrev(); } for (Text fam : families) { scanner.fetchColumnFamily(fam); } for (ColumnFQ col : qualifiers) { col.fetch(scanner); } if (families.size() == 0 && qualifiers.size() == 0) { fetchedCols = EnumSet.allOf(FetchedColumns.class); } Iterable<TabletMetadata> tmi = TabletMetadata.convert(scanner, fetchedCols, checkConsistency, saveKeyValues); if (endRow != null) { // create an iterable that will stop at the tablet which contains the endRow return new TabletsMetadata(scanner, () -> new TabletMetadataIterator(tmi.iterator(), endRow)); } else { return new TabletsMetadata(scanner, tmi); } } catch (TableNotFoundException e) { throw new RuntimeException(e); } }
TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
final Scanner scanner = shellState.getAccumuloClient().createScanner(systemTableToCheck, Authorizations.EMPTY); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); final Text start = new Text( shellState.getAccumuloClient().tableOperations().tableIdMap().get(tableName));
log.info("Rewriting entries for {}", tableName); Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY); DIRECTORY_COLUMN.fetch(scanner); scanner.setRange(TabletsSection.getRange(tableId)); BatchWriter writer = context.createBatchWriter(MetadataTable.NAME, null);
true, null, false); Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY); MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LastLocationColumnFamily.NAME); scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
private KeyExtent getHighTablet(KeyExtent range) throws AccumuloException { try { AccumuloClient client = this.master.getContext(); Scanner scanner = client.createScanner(range.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); KeyExtent start = new KeyExtent(range.getTableId(), range.getEndRow(), null); scanner.setRange(new Range(start.getMetadataEntry(), null)); Iterator<Entry<Key,Value>> iterator = scanner.iterator(); if (!iterator.hasNext()) { throw new AccumuloException("No last tablet for a merge " + range); } Entry<Key,Value> entry = iterator.next(); KeyExtent highTablet = new KeyExtent(entry.getKey().getRow(), KeyExtent.decodePrevEndRow(entry.getValue())); if (!highTablet.getTableId().equals(range.getTableId())) { throw new AccumuloException("No last tablet for merge " + range + " " + highTablet); } return highTablet; } catch (Exception ex) { throw new AccumuloException("Unexpected failure finding the last tablet for a merge " + range, ex); } }
Scanner scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY); scanner.setRange(scanRange); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(scanner); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); scanner.fetchColumnFamily(DataFileColumnFamily.NAME); Mutation m = new Mutation(stopRow); TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(scanner); for (Entry<Key,Value> entry : scanner) { if (TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
Scanner scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY); scanner.setRange(deleteRange); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(scanner); scanner.fetchColumnFamily(DataFileColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(ms);
/** * * @param s * A scanner over the entire metadata table configure to fetch needed columns. */ public TabletIterator(Scanner s, Range range, boolean returnPrevEndRow, boolean returnDir) { this.scanner = s; this.range = range; this.scanner.setRange(range); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); this.iter = s.iterator(); this.returnPrevEndRow = returnPrevEndRow; this.returnDir = returnDir; }
/** * * @param s * A scanner over the entire metadata table configure to fetch needed columns. */ public TabletIterator(Scanner s, Range range, boolean returnPrevEndRow, boolean returnDir) { this.scanner = s; this.range = range; this.scanner.setRange(range); Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner); Constants.METADATA_DIRECTORY_COLUMN.fetch(scanner); this.iter = s.iterator(); this.returnPrevEndRow = returnPrevEndRow; this.returnDir = returnDir; }
public static void configureScanner(ScannerBase scanner, CurrentState state) { TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME); scanner .fetchColumnFamily(TabletsSection.SuspendLocationColumn.SUSPEND_COLUMN.getColumnFamily()); scanner.fetchColumnFamily(LogColumnFamily.NAME); scanner.fetchColumnFamily(ChoppedColumnFamily.NAME); scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class)); IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class); if (state != null) { TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers()); TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables()); TabletStateChangeIterator.setMerges(tabletChange, state.merges()); TabletStateChangeIterator.setMigrations(tabletChange, state.migrationsSnapshot()); TabletStateChangeIterator.setMasterState(tabletChange, state.getMasterState()); TabletStateChangeIterator.setShuttingDown(tabletChange, state.shutdownServers()); } scanner.addScanIterator(tabletChange); }
private static Scanner createCloneScanner(String tableId, Connector conn) throws TableNotFoundException { Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS)); mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange()); mscanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY); mscanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY); mscanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY); mscanner.fetchColumnFamily(Constants.METADATA_CLONED_COLUMN_FAMILY); Constants.METADATA_PREV_ROW_COLUMN.fetch(mscanner); Constants.METADATA_TIME_COLUMN.fetch(mscanner); return mscanner; }
@Override public Iterator<Pair<KeyExtent,Location>> iterator() { try { Scanner scanner = new IsolatedScanner( context.getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY)); scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME); MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); scanner.setRange(MetadataSchema.TabletsSection.getRange(tableId)); RowIterator rowIter = new RowIterator(scanner); return Iterators.transform(rowIter, new LocationFunction()); } catch (Exception e) { throw new RuntimeException(e); } } }
/** * Create an IsolatedScanner over the given table, fetching the columns necessary to determine * when a table has transitioned to online or offline. */ protected IsolatedScanner createMetadataScanner(String metaTable, Range range) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { Scanner scanner = context.getConnector().createScanner(metaTable, Authorizations.EMPTY); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); scanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); scanner.setRange(range); return new IsolatedScanner(scanner); }
private static ScanStats runScanTest(Connector connector, int numLoop, List<Range> ranges) throws Exception { Scanner scanner = null; BatchScanner bs = connector.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY, 1); bs.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(bs); bs.setRanges(ranges); // System.out.println(ranges); ScanStats stats = new ScanStats(); for (int i = 0; i < numLoop; i++) { ScanStat ss = scan(bs, ranges, scanner); stats.merge(ss); } return stats; }