static boolean goodTransition(TabletMetadata prev, TabletMetadata curr) { if (!curr.sawPrevEndRow()) { log.warn("Tablet {} had no prev end row.", curr.getExtent()); return false; } if (!curr.getTableId().equals(prev.getTableId())) { if (prev.getEndRow() != null) { log.debug("Non-null end row for last tablet in table: " + prev.getExtent() + " " + curr.getExtent()); return false; } if (curr.getPrevEndRow() != null) { log.debug("First tablet for table had prev end row {} {} ", prev.getExtent(), curr.getExtent()); return false; } } else { if (prev.getEndRow() == null) { throw new IllegalStateException("Null end row for tablet in middle of table: " + prev.getExtent() + " " + curr.getExtent()); } if (curr.getPrevEndRow() == null || !prev.getEndRow().equals(curr.getPrevEndRow())) { log.debug("Tablets end row and prev end row not equals {} {} ", prev.getExtent(), curr.getExtent()); return false; } } return true; }
@Override public Stream<Reference> getReferences() { Stream<TabletMetadata> tabletStream = TabletsMetadata.builder().scanTable(tableName) .checkConsistency().fetchDir().fetchFiles().fetchScans().build(getClient()).stream(); Stream<Reference> refStream = tabletStream.flatMap(tm -> { Stream<Reference> refs = Stream.concat(tm.getFiles().stream(), tm.getScans().stream()) .map(f -> new Reference(tm.getTableId(), f, false)); if (tm.getDir() != null) { refs = Stream.concat(refs, Stream.of(new Reference(tm.getTableId(), tm.getDir(), true))); } return refs; }); return refStream; }
public KeyExtent getExtent() { if (extent == null) { extent = new KeyExtent(getTableId(), getEndRow(), getPrevEndRow()); } return extent; }
/** * Find all the tablets within the provided bulk load mapping range. */ private List<TabletMetadata> findOverlappingTablets(KeyExtent loadRange, Iterator<TabletMetadata> tabletIter) { List<TabletMetadata> tablets = new ArrayList<>(); TabletMetadata currentTablet = tabletIter.next(); // skip tablets until we find the prevEndRow of loadRange while (!Objects.equals(currentTablet.getPrevEndRow(), loadRange.getPrevEndRow())) { currentTablet = tabletIter.next(); } // we have found the first tablet in the range, add it to the list tablets.add(currentTablet); // find the remaining tablets within the loadRange by // adding tablets to the list until the endRow matches the loadRange while (!Objects.equals(currentTablet.getEndRow(), loadRange.getEndRow())) { currentTablet = tabletIter.next(); tablets.add(currentTablet); } return tablets; } }
Text cloneEndRow = cloneTablet.getEndRow(); HashSet<String> cloneFiles = new HashSet<>(); boolean cloneSuccessful = cloneTablet.getCloned() != null; getFiles(cloneFiles, cloneTablet.getFiles(), null); Text srcEndRow = srcTablet.getEndRow(); int cmp = compareEndRows(cloneEndRow, srcEndRow); if (cmp < 0) getFiles(srcFiles, srcTablet.getFiles(), srcTableId); srcEndRow = srcTablet.getEndRow(); cmp = compareEndRows(cloneEndRow, srcEndRow); if (cmp < 0) getFiles(srcFiles, srcTablet.getFiles(), srcTableId); Mutation m = new Mutation(cloneTablet.getExtent().getMetadataEntry()); for (Entry<Key,Value> entry : cloneTablet.getKeyValues().entrySet()) { Key k = entry.getKey(); m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp()); bw.addMutation(createCloneMutation(srcTableId, tableId, st.getKeyValues())); Mutation m = new Mutation(cloneTablet.getExtent().getMetadataEntry()); m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(UTF_8)));
if (!tablet.getExtent().equals(extent)) throw new RuntimeException( "Unexpected extent " + tablet.getExtent() + " expected " + extent); result.addAll(tablet.getLogs()); tablet.getFilesMap().forEach((k, v) -> { sizes.put(new FileRef(k, fs.getFullPath(tablet.getTableId(), k)), v); });
for (String file : tm.getFiles()) { if (fileSelector.test(file)) { String location = entry.getValue().stream().filter(tm -> tm.getLocation() != null) // filter .map(tm -> tm.getLocation().getHostAndPort().toString()) // convert to host:port strings .orElse(entry.getValue().stream().filter(tm -> tm.getLast() != null) // if no locations, .map(tm -> tm.getLast().getHostAndPort().toString()) // convert to host:port strings .mergeOverlapping(Lists.transform(entry.getValue(), tm -> tm.getExtent().toDataRange())); List<TRowRange> ranges = merged.stream().map(r -> toClippedExtent(r).toThrift())
while (tablet.getLocation() != null) { if (Tables.getTableState(context, tableId) != TableState.OFFLINE) { Tables.clearCache(context); if (Tables.getTableState(context, tableId) != TableState.OFFLINE) { throw new AccumuloException("Table is online " + tableId + " cannot scan tablet in offline mode " + tablet.getExtent()); if (!tablet.getExtent().getTableId().equals(tableId)) { throw new AccumuloException( " did not find tablets for table " + tableId + " " + tablet.getExtent()); if (currentExtent != null && !tablet.getExtent().isPreviousExtent(currentExtent)) throw new AccumuloException( " " + currentExtent + " is not previous extent " + tablet.getExtent()); for (String relPath : tablet.getFiles()) { if (relPath.contains(":")) { absFiles.add(relPath); iter = createIterator(tablet.getExtent(), absFiles); iter.seek(range, LocalityGroupUtil.families(options.fetchedColumns), options.fetchedColumns.size() != 0); currentExtent = tablet.getExtent();
protected Iterable<Pair<KeyExtent,Location>> getLocationProvider() { return () -> { try { return TabletsMetadata.builder().forTable(tableId).fetchLocation().fetchPrev() .build(context).stream().map(tm -> { Location loc = Location.NONE; if (tm.hasCurrent()) { loc = new Location(new TServerInstance(tm.getLocation())); } return new Pair<>(tm.getExtent(), loc); }).iterator(); } catch (Exception e) { throw new RuntimeException(e); } }; }
int logs = tablet.getLogs().size(); if ((tablet.hasCurrent() || logs > 0) && tablet.getFlushId().orElse(-1) < flushID) { tabletsToWaitFor++; if (tablet.hasCurrent()) serversToFlush.add(new TServerInstance(tablet.getLocation()));
@Override void load(List<TabletMetadata> tablets, Files files) throws MutationsRejectedException { byte[] fam = TextUtil.getBytes(DataFileColumnFamily.NAME); for (TabletMetadata tablet : tablets) { if (tablet.getLocation() != null) { unloadingTablets.increment(tablet.getLocation().getHostAndPort(), 1L); continue; } Mutation mutation = new Mutation(tablet.getExtent().getMetadataEntry()); for (final Bulk.FileInfo fileInfo : files) { String fullPath = new Path(bulkDir, fileInfo.getFileName()).toString(); byte[] val = new DataFileValue(fileInfo.getEstFileSize(), fileInfo.getEstNumEntries()) .encode(); mutation.put(fam, fullPath.getBytes(UTF_8), val); } bw.addMutation(mutation); } }
@Override public TabletMetadata next() { long sleepTime = 250; TabletMetadata currTablet = null; while (currTablet == null) { TabletMetadata tmp = source.next(); if (prevTablet == null) { if (tmp.sawPrevEndRow()) { currTablet = tmp; } else { log.warn("Tablet has no prev end row " + tmp.getTableId() + " " + tmp.getEndRow()); } } else if (goodTransition(prevTablet, tmp)) { currTablet = tmp; } if (currTablet == null) { sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS); resetSource(); sleepTime = Math.min(2 * sleepTime, 5000); } } prevTablet = currTablet; return currTablet; } }
@Override void load(List<TabletMetadata> tablets, Files files) { for (TabletMetadata tablet : tablets) { // send files to tablet sever // ideally there should only be one tablet location to send all the files TabletMetadata.Location location = tablet.getLocation(); HostAndPort server = null; if (location == null) { locationLess++; continue; } else { server = location.getHostAndPort(); } Set<String> loadedFiles = tablet.getLoaded(); Map<String,MapFileInfo> thriftImports = new HashMap<>(); for (final Bulk.FileInfo fileInfo : files) { String fullPath = new Path(bulkDir, fileInfo.getFileName()).toString(); if (!loadedFiles.contains(fullPath)) { thriftImports.put(fileInfo.getFileName(), new MapFileInfo(fileInfo.getEstFileSize())); } } addToQueue(server, tablet.getExtent(), thriftImports); } sendQueued(4 * 1024 * 1024); }
private void resetSource() { if (prevTablet == null) { source = iteratorFactory.apply(range); } else { // get the metadata table row for the previous tablet Text prevMetaRow = TabletsSection.getRow(prevTablet.getTableId(), prevTablet.getEndRow()); // ensure the previous tablet still exists in the metadata table if (Iterators.size(iteratorFactory.apply(new Range(prevMetaRow))) == 0) { throw new TabletDeletedException("Tablet " + prevMetaRow + " was deleted while iterating"); } // start scanning at next possible row in metadata table Range seekRange = new Range(new Key(prevMetaRow).followingKey(PartialKey.ROW), true, range.getEndKey(), range.isEndKeyInclusive()); log.info("Resetting scanner to {}", seekRange); source = iteratorFactory.apply(seekRange); } }
if (tablet.getCompactId().orElse(-1) < compactId) { tabletsToWaitFor++; if (tablet.hasCurrent()) { serversToFlush.increment(new TServerInstance(tablet.getLocation()), 1);
protected Iterator<Size> getSizeIterator(AccumuloClient client, String tablename, Text start, Text end) throws MergeException { // open up metadata, walk through the tablets. Table.ID tableId; TabletsMetadata tablets; try { ClientContext context = (ClientContext) client; tableId = Tables.getTableId(context, tablename); tablets = TabletsMetadata.builder().scanMetadataTable() .overRange(new KeyExtent(tableId, end, start).toMetadataRange()).fetchFiles().fetchPrev() .build(context); } catch (Exception e) { throw new MergeException(e); } return tablets.stream().map(tm -> { long size = tm.getFilesMap().values().stream().mapToLong(DataFileValue::getSize).sum(); return new Size(tm.getExtent(), size); }).iterator(); }
@Override public TabletMetadata next() { if (sawLast) { throw new NoSuchElementException(); } TabletMetadata next = iter.next(); // impossible to construct a range that stops at the first tablet that contains endRow. That // is why this specialized code exists. if (next.getExtent().contains(endRow)) { sawLast = true; } return next; } }
private int countFiles() { // TODO use a batch scanner + iterator to parallelize counting files return TabletsMetadata.builder().forTable(tableId).overlapping(startRow, endRow).fetchFiles() .fetchPrev().build(ctx).stream().mapToInt(tm -> tm.getFiles().size()).sum(); }
total++; Location loc = tablet.getLocation(); || (expectedState == TableState.OFFLINE && loc != null)) { if (continueRow == null) continueRow = tablet.getExtent().getMetadataEntry(); waitFor++; lastRow = tablet.getExtent().getMetadataEntry(); if (!tablet.getExtent().getTableId().equals(tableId)) { throw new AccumuloException( "Saw unexpected table Id " + tableId + " " + tablet.getExtent()); if (lastExtent != null && !tablet.getExtent().isPreviousExtent(lastExtent)) { holes++; lastExtent = tablet.getExtent();