@VisibleForTesting protected Stream<KeyExtent> lookupExtents(Text row) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { return TabletsMetadata.builder().forTable(tableId).overlapping(row, null).checkConsistency() .fetchPrev().build(ctx).stream().limit(100).map(TabletMetadata::getExtent); }
private int countFiles() { // TODO use a batch scanner + iterator to parallelize counting files return TabletsMetadata.builder().forTable(tableId).overlapping(startRow, endRow).fetchFiles() .fetchPrev().build(ctx).stream().mapToInt(tm -> tm.getFiles().size()).sum(); }
private void checkForMerge(final Master master) throws Exception { VolumeManager fs = master.getFileSystem(); final Path bulkDir = new Path(bulkInfo.sourceDir); try (LoadMappingIterator lmi = BulkSerialize.readLoadMapping(bulkDir.toString(), bulkInfo.tableId, p -> fs.open(p))) { TabletIterFactory tabletIterFactory = startRow -> TabletsMetadata.builder() .forTable(bulkInfo.tableId).overlapping(startRow, null).checkConsistency().fetchPrev() .build(master.getContext()).stream().map(TabletMetadata::getExtent).iterator(); checkForMerge(bulkInfo.tableId.canonicalID(), Iterators.transform(lmi, entry -> entry.getKey()), tabletIterFactory); } }
@Override public Stream<Reference> getReferences() { Stream<TabletMetadata> tabletStream = TabletsMetadata.builder().scanTable(tableName) .checkConsistency().fetchDir().fetchFiles().fetchScans().build(getClient()).stream(); Stream<Reference> refStream = tabletStream.flatMap(tm -> { Stream<Reference> refs = Stream.concat(tm.getFiles().stream(), tm.getScans().stream()) .map(f -> new Reference(tm.getTableId(), f, false)); if (tm.getDir() != null) { refs = Stream.concat(refs, Stream.of(new Reference(tm.getTableId(), tm.getDir(), true))); } return refs; }); return refStream; }
protected Iterator<Size> getSizeIterator(AccumuloClient client, String tablename, Text start, Text end) throws MergeException { // open up metadata, walk through the tablets. Table.ID tableId; TabletsMetadata tablets; try { ClientContext context = (ClientContext) client; tableId = Tables.getTableId(context, tablename); tablets = TabletsMetadata.builder().scanMetadataTable() .overRange(new KeyExtent(tableId, end, start).toMetadataRange()).fetchFiles().fetchPrev() .build(context); } catch (Exception e) { throw new MergeException(e); } return tablets.stream().map(tm -> { long size = tm.getFilesMap().values().stream().mapToLong(DataFileValue::getSize).sum(); return new Size(tm.getExtent(), size); }).iterator(); }
protected Iterable<Pair<KeyExtent,Location>> getLocationProvider() { return () -> { try { return TabletsMetadata.builder().forTable(tableId).fetchLocation().fetchPrev() .build(context).stream().map(tm -> { Location loc = Location.NONE; if (tm.hasCurrent()) { loc = new Location(new TServerInstance(tm.getLocation())); } return new Pair<>(tm.getExtent(), loc); }).iterator(); } catch (Exception e) { throw new RuntimeException(e); } }; }