@Override public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException { FileRef ref = new FileRef(mapFileName, new Path(mapFileName)); return trm.openFiles(Collections.singletonMap(ref, files.get(ref)), false, null).get(0); }
@Override public void loadFiles(TInfo tinfo, TCredentials credentials, long tid, String dir, Map<TKeyExtent,Map<String,MapFileInfo>> tabletImports, boolean setTime) throws ThriftSecurityException { if (!security.canPerformSystemActions(credentials)) throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); watcher.runQuietly(Constants.BULK_ARBITRATOR_TYPE, tid, () -> { tabletImports.forEach((tke, fileMap) -> { Map<FileRef,MapFileInfo> fileRefMap = new HashMap<>(); for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) { Path path = new Path(dir, mapping.getKey()); FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); path = ns.makeQualified(path); fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue()); } Tablet importTablet = onlineTablets.get(new KeyExtent(tke)); if (importTablet != null) { try { importTablet.importMapFiles(tid, fileRefMap, setTime); } catch (IOException ioe) { log.debug("files {} not imported to {}: {}", fileMap.keySet(), new KeyExtent(tke), ioe.getMessage()); } } }); }); }
public static Map<Long,? extends Collection<FileRef>> getBulkFilesLoaded(ServerContext context, KeyExtent extent) { Text metadataRow = extent.getMetadataEntry(); Map<Long,List<FileRef>> result = new HashMap<>(); VolumeManager fs = context.getVolumeManager(); try (Scanner scanner = new ScannerImpl(context, extent.isMeta() ? RootTable.ID : MetadataTable.ID, Authorizations.EMPTY)) { scanner.setRange(new Range(metadataRow)); scanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME); for (Entry<Key,Value> entry : scanner) { Long tid = Long.parseLong(entry.getValue().toString()); List<FileRef> lst = result.get(tid); if (lst == null) { result.put(tid, lst = new ArrayList<>()); } lst.add(new FileRef(fs, entry.getKey())); } } return result; }
FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); path = ns.makeQualified(path); fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
public static List<FileRef> getBulkFilesLoaded(ServerContext context, AccumuloClient client, KeyExtent extent, long tid) { List<FileRef> result = new ArrayList<>(); try (Scanner mscanner = new IsolatedScanner(client.createScanner( extent.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY))) { VolumeManager fs = context.getVolumeManager(); mscanner.setRange(extent.toMetadataRange()); mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME); for (Entry<Key,Value> entry : mscanner) { if (Long.parseLong(entry.getValue().toString()) == tid) { result.add(new FileRef(fs, entry.getKey())); } } return result; } catch (TableNotFoundException ex) { // unlikely throw new RuntimeException("Onos! teh metadata table has vanished!!"); } }
FileRef getNextMapFilename(String prefix) throws IOException { String extension = FileOperations.getNewFileExtension(tableConfiguration); checkTabletDir(); return new FileRef( location + "/" + prefix + context.getUniqueNameAllocator().getNextName() + "." + extension); }
if (switchedPath != null) { filesToRemove.add(entry.getKey()); FileRef switchedRef = new FileRef(switchedPath, new Path(switchedPath)); filesToAdd.put(switchedRef, entry.getValue()); ret.datafiles.put(switchedRef, entry.getValue()); for (Entry<FileRef,DataFileValue> entry : copy.entrySet()) { ret.datafiles.put( new FileRef(new Path(ret.dir, entry.getKey().path().getName()).toString()), entry.getValue());
public static SortedMap<FileRef,DataFileValue> getDataFileSizes(KeyExtent extent, ServerContext context) { TreeMap<FileRef,DataFileValue> sizes = new TreeMap<>(); try (Scanner mdScanner = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) { mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME); Text row = extent.getMetadataEntry(); Key endKey = new Key(row, DataFileColumnFamily.NAME, new Text("")); endKey = endKey.followingKey(PartialKey.ROW_COLFAM); mdScanner.setRange(new Range(new Key(row), endKey)); for (Entry<Key,Value> entry : mdScanner) { if (!entry.getKey().getRow().equals(row)) break; DataFileValue dfv = new DataFileValue(entry.getValue().get()); sizes.put(new FileRef(context.getVolumeManager(), entry.getKey()), dfv); } return sizes; } }
FileRef ref = new FileRef(fs, key); dataFiles.put(ref, new DataFileValue(entry.getValue().get())); } else if (DIRECTORY_COLUMN.hasColumns(key)) { logEntries.add(LogEntry.fromKeyValue(key, entry.getValue())); } else if (family.equals(ScanFileColumnFamily.NAME)) { scanFiles.add(new FileRef(fs, key)); } else if (FLUSH_COLUMN.hasColumns(key)) { flushID = Long.parseLong(value.toString()); bulkImported.put(id, lst = new ArrayList<>()); lst.add(new FileRef(fs, key)); } else if (PREV_ROW_COLUMN.hasColumns(key)) { KeyExtent check = new KeyExtent(key.getRow(), value);
DataFileValue value = files.get(new FileRef(filename)); if (value.isTimeSet()) { iter = new TimeSettingIterator(iter, value.getTime());
origDatafileSizes.put(new FileRef(context.getVolumeManager(), entry.getKey()), new DataFileValue(entry.getValue().get()));
Path path = new Path(good); String filename = path.getName(); FileRef ref = new FileRef(location + "/" + filename, path); DataFileValue dfv = new DataFileValue(0, 0); dataFiles.put(ref, dfv);
FileRef compactTmpName = new FileRef(fileName.path() + "_tmp");
Key key = entry.getKey(); if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) { datafiles.add(new FileRef(this.master.fs, key)); if (datafiles.size() > 1000) { MetadataTableUtil.addDeleteEntries(extent, datafiles, master.getContext()); datafiles.add(new FileRef(path)); } else { datafiles.add(new FileRef(path, this.master.fs.getFullPath(FileType.TABLE, Path.SEPARATOR + extent.getTableId() + path)));
try { FileRef newMapfileLocation = tablet.getNextMapFilename(mergeFile == null ? "F" : "M"); FileRef tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp"); Span span = Trace.start("waitForCommits"); synchronized (tablet) {
FileRef ref = new FileRef(context.getVolumeManager(), key); bw.addMutation(createDeleteMutation(context, tableId, ref.meta().toString()));
@Override public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException { FileRef ref = new FileRef(mapFileName, new Path(mapFileName)); return trm.openFiles(Collections.singletonMap(ref, files.get(ref)), false, null).get(0); }
FileRef getNextMapFilename(String prefix) throws IOException { String extension = FileOperations.getNewFileExtension(tableConfiguration); checkTabletDir(); return new FileRef(location.toString() + "/" + prefix + UniqueNameAllocator.getInstance().getNextName() + "." + extension); }