@Override public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException { FileRef ref = new FileRef(mapFileName, new Path(mapFileName)); return trm.openFiles(Collections.singletonMap(ref, files.get(ref)), false, null).get(0); }
public static Collection<String> toPathStrings(Collection<FileRef> refs) { ArrayList<String> ret = new ArrayList<>(); for (FileRef fileRef : refs) { ret.add(fileRef.path().toString()); } return ret; }
@Override public boolean equals(Object obj) { if (obj instanceof FileRef) { return compareTo((FileRef) obj) == 0; } return false; }
} else { for (Entry<FileRef,DataFileValue> entry : tabletFiles.datafiles.entrySet()) { String metaPath = entry.getKey().meta().toString(); String switchedPath = switchVolume(metaPath, FileType.TABLE, replacements); if (switchedPath != null) { filesToRemove.add(entry.getKey()); FileRef switchedRef = new FileRef(switchedPath, new Path(switchedPath)); filesToAdd.put(switchedRef, entry.getValue()); ret.datafiles.put(switchedRef, entry.getValue()); for (Entry<FileRef,DataFileValue> entry : copy.entrySet()) { ret.datafiles.put( new FileRef(new Path(ret.dir, entry.getKey().path().getName()).toString()), entry.getValue());
try { FileRef newMapfileLocation = tablet.getNextMapFilename(mergeFile == null ? "F" : "M"); FileRef tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp"); Span span = Trace.start("waitForCommits"); synchronized (tablet) { commitSession.getWALogSeq() + 1, newMapfileLocation.path().toString()); break; } catch (IOException e) {
String getOutputFile() { return outputFile.toString(); }
public static void updateTabletDataFile(long tid, KeyExtent extent, Map<FileRef,DataFileValue> estSizes, String time, ServerContext context, ZooLock zooLock) { Mutation m = new Mutation(extent.getMetadataEntry()); byte[] tidBytes = Long.toString(tid).getBytes(UTF_8); for (Entry<FileRef,DataFileValue> entry : estSizes.entrySet()) { Text file = entry.getKey().meta(); m.put(DataFileColumnFamily.NAME, file, new Value(entry.getValue().encode())); m.put(TabletsSection.BulkFileColumnFamily.NAME, file, new Value(tidBytes)); } TabletsSection.ServerColumnFamily.TIME_COLUMN.put(m, new Value(time.getBytes(UTF_8))); update(context, zooLock, m, extent); }
public FileRef(String metaReference, Path fullReference) { this.metaReference = metaReference; this.fullReference = fullReference; this.suffix = extractSuffix(fullReference); }
void unreserveMergingMinorCompactionFile(FileRef file) { if ((file == null && mergingMinorCompactionFile != null) || (file != null && mergingMinorCompactionFile == null) || (file != null && mergingMinorCompactionFile != null && !file.equals(mergingMinorCompactionFile))) throw new IllegalStateException("Disagreement " + file + " " + mergingMinorCompactionFile); mergingMinorCompactionFile = null; }
} else { for (Entry<FileRef,DataFileValue> entry : tabletFiles.datafiles.entrySet()) { String metaPath = entry.getKey().meta().toString(); String switchedPath = switchVolume(metaPath, FileType.TABLE, replacements); if (switchedPath != null) { filesToRemove.add(entry.getKey()); FileRef switchedRef = new FileRef(switchedPath, new Path(switchedPath)); filesToAdd.put(switchedRef, entry.getValue()); ret.datafiles.put(switchedRef, entry.getValue()); for (Entry<FileRef,DataFileValue> entry : copy.entrySet()) { ret.datafiles.put( new FileRef(new Path(ret.dir, entry.getKey().path().getName()).toString()), entry.getValue());
filesToCompact.add(ref.toString()); return new ActiveCompaction(compactor.extent.toThrift(), System.currentTimeMillis() - compactor.getStartTime(), filesToCompact,
public static void finishSplit(Text metadataEntry, Map<FileRef,DataFileValue> datafileSizes, List<FileRef> highDatafilesToRemove, final ServerContext context, ZooLock zooLock) { Mutation m = new Mutation(metadataEntry); TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m); TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m); ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m); for (Entry<FileRef,DataFileValue> entry : datafileSizes.entrySet()) { m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode())); } for (FileRef pathToRemove : highDatafilesToRemove) { m.putDelete(DataFileColumnFamily.NAME, pathToRemove.meta()); } update(context, zooLock, m, new KeyExtent(metadataEntry, (Text) null)); }
public FileRef(String metaReference, Path fullReference) { this.metaReference = metaReference; this.fullReference = fullReference; this.suffix = extractSuffix(fullReference); }
void unreserveMergingMinorCompactionFile(FileRef file) { if ((file == null && mergingMinorCompactionFile != null) || (file != null && mergingMinorCompactionFile == null) || (file != null && mergingMinorCompactionFile != null && !file.equals(mergingMinorCompactionFile))) throw new IllegalStateException("Disagreement " + file + " " + mergingMinorCompactionFile); mergingMinorCompactionFile = null; }
@Override public void loadFiles(TInfo tinfo, TCredentials credentials, long tid, String dir, Map<TKeyExtent,Map<String,MapFileInfo>> tabletImports, boolean setTime) throws ThriftSecurityException { if (!security.canPerformSystemActions(credentials)) throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); watcher.runQuietly(Constants.BULK_ARBITRATOR_TYPE, tid, () -> { tabletImports.forEach((tke, fileMap) -> { Map<FileRef,MapFileInfo> fileRefMap = new HashMap<>(); for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) { Path path = new Path(dir, mapping.getKey()); FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); path = ns.makeQualified(path); fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue()); } Tablet importTablet = onlineTablets.get(new KeyExtent(tke)); if (importTablet != null) { try { importTablet.importMapFiles(tid, fileRefMap, setTime); } catch (IOException ioe) { log.debug("files {} not imported to {}: {}", fileMap.keySet(), new KeyExtent(tke), ioe.getMessage()); } } }); }); }