@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { String hfile = storeFile.getName(); if (HFileLink.isHFileLink(hfile)) { names.add(HFileLink.getReferencedHFileName(hfile)); } else { names.add(hfile); } } });
final Map<Path, Integer> filesMap) throws IOException { HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean isCorrupted = false; boolean inArchive = false;
if (!storeFile.hasReference()) { HFileLink link = HFileLink.build(conf, snapshot.getTableName(), regionInfo.getEncodedName(), family, storeFile.getName()); long size; Integer count;
/** * <code>required string name = 1;</code> */ public Builder clearName() { bitField0_ = (bitField0_ & ~0x00000001); name_ = getDefaultInstance().getName(); onChanged(); return this; } /**
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); HFileLink link = HFileLink.build(conf, table, region, family, hfile); if (corruptedFiles.size() % 2 == 0) { fs.delete(link.getAvailablePath(fs), true); corruptedFiles.add(hfile); } } });
String hfileName = storeFile.getName();
final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { TableName table = TableName.valueOf(snapshot.getTable()); String fileName = storeFile.getName();
List<SnapshotRegionManifest.StoreFile> hfilesToAdd = new ArrayList<>(); for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) { if (familyFiles.contains(storeFile.getName())) { familyFiles.remove(storeFile.getName()); } else { LOG.debug("Adding HFileLink " + storeFile.getName() + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
Long valueOrNull = filesToUpdate.remove(sf.getName()); if (valueOrNull != null) {
String storeFileName = sf.getName();
@java.lang.Override public boolean equals(final java.lang.Object obj) { if (obj == this) { return true; } if (!(obj instanceof org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile)) { return super.equals(obj); } org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile other = (org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile) obj; boolean result = true; result = result && (hasName() == other.hasName()); if (hasName()) { result = result && getName() .equals(other.getName()); } result = result && (hasReference() == other.hasReference()); if (hasReference()) { result = result && getReference() .equals(other.getReference()); } result = result && (hasFileSize() == other.hasFileSize()); if (hasFileSize()) { result = result && (getFileSize() == other.getFileSize()); } result = result && unknownFields.equals(other.unknownFields); return result; }
/** * Create a new {@link HFileLink} to reference the store file. * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference. * <ul> * <li>hfile: abc -> table=region-abc * <li>reference: abc.1234 -> table=region-abc.1234 * <li>hfilelink: table=region-hfile -> table=region-hfile * </ul> * @param familyDir destination directory for the store file * @param regionInfo destination region info for the table * @param createBackRef - Whether back reference should be created. Defaults to true. * @param storeFile store file name (can be a Reference, HFileLink or simple HFile) */ private void restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef) throws IOException { String hfileName = storeFile.getName(); if (HFileLink.isHFileLink(hfileName)) { HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef); } else if (StoreFileInfo.isReference(hfileName)) { restoreReferenceFile(familyDir, regionInfo, storeFile); } else { HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef); } }
@java.lang.Override public int hashCode() { if (memoizedHashCode != 0) { return memoizedHashCode; } int hash = 41; hash = (19 * hash) + getDescriptor().hashCode(); if (hasName()) { hash = (37 * hash) + NAME_FIELD_NUMBER; hash = (53 * hash) + getName().hashCode(); } if (hasReference()) { hash = (37 * hash) + REFERENCE_FIELD_NUMBER; hash = (53 * hash) + getReference().hashCode(); } if (hasFileSize()) { hash = (37 * hash) + FILE_SIZE_FIELD_NUMBER; hash = (53 * hash) + org.apache.hbase.thirdparty.com.google.protobuf.Internal.hashLong( getFileSize()); } hash = (29 * hash) + unknownFields.hashCode(); memoizedHashCode = hash; return hash; }
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { // for storeFile.hasReference() case, copied as part of the manifest if (!storeFile.hasReference()) { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); Path path = HFileLink.createPath(table, region, family, hfile); SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() .setType(SnapshotFileInfo.Type.HFILE) .setHfile(path.toString()) .build(); long size; if (storeFile.hasFileSize()) { size = storeFile.getFileSize(); } else { size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen(); } files.add(new Pair<>(fileInfo, size)); } } });
/** * Clone region directory content from the snapshot info. * * Each region is encoded with the table name, so the cloned region will have * a different region name. * * Instead of copying the hfiles a HFileLink is created. * * @param regionDir {@link Path} cloned dir * @param snapshotRegionInfo */ private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { final String tableName = tableDesc.getTableName().getNameAsString(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); } } }
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException { HashSet<String> files = new HashSet<>(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( snapshotName, FSUtils.getRootDir(conf)); SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo( fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); // For each region referenced by the snapshot for (SnapshotRegionManifest rm : manifest.getRegionManifests()) { // For each column family in this region for (FamilyFiles ff : rm.getFamilyFilesList()) { // And each store file in that family for (StoreFile sf : ff.getStoreFilesList()) { files.add(sf.getName()); } } } return files; }
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) return; String hfile = storeFile.getName(); snapshotFiles.add(hfile); if (storeFile.hasReference()) { // Nothing to do here, we have already the reference embedded } else { verifyNonEmptyFile(new Path(exportedArchive, new Path(FSUtils.getTableDir(new Path("./"), tableName), new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); } }
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (storeFile.hasReference()) return; SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile, null); if (showFiles) { String state = info.getStateToString(); System.out.printf("%8s %s/%s/%s/%s %s%n", (info.isMissing() ? "-" : fileSizeToString(info.getSize())), table, regionInfo.getEncodedName(), family, storeFile.getName(), state == null ? "" : "(" + state + ")"); } } });
/** * <code>required string name = 1;</code> */ public Builder clearName() { bitField0_ = (bitField0_ & ~0x00000001); name_ = getDefaultInstance().getName(); onChanged(); return this; } /**
@Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { String region = regionInfo.getEncodedName(); String hfile = storeFile.getName(); HFileLink link = HFileLink.build(conf, table, region, family, hfile); if (corruptedFiles.size() % 2 == 0) { fs.delete(link.getAvailablePath(fs), true); corruptedFiles.add(hfile); } } });