private SnapshotDescription toSnapshotDescription(ProcedureDescription desc) throws IOException { SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); if (!desc.hasInstance()) { throw new IOException("Snapshot name is not defined: " + desc.toString()); } String snapshotName = desc.getInstance(); List<NameStringPair> props = desc.getConfigurationList(); String table = null; for (NameStringPair prop : props) { if ("table".equalsIgnoreCase(prop.getName())) { table = prop.getValue(); } } if (table == null) { throw new IOException("Snapshot table is not defined: " + desc.toString()); } TableName tableName = TableName.valueOf(table); builder.setTable(tableName.getNameAsString()); builder.setName(snapshotName); builder.setType(SnapshotDescription.Type.FLUSH); return builder.build(); } }
.setTable(tableName.getNameAsString()) .setType(SnapshotProtos.SnapshotDescription.Type.FLUSH) .setName("test_table_snapshot")
.setName(snapshotName).setTable(TABLE_NAME.getNameAsString()).build(); builder.setSnapshot(desc); SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(),
.setTable(tableName.getNameAsString()) .setType(SnapshotProtos.SnapshotDescription.Type.FLUSH) .setName("test_table_snapshot")
.setName(snapshotName).setTable(TABLE_NAME.getNameAsString()).build(); builder.setSnapshot(desc); SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(),
final SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName(snapshotName).setTable(tableName.getNameAsString()).build();
/** * Convert from {@link SnapshotDescription} to * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} * @param snapshotDesc the POJO SnapshotDescription * @return the protobuf SnapshotDescription */ public static SnapshotProtos.SnapshotDescription createHBaseProtosSnapshotDesc(SnapshotDescription snapshotDesc) { SnapshotProtos.SnapshotDescription.Builder builder = SnapshotProtos.SnapshotDescription.newBuilder(); if (snapshotDesc.getTableName() != null) { builder.setTable(snapshotDesc.getTableNameAsString()); } if (snapshotDesc.getName() != null) { builder.setName(snapshotDesc.getName()); } if (snapshotDesc.getOwner() != null) { builder.setOwner(snapshotDesc.getOwner()); } if (snapshotDesc.getCreationTime() != -1L) { builder.setCreationTime(snapshotDesc.getCreationTime()); } if (snapshotDesc.getVersion() != -1) { builder.setVersion(snapshotDesc.getVersion()); } builder.setType(ProtobufUtil.createProtosSnapShotDescType(snapshotDesc.getType())); SnapshotProtos.SnapshotDescription snapshot = builder.build(); return snapshot; }
private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException { // Test Rolling-Upgrade like Snapshot. // half machines writing using v1 and the others using v2 format. SnapshotMock snapshotMock = createSnapshotMock(); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName); builder.addRegionV1(); builder.addRegionV2(); builder.addRegionV2(); builder.addRegionV1(); Path snapshotDir = builder.commit(); TableDescriptor htd = builder.getTableDescriptor(); SnapshotDescription desc = builder.getSnapshotDescription(); // Test clone a snapshot TableDescriptor htdClone = snapshotMock.createHtd("testtb-clone"); testRestore(snapshotDir, desc, htdClone); verifyRestore(rootDir, htd, htdClone); // Test clone a clone ("link to link") SnapshotDescription cloneDesc = SnapshotDescription.newBuilder() .setName("cloneSnapshot") .setTable("testtb-clone") .build(); Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName()); TableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2"); testRestore(cloneDir, cloneDesc, htdClone2); verifyRestore(rootDir, htd, htdClone2); }
private CompletableFuture<Void> internalRestoreSnapshot(String snapshotName, TableName tableName) { SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName(snapshotName).setTable(tableName.getNameAsString()).build(); try { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); } catch (IllegalArgumentException e) { return failedFuture(e); } return waitProcedureResult(this .<Long> newMasterCaller() .action( (controller, stub) -> this.<RestoreSnapshotRequest, RestoreSnapshotResponse, Long> call( controller, stub, RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot) .setNonceGroup(ng.getNonceGroup()).setNonce(ng.newNonce()).build(), (s, c, req, done) -> s.restoreSnapshot(c, req, done), (resp) -> resp.getProcId())).call()); }
@Test public void testAsyncFlushSnapshot() throws Exception { SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName("asyncSnapshot").setTable(TABLE_NAME.getNameAsString()) .setType(SnapshotProtos.SnapshotDescription.Type.FLUSH).build(); // take the snapshot async admin.takeSnapshotAsync( new SnapshotDescription("asyncSnapshot", TABLE_NAME, SnapshotType.FLUSH)); // constantly loop, looking for the snapshot to complete HMaster master = UTIL.getMiniHBaseCluster().getMaster(); SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200); LOG.info(" === Async Snapshot Completed ==="); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); // make sure we get the snapshot SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot); }
private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, final int numRegions, final int version) throws IOException { TableDescriptor htd = createHtd(tableName); RegionData[] regions = createTable(htd, numRegions); SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder() .setTable(htd.getTableName().getNameAsString()) .setName(snapshotName) .setVersion(version) .build(); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); }
final SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName(snapshotName).setTable(tableName.getNameAsString()).build();
/** * Convert from {@link SnapshotDescription} to * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} * @param snapshotDesc the POJO SnapshotDescription * @return the protobuf SnapshotDescription */ public static SnapshotProtos.SnapshotDescription createHBaseProtosSnapshotDesc(SnapshotDescription snapshotDesc) { SnapshotProtos.SnapshotDescription.Builder builder = SnapshotProtos.SnapshotDescription.newBuilder(); if (snapshotDesc.getTableName() != null) { builder.setTable(snapshotDesc.getTableNameAsString()); } if (snapshotDesc.getName() != null) { builder.setName(snapshotDesc.getName()); } if (snapshotDesc.getOwner() != null) { builder.setOwner(snapshotDesc.getOwner()); } if (snapshotDesc.getCreationTime() != -1L) { builder.setCreationTime(snapshotDesc.getCreationTime()); } if (snapshotDesc.getVersion() != -1) { builder.setVersion(snapshotDesc.getVersion()); } builder.setType(ProtobufUtil.createProtosSnapShotDescType(snapshotDesc.getType())); SnapshotProtos.SnapshotDescription snapshot = builder.build(); return snapshot; }
private CompletableFuture<Void> internalRestoreSnapshot(String snapshotName, TableName tableName) { SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName(snapshotName).setTable(tableName.getNameAsString()).build(); try { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); } catch (IllegalArgumentException e) { return failedFuture(e); } return waitProcedureResult(this .<Long> newMasterCaller() .action( (controller, stub) -> this.<RestoreSnapshotRequest, RestoreSnapshotResponse, Long> call( controller, stub, RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot) .setNonceGroup(ng.getNonceGroup()).setNonce(ng.newNonce()).build(), (s, c, req, done) -> s.restoreSnapshot(c, req, done), (resp) -> resp.getProcId())).call()); }
final SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName(snapshotName).setTable(tableName.getNameAsString()).build();
/** * Convert from {@link SnapshotDescription} to * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} * @param snapshotDesc the POJO SnapshotDescription * @return the protobuf SnapshotDescription */ public static SnapshotProtos.SnapshotDescription createHBaseProtosSnapshotDesc(SnapshotDescription snapshotDesc) { SnapshotProtos.SnapshotDescription.Builder builder = SnapshotProtos.SnapshotDescription.newBuilder(); if (snapshotDesc.getTableName() != null) { builder.setTable(snapshotDesc.getTableNameAsString()); } if (snapshotDesc.getName() != null) { builder.setName(snapshotDesc.getName()); } if (snapshotDesc.getOwner() != null) { builder.setOwner(snapshotDesc.getOwner()); } if (snapshotDesc.getCreationTime() != -1L) { builder.setCreationTime(snapshotDesc.getCreationTime()); } if (snapshotDesc.getVersion() != -1) { builder.setVersion(snapshotDesc.getVersion()); } builder.setType(ProtobufUtil.createProtosSnapShotDescType(snapshotDesc.getType())); SnapshotProtos.SnapshotDescription snapshot = builder.build(); return snapshot; }
private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException { // Test Rolling-Upgrade like Snapshot. // half machines writing using v1 and the others using v2 format. SnapshotMock snapshotMock = createSnapshotMock(); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName); builder.addRegionV1(); builder.addRegionV2(); builder.addRegionV2(); builder.addRegionV1(); Path snapshotDir = builder.commit(); TableDescriptor htd = builder.getTableDescriptor(); SnapshotDescription desc = builder.getSnapshotDescription(); // Test clone a snapshot TableDescriptor htdClone = snapshotMock.createHtd("testtb-clone"); testRestore(snapshotDir, desc, htdClone); verifyRestore(rootDir, htd, htdClone); // Test clone a clone ("link to link") SnapshotDescription cloneDesc = SnapshotDescription.newBuilder() .setName("cloneSnapshot") .setTable("testtb-clone") .build(); Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName()); TableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2"); testRestore(cloneDir, cloneDesc, htdClone2); verifyRestore(rootDir, htd, htdClone2); }
@Test public void testAsyncFlushSnapshot() throws Exception { SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName("asyncSnapshot").setTable(TABLE_NAME.getNameAsString()) .setType(SnapshotProtos.SnapshotDescription.Type.FLUSH).build(); // take the snapshot async admin.takeSnapshotAsync( new SnapshotDescription("asyncSnapshot", TABLE_NAME, SnapshotType.FLUSH)); // constantly loop, looking for the snapshot to complete HMaster master = UTIL.getMiniHBaseCluster().getMaster(); SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200); LOG.info(" === Async Snapshot Completed ==="); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); // make sure we get the snapshot SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot); }
private CompletableFuture<Void> internalRestoreSnapshot(String snapshotName, TableName tableName) { SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName(snapshotName).setTable(tableName.getNameAsString()).build(); try { ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot); } catch (IllegalArgumentException e) { return failedFuture(e); } return waitProcedureResult(this .<Long> newMasterCaller() .action( (controller, stub) -> this.<RestoreSnapshotRequest, RestoreSnapshotResponse, Long> call( controller, stub, RestoreSnapshotRequest.newBuilder().setSnapshot(snapshot) .setNonceGroup(ng.getNonceGroup()).setNonce(ng.newNonce()).build(), (s, c, req, done) -> s.restoreSnapshot(c, req, done), (resp) -> resp.getProcId())).call()); }
private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, final int numRegions, final int version) throws IOException { TableDescriptor htd = createHtd(tableName); RegionData[] regions = createTable(htd, numRegions); SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder() .setTable(htd.getTableName().getNameAsString()) .setName(snapshotName) .setVersion(version) .build(); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf); SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); }