protected Path getBulkOutputDir() { String backupId = backupInfo.getBackupId(); Path path = new Path(backupInfo.getBackupRootDir()); path = new Path(path, ".tmp"); path = new Path(path, backupId); return path; } }
/** * Write the start code (timestamp) to backup system table. If passed in null, then write 0 byte. * @param startCode start code * @throws IOException exception */ public void writeBackupStartCode(Long startCode) throws IOException { systemTable.writeBackupStartCode(startCode, backupInfo.getBackupRootDir()); }
/** * Return the current tables covered by incremental backup. * @return set of tableNames * @throws IOException exception */ public Set<TableName> getIncrementalBackupTableSet() throws IOException { return systemTable.getIncrementalBackupTableSet(backupInfo.getBackupRootDir()); }
/** * Read the last backup start code (timestamp) of last successful backup. Will return null if * there is no startcode stored in backup system table or the value is of length 0. These two * cases indicate there is no successful backup completed so far. * @return the timestamp of a last successful backup * @throws IOException exception */ public String readBackupStartCode() throws IOException { return systemTable.readBackupStartCode(backupInfo.getBackupRootDir()); }
/** * Adds set of tables to overall incremental backup table set * @param tables tables * @throws IOException exception */ public void addIncrementalBackupTableSet(Set<TableName> tables) throws IOException { systemTable.addIncrementalBackupTableSet(tables, backupInfo.getBackupRootDir()); }
/** * Get the RS log information after the last log roll from backup system table. * @return RS log info * @throws IOException exception */ public HashMap<String, Long> readRegionServerLastLogRollResult() throws IOException { return systemTable.readRegionServerLastLogRollResult(backupInfo.getBackupRootDir()); }
/** * Write the current timestamps for each regionserver to backup system table after a successful * full or incremental backup. Each table may have a different set of log timestamps. The saved * timestamp is of the last log file that was backed up already. * @param tables tables * @throws IOException exception */ public void writeRegionServerLogTimestamp(Set<TableName> tables, HashMap<String, Long> newTimestamps) throws IOException { systemTable.writeRegionServerLogTimestamp(tables, newTimestamps, backupInfo.getBackupRootDir()); }
/** * Get WAL files iterator. * @return WAL files iterator from backup system table * @throws IOException if getting the WAL files iterator fails */ public Iterator<BackupSystemTable.WALItem> getWALFilesFromBackupSystem() throws IOException { return systemTable.getWALFilesIterator(backupInfo.getBackupRootDir()); }
/** * Read the timestamp for each region server log after the last successful backup. Each table has * its own set of the timestamps. * @return the timestamp for each region server. key: tableName value: * RegionServer,PreviousTimeStamp * @throws IOException exception */ public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap() throws IOException { return systemTable.readLogTimestampMap(backupInfo.getBackupRootDir()); }
/** * Get history for backup destination * @param backupRoot backup destination path * @return List of backup info * @throws IOException if getting the backup history fails */ public List<BackupInfo> getBackupHistory(String backupRoot) throws IOException { ArrayList<BackupInfo> history = getBackupHistory(false); for (Iterator<BackupInfo> iterator = history.iterator(); iterator.hasNext();) { BackupInfo info = iterator.next(); if (!backupRoot.equals(info.getBackupRootDir())) { iterator.remove(); } } return history; }
/** * Get backup request meta data dir as string. * @param backupInfo backup info * @return meta data dir */ protected String obtainBackupMetaDataStr(BackupInfo backupInfo) { StringBuffer sb = new StringBuffer(); sb.append("type=" + backupInfo.getType() + ",tablelist="); for (TableName table : backupInfo.getTables()) { sb.append(table + ";"); } if (sb.lastIndexOf(";") > 0) { sb.delete(sb.lastIndexOf(";"), sb.lastIndexOf(";") + 1); } sb.append(",targetRootDir=" + backupInfo.getBackupRootDir()); return sb.toString(); }
/** * Clean up the data at target directory * @throws IOException if cleaning up the backup directory fails */ private void cleanupBackupDir(BackupInfo backupInfo, TableName table, Configuration conf) throws IOException { try { // clean up the data at target directory String targetDir = backupInfo.getBackupRootDir(); if (targetDir == null) { LOG.warn("No target directory specified for " + backupInfo.getBackupId()); return; } FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf); Path targetDirPath = new Path(BackupUtils.getTableBackupDir(backupInfo.getBackupRootDir(), backupInfo.getBackupId(), table)); if (outputFs.delete(targetDirPath, true)) { LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done."); } else { LOG.info("No data has been found in " + targetDirPath.toString() + "."); } } catch (IOException e1) { LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " for table " + table + "at " + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + "."); throw e1; } }
public Map<TableName, ArrayList<BackupInfo>> getBackupHistoryForTableSet(Set<TableName> set, String backupRoot) throws IOException { List<BackupInfo> history = getBackupHistory(backupRoot); Map<TableName, ArrayList<BackupInfo>> tableHistoryMap = new HashMap<>(); for (Iterator<BackupInfo> iterator = history.iterator(); iterator.hasNext();) { BackupInfo info = iterator.next(); if (!backupRoot.equals(info.getBackupRootDir())) { continue; } List<TableName> tables = info.getTableNames(); for (TableName tableName : tables) { if (set.contains(tableName)) { ArrayList<BackupInfo> list = tableHistoryMap.get(tableName); if (list == null) { list = new ArrayList<>(); tableHistoryMap.put(tableName, list); } list.add(info); } } } return tableHistoryMap; }
/** * Saves list of WAL files after incremental backup operation. These files will be stored until * TTL expiration and are used by Backup Log Cleaner plug-in to determine which WAL files can be * safely purged. */ public void recordWALFiles(List<String> files) throws IOException { systemTable.addWALFiles(files, backupInfo.getBackupId(), backupInfo.getBackupRootDir()); }
private List<BackupInfo> getAffectedBackupSessions(BackupInfo backupInfo, TableName tn, BackupSystemTable table) throws IOException { LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn); long ts = backupInfo.getStartTs(); List<BackupInfo> list = new ArrayList<>(); List<BackupInfo> history = table.getBackupHistory(backupInfo.getBackupRootDir()); // Scan from most recent to backupInfo // break when backupInfo reached for (BackupInfo info : history) { if (info.getStartTs() == ts) { break; } List<TableName> tables = info.getTableNames(); if (tables.contains(tn)) { BackupType bt = info.getType(); if (bt == BackupType.FULL) { // Clear list if we encounter FULL backup list.clear(); } else { LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn + " added " + info.getBackupId() + " tables=" + info.getTableListAsString()); list.add(info); } } } return list; }
/** * Construct a table level manifest for a backup of the named table. * @param backup The ongoing backup session info */ public BackupManifest(BackupInfo backup, TableName table) { this.tableBackupDir = backup.getTableBackupDir(table); List<TableName> tables = new ArrayList<TableName>(); tables.add(table); BackupImage.Builder builder = BackupImage.newBuilder(); this.backupImage = builder.withBackupId(backup.getBackupId()).withType(backup.getType()) .withRootDir(backup.getBackupRootDir()).withTableList(tables) .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); }
private boolean compare(BackupInfo one, BackupInfo two) { return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType()) && one.getBackupRootDir().equals(two.getBackupRootDir()) && one.getStartTs() == two.getStartTs() && one.getCompleteTs() == two.getCompleteTs(); }
/** * Verify that full backup is created on a single table with data correctly. Verify that history * works as expected. * * @throws Exception if doing the backup or an operation on the tables fails */ @Test public void testBackupDelete() throws Exception { LOG.info("test backup delete on a single table with data"); List<TableName> tableList = Lists.newArrayList(table1); String backupId = fullTableBackup(tableList); assertTrue(checkSucceeded(backupId)); LOG.info("backup complete"); String[] backupIds = new String[] { backupId }; BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection()); BackupInfo info = table.readBackupInfo(backupId); Path path = new Path(info.getBackupRootDir(), backupId); FileSystem fs = FileSystem.get(path.toUri(), conf1); assertTrue(fs.exists(path)); int deleted = getBackupAdmin().deleteBackups(backupIds); assertTrue(!fs.exists(path)); assertTrue(fs.exists(new Path(info.getBackupRootDir()))); assertTrue(1 == deleted); table.close(); LOG.info("delete_backup"); }
/** * Construct manifest for a ongoing backup. * @param backup The ongoing backup info */ public BackupManifest(BackupInfo backup) { BackupImage.Builder builder = BackupImage.newBuilder(); this.backupImage = builder.withBackupId(backup.getBackupId()).withType(backup.getType()) .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames()) .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); }
public BackupProtos.BackupInfo toProtosBackupInfo() { BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder(); builder.setBackupId(getBackupId()); setBackupTableInfoMap(builder); builder.setCompleteTs(getCompleteTs()); if (getFailedMsg() != null) { builder.setFailedMessage(getFailedMsg()); } if (getState() != null) { builder.setBackupState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name())); } if (getPhase() != null) { builder.setBackupPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name())); } builder.setProgress(getProgress()); builder.setStartTs(getStartTs()); builder.setBackupRootDir(getBackupRootDir()); builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name())); builder.setWorkersNumber(workers); builder.setBandwidth(bandwidth); return builder.build(); }