public static String getMetadataTableDir() { return getTablesDir() + "/" + METADATA_TABLE_ID; }
private static String rel2abs(String relPath, KeyExtent extent) { if (relPath.startsWith("../")) return ServerConstants.getTablesDir() + relPath.substring(2); else return ServerConstants.getTablesDir() + "/" + extent.getTableId() + relPath; }
private void removeFilesAfterScanRel(Set<String> relPaths) { Set<Path> scanFiles = new HashSet<Path>(); for (String rpath : relPaths) scanFiles.add(new Path(ServerConstants.getTablesDir() + "/" + extent.getTableId() + rpath)); removeFilesAfterScan(scanFiles); }
private static void createDistcpFile(FileSystem fs, String exportDir, Path exportMetaFilePath, Map<String,String> uniqueFiles) throws IOException { BufferedWriter distcpOut = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(exportDir, "distcp.txt"), false), UTF_8)); try { URI uri = fs.getUri(); for (String relPath : uniqueFiles.values()) { Path absPath = new Path(uri.getScheme(), uri.getAuthority(), ServerConstants.getTablesDir() + relPath); distcpOut.append(absPath.toUri().toString()); distcpOut.newLine(); } Path absEMP = exportMetaFilePath; if (!exportMetaFilePath.isAbsolute()) absEMP = new Path(fs.getWorkingDirectory().toUri().getPath(), exportMetaFilePath); distcpOut.append(new Path(uri.getScheme(), uri.getAuthority(), absEMP.toString()).toUri().toString()); distcpOut.newLine(); distcpOut.close(); distcpOut = null; } finally { if (distcpOut != null) distcpOut.close(); } }
@Override public Repo<Master> call(long tid, Master environment) throws Exception { UniqueNameAllocator namer = UniqueNameAllocator.getInstance(); Path directory = new Path(ServerConstants.getTablesDir() + "/" + tableInfo.tableId); Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName()); tableInfo.importDir = newBulkDir.toString(); return new MapImportFileNames(tableInfo); } }
@Override public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException { if (args.length != 2) { System.out.println("Usage: CountRowKeys tableName outputPath"); return 1; } Job job = new Job(getConf(), this.getClass().getName()); job.setJarByClass(this.getClass()); job.setInputFormatClass(SequenceFileInputFormat.class); SequenceFileInputFormat.addInputPath(job, new Path(ServerConstants.getTablesDir() + "/" + args[0] + "/*/*/data")); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(NullWritable.class); job.setReducerClass(MyReducer.class); TextOutputFormat.setOutputPath(job, new Path(args[1])); job.waitForCompletion(true); return job.isSuccessful() ? 0 : 1; }
private void addBlocks(FileSystem fs, String host, ArrayList<String> files, Map<String,Long> totalBlocks, Map<String,Long> localBlocks) throws Exception { long allBlocks = 0; long matchingBlocks = 0; if (!totalBlocks.containsKey(host)) { totalBlocks.put(host, 0L); localBlocks.put(host, 0L); } for (String file : files) { Path filePath = new Path(ServerConstants.getTablesDir() + "/" + file); FileStatus fileStatus = fs.getFileStatus(filePath); BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); for (BlockLocation blockLocation : fileBlockLocations) { allBlocks++; for (String location : blockLocation.getHosts()) { InetSocketAddress inetSocketAddress = new InetSocketAddress(location, 0); if (inetSocketAddress.getAddress().getHostAddress().equals(host)) { matchingBlocks++; break; } } } } totalBlocks.put(host, allBlocks + totalBlocks.get(host)); localBlocks.put(host, matchingBlocks + localBlocks.get(host)); }
private Path createNewBulkDir(FileSystem fs, String tableId) throws IOException { Path directory = new Path(ServerConstants.getTablesDir() + "/" + tableId); fs.mkdirs(directory); // only one should be able to create the lock file // the purpose of the lock file is to avoid a race // condition between the call to fs.exists() and // fs.mkdirs()... if only hadoop had a mkdir() function // that failed when the dir existed UniqueNameAllocator namer = UniqueNameAllocator.getInstance(); while (true) { Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName()); if (fs.exists(newBulkDir)) // sanity check throw new IllegalStateException("Dir exist when it should not " + newBulkDir); if (fs.mkdirs(newBulkDir)) return newBulkDir; log.warn("Failed to create " + newBulkDir + " for unknown reason"); UtilWaitThread.sleep(3000); } }
try { for (String validExtension : FileOperations.getValidExtensions()) { for (FileStatus stat : fs.globStatus(new Path(ServerConstants.getTablesDir() + "/*/*/*." + validExtension))) { String cand = stat.getPath().toUri().getPath(); if (!cand.contains(ServerConstants.getRootTabletDir())) { candidates.add(cand.substring(ServerConstants.getTablesDir().length())); log.debug("Offline candidate: " + cand);
tabletDirs = fs.listStatus(new Path(ServerConstants.getTablesDir() + "/" + delTableId)); } catch (FileNotFoundException ex) { Path p = new Path(ServerConstants.getTablesDir() + "/" + delTableId); if (!moveToTrash(p)) fs.delete(p, false);
private static int addUnknownFiles(FileSystem fs, String directory, Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean update) throws Exception { int count = 0; final String tableId = ke.getTableId().toString(); final Text row = ke.getMetadataEntry(); log.info(row.toString()); final Path path = new Path(ServerConstants.getTablesDir() + "/" + tableId + directory); for (FileStatus file : fs.listStatus(path)) { if (file.getPath().getName().endsWith("_tmp") || file.getPath().getName().endsWith("_tmp.rf")) continue; final String filename = directory + "/" + file.getPath().getName(); if (!knownFiles.contains(filename)) { count++; final Mutation m = new Mutation(row); String size = Long.toString(file.getLen()); String entries = "1"; // lie String value = size + "," + entries; m.put(Constants.METADATA_DATAFILE_COLUMN_FAMILY, new Text(filename), new Value(value.getBytes(UTF_8))); if (update) { writer.getBatchWriter(Constants.METADATA_TABLE_NAME).addMutation(m); } } } return count; }
@Override public void undo(long tid, Master environment) throws Exception { FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration())); String dir = ServerConstants.getTablesDir() + "/" + tableInfo.tableId; fs.delete(new Path(dir), true); } }
boolean removeFlag; String fullPath = ServerConstants.getTablesDir() + delete; log.debug("Deleting " + fullPath); try {
if (!tpath.getParent().getParent().equals(new Path(ServerConstants.getTablesDir() + "/" + extent.getTableId()))) { throw new IOException("Map file " + tpath + " not in table dir " + ServerConstants.getTablesDir() + "/" + extent.getTableId()); if (paths.keySet().remove(new Path(ServerConstants.getTablesDir() + "/" + extent.getTableId() + file))) log.debug("Ignoring request to re-import a file already imported: " + extent + ": " + file);
private static SortedMap<String,DataFileValue> lookupDatafiles(AccumuloConfiguration conf, Text locText, FileSystem fs, KeyExtent extent, SortedMap<Key,Value> tabletsKeyValues) throws IOException { Path location = new Path(ServerConstants.getTablesDir() + "/" + extent.getTableId().toString() + locText.toString());
@Override public Repo<Master> call(long tid, Master environment) throws Exception { FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration())); String dir = ServerConstants.getTablesDir() + "/" + tableInfo.tableId; TabletOperations.createTabletDirectory(fs, dir, null); return new PopulateMetadata(tableInfo); }
if (!file.startsWith("/")) file = "/" + file; Path map = new Path(ServerConstants.getTablesDir() + "/" + table + file); if (!fs.exists(map)) { missing++;
fs.delete(new Path(ServerConstants.getTablesDir(), tableId), true); } catch (IOException e) { log.error("Unable to remove deleted table directory", e);
final FileSystem fs, final List<LogEntry> logEntries, final SortedMap<String,DataFileValue> datafiles, String time, final TServerInstance lastLocation, Set<String> scanFiles, long initFlushID, long initCompactID) throws IOException { this.location = new Path(ServerConstants.getTablesDir() + "/" + extent.getTableId().toString() + location.toString()); this.lastLocation = lastLocation; this.tabletDirectory = location.toString();