public CrailBenchmark(int warmup) throws Exception { this.warmup = warmup; this.conf = new CrailConfiguration(); this.fs = null; }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(NAMENODE_TCP_QUEUEDEPTH_KEY) != null) { NAMENODE_TCP_QUEUEDEPTH = Integer.parseInt(conf.get(NAMENODE_TCP_QUEUEDEPTH_KEY)); } if (conf.get(NAMENODE_TCP_MESSAGESIZE_KEY) != null) { NAMENODE_TCP_MESSAGESIZE = Integer.parseInt(conf.get(NAMENODE_TCP_MESSAGESIZE_KEY)); } }
public CrailConfiguration() throws IOException{ conf = new ConcurrentHashMap<>(); Properties properties = loadProperties("crail-site.conf"); mergeProperties(properties); }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(NAMENODE_DARPC_POLLING_KEY) != null) { NAMENODE_DARPC_POLLING = conf.getBoolean(NAMENODE_DARPC_POLLING_KEY, false); } if (conf.get(NAMENODE_DARPC_TYPE_KEY) != null) { NAMENODE_DARPC_TYPE = conf.get(NAMENODE_DARPC_TYPE_KEY); } if (conf.get(NAMENODE_DARPC_AFFINITY_KEY) != null) { NAMENODE_DARPC_AFFINITY = conf.get(NAMENODE_DARPC_AFFINITY_KEY); } if (conf.get(NAMENODE_DARPC_MAXINLINE_KEY) != null) { NAMENODE_DARPC_MAXINLINE = Integer.parseInt(conf.get(NAMENODE_DARPC_MAXINLINE_KEY)); } if (conf.get(NAMENODE_DARPC_RECVQUEUE_KEY) != null) { NAMENODE_DARPC_RECVQUEUE = Integer.parseInt(conf.get(NAMENODE_DARPC_RECVQUEUE_KEY)); } if (conf.get(NAMENODE_DARPC_SENDQUEUE_KEY) != null) { NAMENODE_DARPC_SENDQUEUE = Integer.parseInt(conf.get(NAMENODE_DARPC_SENDQUEUE_KEY)); } if (conf.get(NAMENODE_DARPC_POLLSIZE_KEY) != null) { NAMENODE_DARPC_POLLSIZE = Integer.parseInt(conf.get(NAMENODE_DARPC_POLLSIZE_KEY)); } if (conf.get(NAMENODE_DARPC_CLUSTERSIZE_KEY) != null) { NAMENODE_DARPC_CLUSTERSIZE = Integer.parseInt(conf.get(NAMENODE_DARPC_CLUSTERSIZE_KEY)); } }
public static void main(String args[]) throws Exception { LOG.info("initalizing namenode "); CrailConfiguration conf = new CrailConfiguration(); CrailConstants.updateConstants(conf); CrailConstants.printConf(); CrailConstants.verify();
public void ping() throws Exception { CrailConfiguration conf = new CrailConfiguration(); CrailConstants.updateConstants(conf); CoreFileSystem fs = new CoreFileSystem(conf); fs.ping(); fs.closeFileSystem(); }
public static CrailFS newInstance(CrailConfiguration conf) throws Exception { synchronized(referenceCounter){ boolean isSingleton = conf.getBoolean(CrailConstants.SINGLETON_KEY, false); if (isSingleton) { referenceCounter.incrementAndGet(); if (instance == null) { LOG.info("creating singleton crail file system"); instance = new CoreFileSystem(conf); return instance; } else { LOG.info("returning singleton crail file system"); return instance; } } else { LOG.info("creating non-singleton crail file system"); return new CoreFileSystem(conf); } } } }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(STORAGE_RDMA_INTERFACE_KEY) != null) { STORAGE_RDMA_INTERFACE = conf.get(STORAGE_RDMA_INTERFACE_KEY); } if (conf.get(STORAGE_RDMA_PORT_KEY) != null) { STORAGE_RDMA_PORT = Integer.parseInt(conf.get(STORAGE_RDMA_PORT_KEY)); } if (conf.get(STORAGE_RDMA_STORAGE_LIMIT_KEY) != null) { STORAGE_RDMA_STORAGE_LIMIT = Long.parseLong(conf.get(STORAGE_RDMA_STORAGE_LIMIT_KEY)); } if (conf.get(STORAGE_RDMA_ALLOCATION_SIZE_KEY) != null) { STORAGE_RDMA_ALLOCATION_SIZE = Long.parseLong(conf.get(STORAGE_RDMA_ALLOCATION_SIZE_KEY)); } if (conf.get(STORAGE_RDMA_DATA_PATH_KEY) != null) { STORAGE_RDMA_DATA_PATH = conf.get(STORAGE_RDMA_DATA_PATH_KEY); } if (conf.get(STORAGE_RDMA_LOCAL_MAP_KEY) != null) { STORAGE_RDMA_LOCAL_MAP = conf.getBoolean(STORAGE_RDMA_LOCAL_MAP_KEY, false); } if (conf.get(STORAGE_RDMA_QUEUESIZE_KEY) != null) { STORAGE_RDMA_QUEUESIZE = Integer.parseInt(conf.get(STORAGE_RDMA_QUEUESIZE_KEY)); } if (conf.get(STORAGE_RDMA_TYPE_KEY) != null) { STORAGE_RDMA_TYPE = conf.get(STORAGE_RDMA_TYPE_KEY); } if (conf.get(STORAGE_RDMA_PERSISTENT_KEY) != null) { STORAGE_RDMA_PERSISTENT = conf.getBoolean(STORAGE_RDMA_PERSISTENT_KEY, false); } }
public CrailHDFS(final URI uri, final Configuration conf) throws IOException, URISyntaxException { super(uri, "crail", true, 9000); try { CrailConfiguration crailConf = new CrailConfiguration(); this.dfs = CrailFS.newInstance(crailConf); Path _workingDir = new Path("/user/" + CrailConstants.USER); this.workingDir = new Path("/user/" + CrailConstants.USER).makeQualified(uri, _workingDir); LOG.info("CrailHDFS initialization done.."); } catch(Exception e){ throw new IOException(e); } }
public void directoryDump(String filename, boolean randomize) throws Exception { CrailConfiguration conf = new CrailConfiguration(); CrailConstants.updateConstants(conf); CoreFileSystem fs = new CoreFileSystem(conf); DirectoryInputStream iter = fs._listEntries(filename, randomize); System.out.println("#hash \t\tname\t\tfilecomponent"); int i = 0; while(iter.hasRecord()){ DirectoryRecord record = iter.nextRecord(); String path = CrailUtils.combinePath(record.getParent(), record.getFile()); FileName hash = new FileName(path); System.out.format(i + ": " + "%08d\t\t%s\t%d\n", record.isValid() ? 1 : 0, padRight(record.getFile(), 8), hash.getFileComponent()); i++; } iter.close(); fs.closeFileSystem(); }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(STORAGE_TCP_INTERFACE_KEY) != null) { STORAGE_TCP_INTERFACE = conf.get(STORAGE_TCP_INTERFACE_KEY); } if (conf.get(STORAGE_TCP_PORT_KEY) != null) { STORAGE_TCP_PORT = Integer.parseInt(conf.get(STORAGE_TCP_PORT_KEY)); } if (conf.get(STORAGE_TCP_STORAGE_LIMIT_KEY) != null) { STORAGE_TCP_STORAGE_LIMIT = Long.parseLong(conf.get(STORAGE_TCP_STORAGE_LIMIT_KEY)); } if (conf.get(STORAGE_TCP_ALLOCATION_SIZE_KEY) != null) { STORAGE_TCP_ALLOCATION_SIZE = Integer.parseInt(conf.get(STORAGE_TCP_ALLOCATION_SIZE_KEY)); } if (conf.get(STORAGE_TCP_DATA_PATH_KEY) != null) { STORAGE_TCP_DATA_PATH = conf.get(STORAGE_TCP_DATA_PATH_KEY); } if (conf.get(STORAGE_TCP_QUEUE_DEPTH_KEY) != null) { STORAGE_TCP_QUEUE_DEPTH = Integer.parseInt(conf.get(STORAGE_TCP_QUEUE_DEPTH_KEY)); } }
if (conf.get(DIRECTORY_DEPTH_KEY) != null) { DIRECTORY_DEPTH = Integer.parseInt(conf.get(DIRECTORY_DEPTH_KEY)); if (conf.get(TOKEN_EXPIRATION_KEY) != null) { TOKEN_EXPIRATION = Long.parseLong(conf.get(TOKEN_EXPIRATION_KEY)); if (conf.get(BLOCK_SIZE_KEY) != null) { BLOCK_SIZE = Long.parseLong(conf.get(BLOCK_SIZE_KEY)); if (conf.get(CACHE_LIMIT_KEY) != null) { CACHE_LIMIT = Long.parseLong(conf.get(CACHE_LIMIT_KEY)); if (conf.get(CACHE_PATH_KEY) != null) { CACHE_PATH = conf.get(CACHE_PATH_KEY); if (conf.get(USER_KEY) != null) { USER = conf.get(CrailConstants.USER_KEY); if (conf.get(SHADOW_REPLICATION_KEY) != null) { SHADOW_REPLICATION = Integer.parseInt(conf.get(SHADOW_REPLICATION_KEY)); if (conf.get(DEBUG_KEY) != null) { DEBUG = Boolean.parseBoolean(conf.get(DEBUG_KEY)); if (conf.get(STATISTICS_KEY) != null) { STATISTICS = Boolean.parseBoolean(conf.get(STATISTICS_KEY)); if (conf.get(RPC_TIMEOUT_KEY) != null) { RPC_TIMEOUT = Integer.parseInt(conf.get(RPC_TIMEOUT_KEY));
public void namenodeDump() throws Exception { CrailConfiguration conf = new CrailConfiguration(); CoreFileSystem fs = new CoreFileSystem(conf); fs.dumpNameNode(); fs.close(); }
private static String get(CrailConfiguration conf, String key) { return conf.get(fullKey(key)); }
@Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); setConf(conf); try { CrailConfiguration crailConf = new CrailConfiguration(); this.dfs = CrailFS.newInstance(crailConf); Path _workingDir = new Path("/user/" + CrailConstants.USER); this.workingDir = new Path("/user/" + CrailConstants.USER).makeQualified(uri, _workingDir); this.uri = URI.create(CrailConstants.NAMENODE_ADDRESS); LOG.info("CrailHadoopFileSystem fs initialization done.."); } catch(Exception e){ throw new IOException(e); } }
public void getLocations(String filename, long offset, long length) throws Exception { System.out.println("getLocations, filename " + filename + ", offset " + offset + ", len " + length); CrailConfiguration conf = new CrailConfiguration(); CrailFS fs = CrailFS.newInstance(conf); CrailBlockLocation locations[] = fs.lookup(filename).get().getBlockLocations(offset, length); for (int i = 0; i < locations.length; i++){ System.out.println("location " + i + " : " + locations[i].toString()); } fs.close(); }
public void createDirectory(String filename, int storageClass, int locationClass) throws Exception { System.out.println("createDirectory, filename " + filename + ", storageClass " + storageClass + ", locationClass " + locationClass); CrailConfiguration conf = new CrailConfiguration(); CrailFS fs = CrailFS.newInstance(conf); fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass)).get().syncDir(); fs.close(); }
@Before public void init() throws Exception { CrailConfiguration conf = new CrailConfiguration(); fs = CrailFS.newInstance(conf); fs.create(basePath, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get(); }
public void blockStatistics(String filename) throws Exception { HashMap<String, AtomicInteger> stats = new HashMap<String, AtomicInteger>(); CrailConfiguration conf = new CrailConfiguration(); CrailFS fs = CrailFS.newInstance(conf); CrailNode node = fs.lookup(filename).get(); if (node.getType() == CrailNodeType.DIRECTORY){ CrailDirectory directory = node.asDirectory(); Iterator<String> iter = directory.listEntries(); while (iter.hasNext()) { String path = iter.next(); CrailFile child = fs.lookup(path).get().asFile(); walkBlocks(stats, fs, child.getPath(), 0, child.getCapacity()); } } else if (node.getType() == CrailNodeType.DATAFILE){ CrailFile file = node.asFile(); walkBlocks(stats, fs, file.getPath(), 0, file.getCapacity()); } else if (node.getType() == CrailNodeType.MULTIFILE){ CrailMultiFile directory = node.asMultiFile(); Iterator<String> iter = directory.listEntries(); while (iter.hasNext()) { String path = iter.next(); CrailFile child = fs.lookup(path).get().asFile(); walkBlocks(stats, fs, child.getPath(), 0, child.getCapacity()); } } printStats(stats); fs.close(); }