public CrailBenchmark(int warmup) throws Exception { this.warmup = warmup; this.conf = new CrailConfiguration(); this.fs = null; }
public CrailConfiguration() throws IOException { conf = new ConcurrentHashMap<>(); Properties properties = loadProperties("crail-site.conf"); mergeProperties(properties); }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(NAMENODE_TCP_QUEUEDEPTH_KEY) != null) { NAMENODE_TCP_QUEUEDEPTH = Integer.parseInt(conf.get(NAMENODE_TCP_QUEUEDEPTH_KEY)); } if (conf.get(NAMENODE_TCP_MESSAGESIZE_KEY) != null) { NAMENODE_TCP_MESSAGESIZE = Integer.parseInt(conf.get(NAMENODE_TCP_MESSAGESIZE_KEY)); } if (conf.get(NAMENODE_TCP_CORES_KEY) != null) { NAMENODE_TCP_CORES = Integer.parseInt(conf.get(NAMENODE_TCP_CORES_KEY)); } }
public void ping() throws Exception { CrailConfiguration conf = new CrailConfiguration(); CrailConstants.updateConstants(conf); CoreDataStore fs = new CoreDataStore(conf); fs.ping(); fs.closeFileSystem(); }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(NAMENODE_DARPC_POLLING_KEY) != null) { NAMENODE_DARPC_POLLING = conf.getBoolean(NAMENODE_DARPC_POLLING_KEY, false); if (conf.get(NAMENODE_DARPC_TYPE_KEY) != null) { NAMENODE_DARPC_TYPE = conf.get(NAMENODE_DARPC_TYPE_KEY); if (conf.get(NAMENODE_DARPC_AFFINITY_KEY) != null) { NAMENODE_DARPC_AFFINITY = conf.get(NAMENODE_DARPC_AFFINITY_KEY); if (conf.get(NAMENODE_DARPC_MAXINLINE_KEY) != null) { NAMENODE_DARPC_MAXINLINE = Integer.parseInt(conf.get(NAMENODE_DARPC_MAXINLINE_KEY)); if (conf.get(NAMENODE_DARPC_RECVQUEUE_KEY) != null) { NAMENODE_DARPC_RECVQUEUE = Integer.parseInt(conf.get(NAMENODE_DARPC_RECVQUEUE_KEY)); if (conf.get(NAMENODE_DARPC_SENDQUEUE_KEY) != null) { NAMENODE_DARPC_SENDQUEUE = Integer.parseInt(conf.get(NAMENODE_DARPC_SENDQUEUE_KEY)); if (conf.get(NAMENODE_DARPC_POLLSIZE_KEY) != null) { NAMENODE_DARPC_POLLSIZE = Integer.parseInt(conf.get(NAMENODE_DARPC_POLLSIZE_KEY)); if (conf.get(NAMENODE_DARPC_CLUSTERSIZE_KEY) != null) { NAMENODE_DARPC_CLUSTERSIZE = Integer.parseInt(conf.get(NAMENODE_DARPC_CLUSTERSIZE_KEY)); if (conf.get(NAMENODE_DARPC_BACKLOG_KEY) != null) { NAMENODE_DARPC_BACKLOG = Integer.parseInt(conf.get(NAMENODE_DARPC_BACKLOG_KEY)); if (conf.get(NAMENODE_DARPC_CONNECTTIMEOUT_KEY) != null) { NAMENODE_DARPC_CONNECTTIMEOUT = Integer.parseInt(conf.get(NAMENODE_DARPC_CONNECTTIMEOUT_KEY));
public static CrailStore newInstance(CrailConfiguration conf) throws Exception { synchronized(referenceCounter){ boolean isSingleton = conf.getBoolean(CrailConstants.SINGLETON_KEY, CrailConstants.SINGLETON); if (isSingleton) { referenceCounter.incrementAndGet(); if (instance == null) { LOG.info("creating singleton crail file system"); instance = new CoreDataStore(conf); return instance; } else { LOG.info("returning singleton crail file system"); return instance; } } else { LOG.info("creating non-singleton crail file system"); return new CoreDataStore(conf); } } } }
public void ping() throws Exception { CrailConfiguration conf = new CrailConfiguration(); CrailConstants.updateConstants(conf); CoreDataStore fs = new CoreDataStore(conf); fs.ping(); fs.closeFileSystem(); }
public CrailBenchmark(int warmup) throws Exception { this.warmup = warmup; this.conf = new CrailConfiguration(); this.fs = null; }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(STORAGE_TCP_INTERFACE_KEY) != null) { STORAGE_TCP_INTERFACE = conf.get(STORAGE_TCP_INTERFACE_KEY); } if (conf.get(STORAGE_TCP_PORT_KEY) != null) { STORAGE_TCP_PORT = Integer.parseInt(conf.get(STORAGE_TCP_PORT_KEY)); } if (conf.get(STORAGE_TCP_STORAGE_LIMIT_KEY) != null) { STORAGE_TCP_STORAGE_LIMIT = Long.parseLong(conf.get(STORAGE_TCP_STORAGE_LIMIT_KEY)); } if (conf.get(STORAGE_TCP_ALLOCATION_SIZE_KEY) != null) { STORAGE_TCP_ALLOCATION_SIZE = Integer.parseInt(conf.get(STORAGE_TCP_ALLOCATION_SIZE_KEY)); } if (conf.get(STORAGE_TCP_DATA_PATH_KEY) != null) { STORAGE_TCP_DATA_PATH = conf.get(STORAGE_TCP_DATA_PATH_KEY); } if (conf.get(STORAGE_TCP_QUEUE_DEPTH_KEY) != null) { STORAGE_TCP_QUEUE_DEPTH = Integer.parseInt(conf.get(STORAGE_TCP_QUEUE_DEPTH_KEY)); } if (conf.get(STORAGE_TCP_CORES_KEY) != null) { STORAGE_TCP_CORES = Integer.parseInt(conf.get(STORAGE_TCP_CORES_KEY)); } }
public static CrailStore newInstance(CrailConfiguration conf) throws Exception { synchronized(referenceCounter){ boolean isSingleton = conf.getBoolean(CrailConstants.SINGLETON_KEY, CrailConstants.SINGLETON); if (isSingleton) { referenceCounter.incrementAndGet(); if (instance == null) { LOG.info("creating singleton crail file system"); instance = new CoreDataStore(conf); return instance; } else { LOG.info("returning singleton crail file system"); return instance; } } else { LOG.info("creating non-singleton crail file system"); return new CoreDataStore(conf); } } } }
public CrailConfiguration() throws IOException { conf = new ConcurrentHashMap<>(); Properties properties = loadProperties("crail-site.conf"); mergeProperties(properties); }
@BeforeClass public static void init() throws IOException { CrailConstants.updateConstants(new CrailConfiguration()); }
public void namenodeDump() throws Exception { CrailConfiguration conf = new CrailConfiguration(); CoreDataStore fs = new CoreDataStore(conf); fs.dumpNameNode(); fs.close(); }
private static String get(CrailConfiguration conf, String key) { return conf.get(fullKey(key)); }
public void directoryDump(String filename, boolean randomize) throws Exception { CrailConfiguration conf = new CrailConfiguration(); CrailConstants.updateConstants(conf); CoreDataStore fs = new CoreDataStore(conf); DirectoryInputStream iter = fs._listEntries(filename, randomize); System.out.println("#hash \t\tname\t\tfilecomponent"); int i = 0; while(iter.hasRecord()){ DirectoryRecord record = iter.nextRecord(); String path = CrailUtils.combinePath(record.getParent(), record.getFile()); FileName hash = new FileName(path); System.out.format(i + ": " + "%08d\t\t%s\t%d\n", record.isValid() ? 1 : 0, padRight(record.getFile(), 8), hash.getFileComponent()); i++; } iter.close(); fs.closeFileSystem(); }
public void namenodeDump() throws Exception { CrailConfiguration conf = new CrailConfiguration(); CoreDataStore fs = new CoreDataStore(conf); fs.dumpNameNode(); fs.close(); }
public void directoryDump(String filename, boolean randomize) throws Exception { CrailConfiguration conf = new CrailConfiguration(); CrailConstants.updateConstants(conf); CoreDataStore fs = new CoreDataStore(conf); DirectoryInputStream iter = fs._listEntries(filename, randomize); System.out.println("#hash \t\tname\t\tfilecomponent"); int i = 0; while(iter.hasRecord()){ DirectoryRecord record = iter.nextRecord(); String path = CrailUtils.combinePath(record.getParent(), record.getFile()); FileName hash = new FileName(path); System.out.format(i + ": " + "%08d\t\t%s\t%d\n", record.isValid() ? 1 : 0, padRight(record.getFile(), 8), hash.getFileComponent()); i++; } iter.close(); fs.closeFileSystem(); }
public void getLocations(String filename, long offset, long length) throws Exception { System.out.println("getLocations, filename " + filename + ", offset " + offset + ", len " + length); CrailConfiguration conf = new CrailConfiguration(); CrailStore fs = CrailStore.newInstance(conf); CrailBlockLocation locations[] = fs.lookup(filename).get().getBlockLocations(offset, length); for (int i = 0; i < locations.length; i++){ System.out.println("location " + i + " : " + locations[i].toString()); } fs.close(); }
public void getLocations(String filename, long offset, long length) throws Exception { System.out.println("getLocations, filename " + filename + ", offset " + offset + ", len " + length); CrailConfiguration conf = new CrailConfiguration(); CrailStore fs = CrailStore.newInstance(conf); CrailBlockLocation locations[] = fs.lookup(filename).get().getBlockLocations(offset, length); for (int i = 0; i < locations.length; i++){ System.out.println("location " + i + " : " + locations[i].toString()); } fs.close(); }
@Before public void init() throws Exception { CrailConfiguration conf = new CrailConfiguration(); fs = CrailStore.newInstance(conf); fs.create(basePath, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT, true).get(); }