public CrailBenchmark(int warmup) throws Exception { this.warmup = warmup; this.conf = new CrailConfiguration(); this.fs = null; }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(NAMENODE_DARPC_POLLING_KEY) != null) { NAMENODE_DARPC_POLLING = conf.getBoolean(NAMENODE_DARPC_POLLING_KEY, false); if (conf.get(NAMENODE_DARPC_TYPE_KEY) != null) { NAMENODE_DARPC_TYPE = conf.get(NAMENODE_DARPC_TYPE_KEY); if (conf.get(NAMENODE_DARPC_AFFINITY_KEY) != null) { NAMENODE_DARPC_AFFINITY = conf.get(NAMENODE_DARPC_AFFINITY_KEY); if (conf.get(NAMENODE_DARPC_MAXINLINE_KEY) != null) { NAMENODE_DARPC_MAXINLINE = Integer.parseInt(conf.get(NAMENODE_DARPC_MAXINLINE_KEY)); if (conf.get(NAMENODE_DARPC_RECVQUEUE_KEY) != null) { NAMENODE_DARPC_RECVQUEUE = Integer.parseInt(conf.get(NAMENODE_DARPC_RECVQUEUE_KEY)); if (conf.get(NAMENODE_DARPC_SENDQUEUE_KEY) != null) { NAMENODE_DARPC_SENDQUEUE = Integer.parseInt(conf.get(NAMENODE_DARPC_SENDQUEUE_KEY)); if (conf.get(NAMENODE_DARPC_POLLSIZE_KEY) != null) { NAMENODE_DARPC_POLLSIZE = Integer.parseInt(conf.get(NAMENODE_DARPC_POLLSIZE_KEY)); if (conf.get(NAMENODE_DARPC_CLUSTERSIZE_KEY) != null) { NAMENODE_DARPC_CLUSTERSIZE = Integer.parseInt(conf.get(NAMENODE_DARPC_CLUSTERSIZE_KEY)); if (conf.get(NAMENODE_DARPC_BACKLOG_KEY) != null) { NAMENODE_DARPC_BACKLOG = Integer.parseInt(conf.get(NAMENODE_DARPC_BACKLOG_KEY)); if (conf.get(NAMENODE_DARPC_CONNECTTIMEOUT_KEY) != null) { NAMENODE_DARPC_CONNECTTIMEOUT = Integer.parseInt(conf.get(NAMENODE_DARPC_CONNECTTIMEOUT_KEY));
private static Properties loadProperties(String resourceName) throws IOException { Properties properties = new Properties(); String base = System.getenv("CRAIL_HOME"); if (base == null || base.isEmpty()) { throw new IllegalArgumentException("CRAIL_HOME environment variable is not set or empty"); } FileInputStream inputStream = new FileInputStream(new File(base + "/conf/" + resourceName)); try { properties.load(inputStream); } finally { inputStream.close(); } for (String key : properties.stringPropertyNames()) { String val = properties.getProperty(key); properties.setProperty(key, expandEnvVars(val)); } return properties; }
public CrailConfiguration() throws IOException { conf = new ConcurrentHashMap<>(); Properties properties = loadProperties("crail-site.conf"); mergeProperties(properties); }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(NAMENODE_TCP_QUEUEDEPTH_KEY) != null) { NAMENODE_TCP_QUEUEDEPTH = Integer.parseInt(conf.get(NAMENODE_TCP_QUEUEDEPTH_KEY)); } if (conf.get(NAMENODE_TCP_MESSAGESIZE_KEY) != null) { NAMENODE_TCP_MESSAGESIZE = Integer.parseInt(conf.get(NAMENODE_TCP_MESSAGESIZE_KEY)); } if (conf.get(NAMENODE_TCP_CORES_KEY) != null) { NAMENODE_TCP_CORES = Integer.parseInt(conf.get(NAMENODE_TCP_CORES_KEY)); } }
public static CrailStore newInstance(CrailConfiguration conf) throws Exception { synchronized(referenceCounter){ boolean isSingleton = conf.getBoolean(CrailConstants.SINGLETON_KEY, CrailConstants.SINGLETON); if (isSingleton) { referenceCounter.incrementAndGet(); if (instance == null) { LOG.info("creating singleton crail file system"); instance = new CoreDataStore(conf); return instance; } else { LOG.info("returning singleton crail file system"); return instance; } } else { LOG.info("creating non-singleton crail file system"); return new CoreDataStore(conf); } } } }
public static void init(CrailConfiguration conf, String[] args) throws Exception { if (args != null) { Option portOption = Option.builder("p").desc("port to start server on").hasArg().build(); Option coresOption = Option.builder("c").desc("number of cores to use").hasArg().build(); Options options = new Options(); options.addOption(portOption); options.addOption(coresOption); CommandLineParser parser = new DefaultParser(); try { CommandLine line = parser.parse(options, Arrays.copyOfRange(args, 0, args.length)); if (line.hasOption(portOption.getOpt())) { String port = line.getOptionValue(portOption.getOpt()); LOG.info("using custom port " + port); conf.set(TcpStorageConstants.STORAGE_TCP_PORT_KEY, port); } if (line.hasOption(coresOption.getOpt())) { String cores = line.getOptionValue(coresOption.getOpt()); LOG.info("number of cores used is " + cores); conf.set(TcpStorageConstants.STORAGE_TCP_CORES_KEY, cores); } } catch (ParseException e) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("RDMA storage tier", options); System.exit(-1); } } TcpStorageConstants.updateConstants(conf); }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(STORAGE_TCP_INTERFACE_KEY) != null) { STORAGE_TCP_INTERFACE = conf.get(STORAGE_TCP_INTERFACE_KEY); } if (conf.get(STORAGE_TCP_PORT_KEY) != null) { STORAGE_TCP_PORT = Integer.parseInt(conf.get(STORAGE_TCP_PORT_KEY)); } if (conf.get(STORAGE_TCP_STORAGE_LIMIT_KEY) != null) { STORAGE_TCP_STORAGE_LIMIT = Long.parseLong(conf.get(STORAGE_TCP_STORAGE_LIMIT_KEY)); } if (conf.get(STORAGE_TCP_ALLOCATION_SIZE_KEY) != null) { STORAGE_TCP_ALLOCATION_SIZE = Integer.parseInt(conf.get(STORAGE_TCP_ALLOCATION_SIZE_KEY)); } if (conf.get(STORAGE_TCP_DATA_PATH_KEY) != null) { STORAGE_TCP_DATA_PATH = conf.get(STORAGE_TCP_DATA_PATH_KEY); } if (conf.get(STORAGE_TCP_QUEUE_DEPTH_KEY) != null) { STORAGE_TCP_QUEUE_DEPTH = Integer.parseInt(conf.get(STORAGE_TCP_QUEUE_DEPTH_KEY)); } if (conf.get(STORAGE_TCP_CORES_KEY) != null) { STORAGE_TCP_CORES = Integer.parseInt(conf.get(STORAGE_TCP_CORES_KEY)); } }
public static CrailStore newInstance(CrailConfiguration conf) throws Exception { synchronized(referenceCounter){ boolean isSingleton = conf.getBoolean(CrailConstants.SINGLETON_KEY, CrailConstants.SINGLETON); if (isSingleton) { referenceCounter.incrementAndGet(); if (instance == null) { LOG.info("creating singleton crail file system"); instance = new CoreDataStore(conf); return instance; } else { LOG.info("returning singleton crail file system"); return instance; } } else { LOG.info("creating non-singleton crail file system"); return new CoreDataStore(conf); } } } }
public CrailConfiguration() throws IOException { conf = new ConcurrentHashMap<>(); Properties properties = loadProperties("crail-site.conf"); mergeProperties(properties); }
public CrailBenchmark(int warmup) throws Exception { this.warmup = warmup; this.conf = new CrailConfiguration(); this.fs = null; }
public static void updateConstants(CrailConfiguration conf){ if (conf.get(STORAGE_RDMA_INTERFACE_KEY) != null) { STORAGE_RDMA_INTERFACE = conf.get(STORAGE_RDMA_INTERFACE_KEY); if (conf.get(STORAGE_RDMA_PORT_KEY) != null) { STORAGE_RDMA_PORT = Integer.parseInt(conf.get(STORAGE_RDMA_PORT_KEY)); if (conf.get(STORAGE_RDMA_STORAGE_LIMIT_KEY) != null) { STORAGE_RDMA_STORAGE_LIMIT = Long.parseLong(conf.get(STORAGE_RDMA_STORAGE_LIMIT_KEY)); if (conf.get(STORAGE_RDMA_ALLOCATION_SIZE_KEY) != null) { STORAGE_RDMA_ALLOCATION_SIZE = Long.parseLong(conf.get(STORAGE_RDMA_ALLOCATION_SIZE_KEY)); if (conf.get(STORAGE_RDMA_DATA_PATH_KEY) != null) { STORAGE_RDMA_DATA_PATH = conf.get(STORAGE_RDMA_DATA_PATH_KEY); if (conf.get(STORAGE_RDMA_LOCAL_MAP_KEY) != null) { STORAGE_RDMA_LOCAL_MAP = conf.getBoolean(STORAGE_RDMA_LOCAL_MAP_KEY, false); if (conf.get(STORAGE_RDMA_QUEUESIZE_KEY) != null) { STORAGE_RDMA_QUEUESIZE = Integer.parseInt(conf.get(STORAGE_RDMA_QUEUESIZE_KEY)); if (conf.get(STORAGE_RDMA_TYPE_KEY) != null) { STORAGE_RDMA_TYPE = conf.get(STORAGE_RDMA_TYPE_KEY); if (conf.get(STORAGE_RDMA_PERSISTENT_KEY) != null) { STORAGE_RDMA_PERSISTENT = conf.getBoolean(STORAGE_RDMA_PERSISTENT_KEY, false); if (conf.get(STORAGE_RDMA_BACKLOG_KEY) != null) { STORAGE_RDMA_BACKLOG = Integer.parseInt(conf.get(STORAGE_RDMA_BACKLOG_KEY));
private static String get(CrailConfiguration conf, String key) { return conf.get(fullKey(key)); }
private static Properties loadProperties(String resourceName) throws IOException { Properties properties = new Properties(); String base = System.getenv("CRAIL_HOME"); if (base == null || base.isEmpty()) { throw new IllegalArgumentException("CRAIL_HOME environment variable is not set or empty"); } FileInputStream inputStream = new FileInputStream(new File(base + "/conf/" + resourceName)); try { properties.load(inputStream); } finally { inputStream.close(); } for (String key : properties.stringPropertyNames()) { String val = properties.getProperty(key); properties.setProperty(key, expandEnvVars(val)); } return properties; }
@Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); setConf(conf); try { CrailConfiguration crailConf = new CrailConfiguration(); this.dfs = CrailStore.newInstance(crailConf); Path _workingDir = new Path("/user/" + CrailConstants.USER); this.workingDir = new Path("/user/" + CrailConstants.USER).makeQualified(uri, _workingDir); this.uri = URI.create(CrailConstants.NAMENODE_ADDRESS); LOG.info("CrailHadoopFileSystem fs initialization done.."); } catch(Exception e){ throw new IOException(e); } }
if (conf.get(DIRECTORY_DEPTH_KEY) != null) { DIRECTORY_DEPTH = Integer.parseInt(conf.get(DIRECTORY_DEPTH_KEY)); if (conf.get(TOKEN_EXPIRATION_KEY) != null) { TOKEN_EXPIRATION = Long.parseLong(conf.get(TOKEN_EXPIRATION_KEY)); if (conf.get(BLOCK_SIZE_KEY) != null) { BLOCK_SIZE = Long.parseLong(conf.get(BLOCK_SIZE_KEY)); if (conf.get(CACHE_LIMIT_KEY) != null) { CACHE_LIMIT = Long.parseLong(conf.get(CACHE_LIMIT_KEY)); if (conf.get(CACHE_PATH_KEY) != null) { CACHE_PATH = conf.get(CACHE_PATH_KEY); if (conf.get(USER_KEY) != null) { USER = conf.get(CrailConstants.USER_KEY); if (conf.get(SHADOW_REPLICATION_KEY) != null) { SHADOW_REPLICATION = Integer.parseInt(conf.get(SHADOW_REPLICATION_KEY)); if (conf.get(DEBUG_KEY) != null) { DEBUG = Boolean.parseBoolean(conf.get(DEBUG_KEY)); if (conf.get(STATISTICS_KEY) != null) { STATISTICS = Boolean.parseBoolean(conf.get(STATISTICS_KEY)); if (conf.get(RPC_TIMEOUT_KEY) != null) { RPC_TIMEOUT = Integer.parseInt(conf.get(RPC_TIMEOUT_KEY));
public void getLocations(String filename, long offset, long length) throws Exception { System.out.println("getLocations, filename " + filename + ", offset " + offset + ", len " + length); CrailConfiguration conf = new CrailConfiguration(); CrailStore fs = CrailStore.newInstance(conf); CrailBlockLocation locations[] = fs.lookup(filename).get().getBlockLocations(offset, length); for (int i = 0; i < locations.length; i++){ System.out.println("location " + i + " : " + locations[i].toString()); } fs.close(); }
if (conf.get(DIRECTORY_DEPTH_KEY) != null) { DIRECTORY_DEPTH = Integer.parseInt(conf.get(DIRECTORY_DEPTH_KEY)); if (conf.get(TOKEN_EXPIRATION_KEY) != null) { TOKEN_EXPIRATION = Long.parseLong(conf.get(TOKEN_EXPIRATION_KEY)); if (conf.get(BLOCK_SIZE_KEY) != null) { BLOCK_SIZE = Long.parseLong(conf.get(BLOCK_SIZE_KEY)); if (conf.get(CACHE_LIMIT_KEY) != null) { CACHE_LIMIT = Long.parseLong(conf.get(CACHE_LIMIT_KEY)); if (conf.get(CACHE_PATH_KEY) != null) { CACHE_PATH = conf.get(CACHE_PATH_KEY); if (conf.get(USER_KEY) != null) { USER = conf.get(CrailConstants.USER_KEY); if (conf.get(SHADOW_REPLICATION_KEY) != null) { SHADOW_REPLICATION = Integer.parseInt(conf.get(SHADOW_REPLICATION_KEY)); if (conf.get(DEBUG_KEY) != null) { DEBUG = Boolean.parseBoolean(conf.get(DEBUG_KEY)); if (conf.get(STATISTICS_KEY) != null) { STATISTICS = Boolean.parseBoolean(conf.get(STATISTICS_KEY)); if (conf.get(RPC_TIMEOUT_KEY) != null) { RPC_TIMEOUT = Integer.parseInt(conf.get(RPC_TIMEOUT_KEY));
public void getLocations(String filename, long offset, long length) throws Exception { System.out.println("getLocations, filename " + filename + ", offset " + offset + ", len " + length); CrailConfiguration conf = new CrailConfiguration(); CrailStore fs = CrailStore.newInstance(conf); CrailBlockLocation locations[] = fs.lookup(filename).get().getBlockLocations(offset, length); for (int i = 0; i < locations.length; i++){ System.out.println("location " + i + " : " + locations[i].toString()); } fs.close(); }