/** * Replaces default key with user provided key. * * @param conf configuration to replace the key in * @param tachyonConf Tachyon configuration with the key * @param key the key to replace */ public static void addKey(Configuration conf, TachyonConf tachyonConf, String key) { if (System.getProperty(key) != null) { conf.set(key, System.getProperty(key)); } else if (tachyonConf.get(key) != null) { conf.set(key, tachyonConf.get(key)); } }
/** * This method adds AWS credentials from system properties to the Tachyon Conf if they are not * already present. * @param tachyonConf the conf to check and add credentials to * @return true if both access and secret key are present, false otherwise */ private boolean addAndCheckAWSCredentials(TachyonConf tachyonConf) { String accessKeyConf = Constants.S3_ACCESS_KEY; if (System.getProperty(accessKeyConf) != null && tachyonConf.get(accessKeyConf) == null) { tachyonConf.set(accessKeyConf, System.getProperty(accessKeyConf)); } String secretKeyConf = Constants.S3_SECRET_KEY; if (System.getProperty(secretKeyConf) != null && tachyonConf.get(secretKeyConf) == null) { tachyonConf.set(secretKeyConf, System.getProperty(secretKeyConf)); } return tachyonConf.get(accessKeyConf) != null && tachyonConf.get(secretKeyConf) != null; } }
/** * Start Tachyon Master by executing the launch script. */ private static void startMaster() { String startMasterCommand = new TachyonConf().get(Constants.TACHYON_HOME) + "/bin/tachyon-start.sh master"; try { Runtime.getRuntime().exec(startMasterCommand).waitFor(); CommonUtils.sleepMs(LOG, 1000); } catch (Exception e) { LOG.error("Error when starting Master", e); } }
@Override public void connectFromMaster(TachyonConf conf, String host) throws IOException { if (!conf.containsKey(Constants.MASTER_KEYTAB_KEY) || !conf.containsKey(Constants.MASTER_PRINCIPAL_KEY)) { return; } String masterKeytab = conf.get(Constants.MASTER_KEYTAB_KEY); String masterPrincipal = conf.get(Constants.MASTER_PRINCIPAL_KEY); login(Constants.MASTER_KEYTAB_KEY, masterKeytab, Constants.MASTER_PRINCIPAL_KEY, masterPrincipal, host); }
/** * Creates a new builder for {@link CreateOptions}. * * @param conf a Tachyon configuration */ public Builder(TachyonConf conf) { mBlockSizeBytes = conf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); mRecursive = false; mTTL = Constants.NO_TTL; WriteType defaultWriteType = conf.getEnum(Constants.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class); mUnderStorageType = defaultWriteType.getUnderStorageType(); }
mWorkerConf.merge(mMasterConf); mWorkerConf.set(Constants.WORKER_PORT, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_DATA_PORT, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_WEB_PORT, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_DATA_FOLDER, mWorkerDataFolder); mWorkerConf.set(Constants.WORKER_MEMORY_SIZE, Long.toString(mWorkerCapacityBytes)); mWorkerConf.set(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS, Integer.toString(15)); mWorkerConf.set(Constants.WORKER_WORKER_BLOCK_THREADS_MIN, Integer.toString(1)); mWorkerConf.set(Constants.WORKER_WORKER_BLOCK_THREADS_MAX, Integer.toString(2048)); mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_WORKER_THREADS, Integer.toString(2)); mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_TIMEOUT, Integer.toString(0)); mWorkerConf.set(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS, Integer.toString(250)); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_ALIAS_FORMAT, 0), "MEM"); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT, 0), ramdiskPath); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA_FORMAT, 0), Long.toString(mWorkerCapacityBytes)); UnderFileSystemUtils.mkdirIfNotExists(ramdiskPath, mWorkerConf); int maxLevel = mWorkerConf.getInt(Constants.WORKER_TIERED_STORAGE_LEVEL_MAX); for (int level = 1; level < maxLevel; level ++) { String tierLevelDirPath = String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT, level); String[] dirPaths = mWorkerConf.get(tierLevelDirPath).split(","); List<String> newPaths = new ArrayList<String>();
/** * Initializes the client context singleton with a given conf. */ public static synchronized void reset(TachyonConf conf) { sTachyonConf = conf; String masterHostname = Preconditions.checkNotNull(sTachyonConf.get(Constants.MASTER_HOSTNAME)); int masterPort = sTachyonConf.getInt(Constants.MASTER_PORT); sMasterAddress = new InetSocketAddress(masterHostname, masterPort); sClientMetrics = new ClientMetrics(); sRandom = new Random(); if (sExecutorService != null) { sExecutorService.shutdown(); } sExecutorService = Executors.newFixedThreadPool( sTachyonConf.getInt(Constants.USER_BLOCK_WORKER_CLIENT_THREADS), ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)); // We must set sInitialized to true before resetting BlockStoreContext and FileSystemContext // as they need ClientContext initialized. sInitialized = true; BlockStoreContext.INSTANCE.reset(); FileSystemContext.INSTANCE.reset(); }
public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException { super(tachyonConf); Preconditions.checkArgument(tachyonConf.containsKey(Constants.S3_ACCESS_KEY), "Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3"); Preconditions.checkArgument(tachyonConf.containsKey(Constants.S3_SECRET_KEY), "Property " + Constants.S3_SECRET_KEY + " is required to connect to S3"); AWSCredentials awsCredentials = new AWSCredentials(tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get( Constants.S3_SECRET_KEY)); mBucketName = bucketName; Jets3tProperties props = new Jets3tProperties(); if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) { props.setProperty("httpclient.proxy-autodetect", "false"); props.setProperty("httpclient.proxy-host", tachyonConf.get(Constants.UNDERFS_S3_PROXY_HOST)); props.setProperty("httpclient.proxy-port", tachyonConf.get(Constants.UNDERFS_S3_PROXY_PORT)); } if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) { props.setProperty("s3service.https-only", Boolean.toString(tachyonConf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY))); } LOG.debug("Initializing S3 underFs with properties: " + props.getProperties()); mClient = new RestS3Service(awsCredentials, null, null, props); mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR; }
TachyonConf tachyonConf = new TachyonConf(); tachyonConf.get(Constants.MASTER_JOURNAL_FOLDER); if (!formatFolder("JOURNAL_FOLDER", masterJournal, tachyonConf)) { System.exit(-1); masterJournal + Constants.FORMAT_FILE_PREFIX + System.currentTimeMillis(), tachyonConf); } else if (args[0].toUpperCase().equals("WORKER")) { String workerDataFolder = tachyonConf.get(Constants.WORKER_DATA_FOLDER); int maxStorageLevels = tachyonConf.getInt(Constants.WORKER_TIERED_STORAGE_LEVEL_MAX); for (int level = 0; level < maxStorageLevels; level ++) { String tierLevelDirPath = String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT, level); String[] dirPaths = tachyonConf.get(tierLevelDirPath).split(","); String name = "TIER_" + level + "_DIR_PATH"; for (String dirPath : dirPaths) {
protected TachyonMaster() { TachyonConf conf = MasterContext.getConf(); mMinWorkerThreads = conf.getInt(Constants.MASTER_WORKER_THREADS_MIN); mMaxWorkerThreads = conf.getInt(Constants.MASTER_WORKER_THREADS_MAX); mPort = NetworkAddressUtils.getThriftPort(mTServerSocket); conf.set(Constants.MASTER_PORT, Integer.toString(mPort)); mMasterAddress = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, conf); String journalDirectory = conf.get(Constants.MASTER_JOURNAL_FOLDER); if (!journalDirectory.endsWith(TachyonURI.SEPARATOR)) { journalDirectory += TachyonURI.SEPARATOR;
mMasterConf.set(Constants.IN_TEST_MODE, "true"); mMasterConf.set(Constants.TACHYON_HOME, mTachyonHome); mMasterConf.set(Constants.ZOOKEEPER_ENABLED, "true"); mMasterConf.set(Constants.MASTER_HOSTNAME, mHostname); mMasterConf.set(Constants.MASTER_BIND_HOST, mHostname); mMasterConf.set(Constants.MASTER_PORT, "0"); mMasterConf.set(Constants.MASTER_WEB_BIND_HOST, mHostname); mMasterConf.set(Constants.MASTER_WEB_PORT, "0"); mMasterConf.set(Constants.ZOOKEEPER_ADDRESS, mCuratorServer.getConnectString()); mMasterConf.set(Constants.ZOOKEEPER_ELECTION_PATH, "/election"); mMasterConf.set(Constants.ZOOKEEPER_LEADER_PATH, "/leader"); mMasterConf.set(Constants.USER_QUOTA_UNIT_BYTES, "10000"); mMasterConf.set(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT, Integer.toString(mUserBlockSize)); mMasterConf.set(Constants.MASTER_TTLCHECKER_INTERVAL_MS, Integer.toString(1000)); mMasterConf.set(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS, "250"); mMasterConf.set(Constants.MASTER_PORT, "0"); mkdir(mMasterConf.get(Constants.UNDERFS_ADDRESS)); mMasterConf.set(Constants.MASTER_PORT, getMasterPort() + ""); mWorkerConf.merge(mMasterConf); mWorkerConf.set(Constants.WORKER_DATA_FOLDER, mWorkerDataFolder); mWorkerConf.set(Constants.WORKER_MEMORY_SIZE, mWorkerCapacityBytes + ""); mWorkerConf.set(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS, 15 + ""); mWorkerConf.set(Constants.WORKER_TIERED_STORAGE_LEVEL_MAX, Integer.toString(maxLevel));
/** * Constructor for PinListSync * * @param blockDataManager the blockDataManager this syncer is updating to * @param masterClient the Tachyon master client */ public PinListSync(BlockDataManager blockDataManager, WorkerFileSystemMasterClient masterClient) { mBlockDataManager = blockDataManager; TachyonConf conf = WorkerContext.getConf(); mMasterClient = masterClient; mSyncIntervalMs = conf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS); mSyncTimeoutMs = conf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_TIMEOUT_MS); mRunning = true; }
private ServerBootstrap createBootstrap() { final ServerBootstrap boot = createBootstrapOfType(mTachyonConf.getEnum(Constants.WORKER_NETWORK_NETTY_CHANNEL, ChannelType.class)); (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_WATERMARK_HIGH)); boot.childOption(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_WATERMARK_LOW)); if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BACKLOG)) { boot.option(ChannelOption.SO_BACKLOG, mTachyonConf.getInt(Constants.WORKER_NETWORK_NETTY_BACKLOG)); if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BUFFER_SEND)) { boot.option(ChannelOption.SO_SNDBUF, (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_BUFFER_SEND)); if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BUFFER_RECEIVE)) { boot.option(ChannelOption.SO_RCVBUF, (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_BUFFER_RECEIVE));
/** * Reset the TachyonConf instance in master context, for test only. * TODO(binfan): consider a better way to mock test TachyonConf */ public static void resetConf() { sTachyonConf = new TachyonConf(); } }
public TachyonMasterFaultTolerant() { super(); TachyonConf conf = MasterContext.getConf(); Preconditions.checkArgument(conf.getBoolean(Constants.ZOOKEEPER_ENABLED)); // Set up zookeeper specific functionality. try { // InetSocketAddress.toString causes test issues, so build the string by hand String zkName = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, conf) + ":" + getMasterAddress().getPort(); String zkAddress = conf.get(Constants.ZOOKEEPER_ADDRESS); String zkElectionPath = conf.get(Constants.ZOOKEEPER_ELECTION_PATH); String zkLeaderPath = conf.get(Constants.ZOOKEEPER_LEADER_PATH); mLeaderSelectorClient = new LeaderSelectorClient(zkAddress, zkElectionPath, zkLeaderPath, zkName); } catch (Exception e) { LOG.error(e.getMessage(), e); throw Throwables.propagate(e); } }
/** * Configures and starts master. * * @throws IOException when the operation fails */ public void startMaster() throws IOException { mMasterConf = MasterContext.getConf(); mMasterConf.set(Constants.IN_TEST_MODE, "true"); mMasterConf.set(Constants.TACHYON_HOME, mTachyonHome); mMasterConf.set(Constants.USER_QUOTA_UNIT_BYTES, Integer.toString(mQuotaUnitBytes)); mMasterConf.set(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT, Integer.toString(mUserBlockSize)); mMasterConf.set(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES, Integer.toString(64)); mMasterConf.set(Constants.MASTER_HOSTNAME, mLocalhostName); mMasterConf.set(Constants.MASTER_PORT, Integer.toString(0)); mMasterConf.set(Constants.MASTER_WEB_PORT, Integer.toString(0)); mMasterConf.set(Constants.MASTER_TTLCHECKER_INTERVAL_MS, Integer.toString(1000)); // default write type becomes MUST_CACHE, set this value to CACHE_THROUGH for tests. // default tachyon storage is STORE, and under storage is SYNC_PERSIST for tests. // TODO(binfan): eliminate this setting after updating integration tests mMasterConf.set(Constants.USER_FILE_WRITE_TYPE_DEFAULT, "CACHE_THROUGH"); mMaster = LocalTachyonMaster.create(mTachyonHome); mMaster.start(); }
private SortedSet<Pair<String, String>> getSortedProperties() { TreeSet<Pair<String, String>> rtn = new TreeSet<Pair<String, String>>(); for (Map.Entry<Object, Object> entry : mTachyonConf.getInternalProperties().entrySet()) { String key = entry.getKey().toString(); if (key.startsWith(TACHYON_CONF_PREFIX) && !TACHYON_CONF_EXCLUDES.contains(key)) { rtn.add(new ImmutablePair<String, String>(key, mTachyonConf.get(key))); } } return rtn; } }
private TachyonFS(TachyonConf tachyonConf) { super(tachyonConf); mMasterAddress = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, tachyonConf); mZookeeperMode = mTachyonConf.getBoolean(Constants.ZOOKEEPER_ENABLED); mFSMasterClient = mCloser.register(FileSystemContext.INSTANCE.acquireMasterClient()); mBlockMasterClient = mCloser.register(BlockStoreContext.INSTANCE.acquireMasterClient()); mRawTableMasterClient = mCloser.register(new RawTableMasterClient(mMasterAddress, mTachyonConf)); mWorkerClient = mCloser.register(BlockStoreContext.INSTANCE.acquireWorkerClient()); mUserFailedSpaceRequestLimits = mTachyonConf.getInt(Constants.USER_FAILED_SPACE_REQUEST_LIMITS); String scheme = mZookeeperMode ? Constants.SCHEME_FT : Constants.SCHEME; String authority = mMasterAddress.getHostName() + ":" + mMasterAddress.getPort(); mRootUri = new TachyonURI(scheme, authority, TachyonURI.SEPARATOR); }
@Override public long getDefaultBlockSize() { return mTachyonConf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); }
mBlockDataManager, mTachyonConf); mTachyonConf.set(Constants.WORKER_DATA_PORT, Integer.toString(mDataServer.getPort())); mPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket); mTachyonConf.set(Constants.WORKER_PORT, Integer.toString(mPort)); mThriftServer = createThriftServer(); mWorkerNetAddress = if (mTachyonConf.getBoolean(Constants.WORKER_TIERED_STORE_RESERVER_ENABLED)) { mSpaceReserver = new SpaceReserver(mBlockDataManager);