@Override public long getDefaultBlockSize() { return mTachyonConf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); }
/** * Creates a new builder for {@link CreatePathOptions}. * * @param conf a Tachyon configuration */ public Builder(TachyonConf conf) { mBlockSizeBytes = conf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); mDirectory = false; mOperationTimeMs = System.currentTimeMillis(); mRecursive = false; mPersisted = false; mTTL = Constants.NO_TTL; }
/** * Creates a new builder for {@link CreateOptions}. * * @param conf a Tachyon configuration */ public Builder(TachyonConf conf) { mBlockSizeBytes = conf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); mOperationTimeMs = System.currentTimeMillis(); mPersisted = false; mRecursive = false; mTTL = Constants.NO_TTL; }
/** * Initializes the internal buffer based on the user's specified size. Any reads above half * this size will not be buffered. * * @return a heap buffer of user configured size */ private ByteBuffer allocateBuffer() { TachyonConf conf = ClientContext.getConf(); return ByteBuffer.allocate( (int) conf.getBytes(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES)); }
/** * @return a newly allocated byte buffer of the user defined default size */ private ByteBuffer allocateBuffer() { TachyonConf conf = ClientContext.getConf(); return ByteBuffer.allocate((int) conf.getBytes(Constants.USER_FILE_BUFFER_BYTES)); } }
/** * Creates a file with the default block size (1GB) in the system. It also creates necessary * folders along the path. * * TODO(hy): It should not create necessary path. * * @param path the path of the file * @return The unique file id. It returns -1 if the creation failed * @throws IOException if the operation fails */ public synchronized long createFile(TachyonURI path) throws IOException { long defaultBlockSize = mTachyonConf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); return createFile(path, defaultBlockSize); }
(int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_WATERMARK_HIGH)); boot.childOption(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_WATERMARK_LOW)); (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_BUFFER_SEND)); (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_BUFFER_RECEIVE));
/** * Creates a new builder for {@link CreateOptions}. * * @param conf a Tachyon configuration */ public Builder(TachyonConf conf) { mBlockSizeBytes = conf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); mRecursive = false; mTTL = Constants.NO_TTL; WriteType defaultWriteType = conf.getEnum(Constants.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class); mUnderStorageType = defaultWriteType.getUnderStorageType(); }
/** * Tries to request space for certain block. Only works when a local worker exists. * * @param blockId the id of the block that space will be allocated for * @param requestSpaceBytes size to request in bytes * @return the size bytes that allocated to the block, -1 if no local worker exists * @throws IOException if the underlying file does not exist */ public synchronized long requestSpace(long blockId, long requestSpaceBytes) throws IOException { if (!hasLocalWorker()) { return -1; } long userQuotaUnitBytes = mTachyonConf.getBytes(Constants.USER_QUOTA_UNIT_BYTES); long toRequestSpaceBytes = Math.max(requestSpaceBytes, userQuotaUnitBytes); for (int attempt = 0; attempt < mUserFailedSpaceRequestLimits; attempt ++) { if (mWorkerClient.requestSpace(blockId, toRequestSpaceBytes)) { return toRequestSpaceBytes; } } return 0; }
public RawTableMaster(FileSystemMaster fileSystemMaster, Journal journal) { super(journal, Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("raw-table-master-%d", true))); TachyonConf conf = MasterContext.getConf(); mMaxTableMetadataBytes = conf.getBytes(Constants.MAX_TABLE_METADATA_BYTE); mMaxColumns = conf.getInt(Constants.MAX_COLUMNS); mFileSystemMaster = fileSystemMaster; }
bufferSize, stats); mTachyonConf = tachyonConf; long bufferBytes = mTachyonConf.getBytes(Constants.USER_FILE_BUFFER_BYTES); mBuffer = new byte[Ints.checkedCast(bufferBytes) * 4]; mCurrentPosition = 0;
/** * Creates a new builder for {@link OutStreamOptions}. * * @param conf a Tachyon configuration */ public Builder(TachyonConf conf) { mBlockSizeBytes = conf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); mHostname = null; WriteType defaultWriteType = conf.getEnum(Constants.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class); mTachyonStorageType = defaultWriteType.getTachyonStorageType(); mUnderStorageType = defaultWriteType.getUnderStorageType(); mTTL = Constants.NO_TTL; }
JournalWriter(Journal journal) { mJournal = Preconditions.checkNotNull(journal); mJournalDirectory = mJournal.getDirectory(); mCompletedDirectory = mJournal.getCompletedDirectory(); mTempCheckpointPath = mJournal.getCheckpointFilePath() + ".tmp"; TachyonConf conf = MasterContext.getConf(); mUfs = UnderFileSystem.get(mJournalDirectory, conf); mMaxLogSize = conf.getBytes(Constants.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX); }
/** * Creates a new local block output stream. * * @param blockId the block id * @param blockSize the block size * @throws IOException if an I/O error occurs */ public LocalBlockOutStream(long blockId, long blockSize) throws IOException { super(blockId, blockSize); mCloser = Closer.create(); mWorkerClient = mContext.acquireWorkerClient(NetworkAddressUtils.getLocalHostName(ClientContext.getConf())); try { long initialSize = ClientContext.getConf().getBytes(Constants.USER_FILE_BUFFER_BYTES); String blockPath = mWorkerClient.requestBlockLocation(mBlockId, initialSize); mReservedBytes += initialSize; FileUtils.createBlockPath(blockPath); RandomAccessFile localFile = mCloser.register(new RandomAccessFile(blockPath, "rw")); mLocalFileChannel = mCloser.register(localFile.getChannel()); // Change the permission of the temporary file in order that the worker can move it. LOG.info("LocalBlockOutStream created new file block, block path: " + blockPath); } catch (IOException ioe) { mContext.releaseWorkerClient(mWorkerClient); throw ioe; } }
long fileBufferBytes = tachyonConf.getBytes(Constants.USER_FILE_BUFFER_BYTES); sResultPrefix = String.format( "Threads %d FilesPerThread %d TotalFiles %d "