@Override protected boolean isZookeeperMode() { return mFsContext.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED); } }
@Override public long getDefaultBlockSize() { return mFsContext.getConf() .getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT); }
@Override public short getDefaultReplication() { return (short) Math.max(1, mFsContext.getConf() .getInt(PropertyKey.USER_FILE_REPLICATION_MIN)); }
@Override public int run(CommandLine cl) { try { InetSocketAddress address = JobContext.create(mFsContext.getConf()).getJobMasterAddress(); System.out.println(address.getHostName()); } catch (Exception e) { LOG.error("Failed to get the primary job master", e); System.out.println("Failed to get the primary job master."); return -1; } return 0; }
/** * Creates an Alluxio block store. * * @param context the file system context * @param tieredIdentity the tiered identity */ @VisibleForTesting AlluxioBlockStore(FileSystemContext context, TieredIdentity tieredIdentity) { mContext = context; mTieredIdentity = tieredIdentity; mWorkerRefreshPolicy = new TimeoutRefresh(mContext.getConf() .getMs(PropertyKey.USER_WORKER_LIST_REFRESH_INTERVAL)); }
/** * Creates an Alluxio block store with default local hostname. * * @param context the file system context * @return the {@link AlluxioBlockStore} created */ public static AlluxioBlockStore create(FileSystemContext context) { return new AlluxioBlockStore(context, TieredIdentityFactory.localIdentity(context.getConf())); }
public static FileSystem get(FileSystemContext context) { if (LOG.isDebugEnabled() && !CONF_LOGGED.getAndSet(true)) { // Sort properties by name to keep output ordered. AlluxioConfiguration conf = context.getConf(); List<PropertyKey> keys = new ArrayList<>(conf.keySet()); Collections.sort(keys, Comparator.comparing(PropertyKey::getName)); for (PropertyKey key : keys) { String value = conf.getOrDefault(key, null); Source source = conf.getSource(key); LOG.debug("{}={} ({})", key.getName(), value, source); } } return BaseFileSystem.create(context); }
@Override public int run(CommandLine cl) throws AlluxioException, IOException { long id = Long.parseLong(cl.getArgs()[0]); try (CloseableResource<JobMasterClient> client = JobContext.create(mFsContext.getConf()) .acquireMasterClientResource()) { JobInfo info = client.get().getStatus(id); System.out.print(formatOutput(cl, info)); } catch (Exception e) { LOG.error("Failed to get status of the job", e); System.out.println("Failed to get status of the job " + id); return -1; } return 0; }
/** * Creates a mock {@link FileOutputStream} which will store any bytes written to it for later * inspection during tests. */ public MockFileOutStream(FileSystemContext fsContext) throws IOException { super(new AlluxioURI("/"), OutStreamOptions.defaults(fsContext.getConf()), fsContext); mStream = new ByteArrayOutputStream(); }
@Test public void resetContextFromZkUriToNonZkUri() throws Exception { org.apache.hadoop.conf.Configuration conf = getConf(); URI uri = URI.create(Constants.HEADER + "zk@zkHost:2181/tmp/path.txt"); FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, conf)); assertTrue(fs.mFsContext.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED)); assertEquals("zkHost:2181", fs.mFsContext.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS)); URI otherUri = URI.create(Constants.HEADER + "alluxioHost:19998/tmp/path.txt"); fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(otherUri, conf)); assertEquals("alluxioHost", fs.mFsContext.getConf().get(PropertyKey.MASTER_HOSTNAME)); assertEquals("19998", fs.mFsContext.getConf().get(PropertyKey.MASTER_RPC_PORT)); assertFalse(fs.mFsContext.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED)); assertFalse(fs.mFsContext.getConf().isSet(PropertyKey.ZOOKEEPER_ADDRESS)); }
@Test public void resetContextUsingMultiMasterUris() throws Exception { // Change to multi-master uri URI uri = URI.create(Constants.HEADER + "host1:19998,host2:19998,host3:19998/tmp/path.txt"); FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf())); assertFalse(fs.mFsContext.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED)); assertEquals("host1:19998,host2:19998,host3:19998", fs.mFsContext.getConf().get(PropertyKey.MASTER_RPC_ADDRESSES)); }
@Test public void resetContextFromZookeeperToMultiMaster() throws Exception { URI uri = URI.create(Constants.HEADER + "zk@zkHost:2181/tmp/path.txt"); FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf())); assertTrue(fs.mFsContext.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED)); assertEquals("zkHost:2181", fs.mFsContext.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS)); uri = URI.create(Constants.HEADER + "host1:19998,host2:19998,host3:19998/tmp/path.txt"); fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf())); assertFalse(fs.mFsContext.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED)); assertEquals(3, ConfigurationUtils.getMasterRpcAddresses(fs.mFsContext.getConf()).size()); assertEquals("host1:19998,host2:19998,host3:19998", fs.mFsContext.getConf().get(PropertyKey.MASTER_RPC_ADDRESSES)); }
@Test public void hadoopShouldLoadFileSystemWithSingleZkUri() throws Exception { org.apache.hadoop.conf.Configuration conf = getConf(); URI uri = URI.create(Constants.HEADER + "zk@zkHost:2181/tmp/path.txt"); FileSystem hfs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, conf)); assertTrue(hfs.mFsContext.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED)); assertEquals("zkHost:2181", hfs.mFsContext.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS)); }
@Test public void parseZkUriWithPlusDelimiters() throws Exception { org.apache.hadoop.fs.FileSystem fs = FileSystem.get(URI.create("alluxio://zk@a:0+b:1+c:2/"), new org.apache.hadoop.conf.Configuration()); assertTrue(fs instanceof AbstractFileSystem); AbstractFileSystem afs = (AbstractFileSystem) fs; assertTrue(afs.mFsContext.getConf() .getBoolean(PropertyKey.ZOOKEEPER_ENABLED)); assertEquals("a:0,b:1,c:2", afs.mFsContext.getConf() .get(PropertyKey.ZOOKEEPER_ADDRESS)); }
/** * Sets up the file system and the context before a test runs. */ @Before public void before() { mClientContext = ClientContext.create(mConf); mFileContext = PowerMockito.mock(FileSystemContext.class); mFileSystem = new DummyAlluxioFileSystem(mFileContext); mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class); when(mFileContext.acquireMasterClient()).thenReturn(mFileSystemMasterClient); when(mFileContext.getClientContext()).thenReturn(mClientContext); when(mFileContext.getConf()).thenReturn(mConf); }
private void printLsString(URIStatus status, boolean hSize) { // detect the extended acls boolean hasExtended = status.getAcl().hasExtended() || !status.getDefaultAcl().isEmpty(); System.out.print(formatLsString(hSize, SecurityUtils.isSecurityEnabled(mFsContext.getConf()), status.isFolder(), FormatUtils.formatMode((short) status.getMode(), status.isFolder(), hasExtended), status.getOwner(), status.getGroup(), status.getLength(), status.getLastModificationTimeMs(), status.getInAlluxioPercentage(), status.getPersistenceState(), status.getPath(), mFsContext.getConf().get(PropertyKey.USER_DATE_FORMAT_PATTERN))); }
@Test public void initializeWithZookeeperSystemProperties() throws Exception { HashMap<String, String> sysProps = new HashMap<>(); sysProps.put(PropertyKey.ZOOKEEPER_ENABLED.getName(), "true"); sysProps.put(PropertyKey.ZOOKEEPER_ADDRESS.getName(), "zkHost:2181"); try (Closeable p = new SystemPropertyRule(sysProps).toResource()) { ConfigurationUtils.reloadProperties(); URI uri = URI.create("alluxio:///"); FileSystem fs = getHadoopFilesystem(org.apache.hadoop.fs.FileSystem.get(uri, getConf())); assertTrue(fs.mFsContext.getConf().getBoolean(PropertyKey.ZOOKEEPER_ENABLED)); assertEquals("zkHost:2181", fs.mFsContext.getConf().get(PropertyKey.ZOOKEEPER_ADDRESS)); } }
@Override public FileInStream openFile(AlluxioURI path, OpenFilePOptions options) throws FileDoesNotExistException, IOException, AlluxioException { checkUri(path); URIStatus status = getStatus(path); if (status.isFolder()) { throw new FileDoesNotExistException( ExceptionMessage.CANNOT_READ_DIRECTORY.getMessage(status.getName())); } InStreamOptions inStreamOptions = new InStreamOptions(status, options, mFsContext.getConf()); return new FileInStream(status, inStreamOptions, mFsContext); }
@Before public void before() throws Exception { mClientContext = ClientContext.create(mConf); mContext = PowerMockito.mock(FileSystemContext.class); mAddress = Mockito.mock(WorkerNetAddress.class); mClient = mock(BlockWorkerClient.class); mRequestObserver = mock(ClientCallStreamObserver.class); PowerMockito.when(mContext.getClientContext()).thenReturn(mClientContext); PowerMockito.when(mContext.getConf()).thenReturn(mConf); PowerMockito.when(mContext.acquireBlockWorkerClient(mAddress)).thenReturn(mClient); PowerMockito.doNothing().when(mContext).releaseBlockWorkerClient(mAddress, mClient); PowerMockito.when(mClient.writeBlock(any(StreamObserver.class))).thenReturn(mRequestObserver); PowerMockito.when(mRequestObserver.isReady()).thenReturn(true); }
@Before public void before() throws Exception { mContext = PowerMockito.mock(FileSystemContext.class); mAddress = mock(WorkerNetAddress.class); mClient = mock(BlockWorkerClient.class); mRequestObserver = mock(ClientCallStreamObserver.class); PowerMockito.when(mContext.acquireBlockWorkerClient(mAddress)).thenReturn(mClient); PowerMockito.when(mContext.getClientContext()) .thenReturn(ClientContext.create(mConf)); PowerMockito.when(mContext.getConf()).thenReturn(mConf); PowerMockito.doNothing().when(mContext).releaseBlockWorkerClient(mAddress, mClient); PowerMockito.when(mClient.writeBlock(any(StreamObserver.class))).thenReturn(mRequestObserver); PowerMockito.when(mRequestObserver.isReady()).thenReturn(true); }