/** * Setup the cluster */ public BlockReaderTestUtil(int replicationFactor) throws Exception { conf = new HdfsConfiguration(); conf.setInt("dfs.replication", replicationFactor); cluster = new MiniDFSCluster.Builder(conf).format(true).build(); cluster.waitActive(); }
public static HdfsConfiguration getTestHdfsConfiguration() { HdfsConfiguration conf = new HdfsConfiguration(); String testDataPath = new File(TEST_PATH + "/" + UUID.randomUUID().toString()).getAbsolutePath(); String namenodeDir = new File(testDataPath, "name").getAbsolutePath(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath); conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeDir); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, namenodeDir); conf.setLong(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, 0); conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); return conf; }
private static HdfsConfiguration createCachingConf() { HdfsConfiguration conf = new HdfsConfiguration(); conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, CACHE_CAPACITY); conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, 1000); conf.setLong(DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS, 1000); // set low limits here for testing purposes conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES, 2); conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES, 2); return conf; }
public BlockReaderTestUtil(int replicationFactor, HdfsConfiguration config) throws Exception { this.conf = config; conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replicationFactor); cluster = new MiniDFSCluster.Builder(conf).format(true).build(); cluster.waitActive(); }
private static HdfsConfiguration getConfiguration( TemporarySocketDirectory socketDir) throws IOException { HdfsConfiguration conf = new HdfsConfiguration(); if (socketDir == null) { conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, ""); } else { conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(socketDir.getDir(), "TestBlockReaderLocalLegacy.%d.sock"). getAbsolutePath()); } conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL, true); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, false); conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, UserGroupInformation.getCurrentUser().getShortUserName()); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC, false); // Set short retry timeouts so this test runs faster conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 10); return conf; }
@BeforeClass static public void setupCluster() throws Exception { if (DomainSocket.getLoadingFailureReason() != null) return; sockDir = new TemporarySocketDirectory(); HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestParallelShortCircuitReadUnCached._PORT.sock").getAbsolutePath()); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); // Enabling data transfer encryption should have no effect when using // short-circuit local reads. This is a regression test for HDFS-5353. conf.setBoolean(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY, true); conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); conf.setBoolean(DFSConfigKeys. DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, false); conf.setBoolean(DFSConfigKeys. DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC, true); // We want to test reading from stale sockets. conf.setInt(DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY, 1); conf.setLong(DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY, 5 * 60 * 1000); conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY, 32); // Avoid using the FileInputStreamCache. conf.setInt(DFSConfigKeys. DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_KEY, 0); DomainSocket.disableBindPathValidation(); DFSInputStream.tcpReadsDisabledForTesting = true; setupCluster(1, conf); }
public static void createCluster() throws IOException { HdfsConfiguration conf = new HdfsConfiguration(); conf.addResource(CONTRACT_HDFS_XML); //hack in a 256 byte block size conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); cluster.waitClusterUp(); }
public static HdfsConfiguration initZeroCopyTest() { Assume.assumeTrue(NativeIO.isAvailable()); Assume.assumeTrue(SystemUtils.IS_OS_UNIX); HdfsConfiguration conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE, 3); conf.setLong(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS, 100); conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestRequestMmapAccess._PORT.sock").getAbsolutePath()); conf.setBoolean(DFSConfigKeys. DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, true); conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setLong(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, 1000); conf.setLong(DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS, 1000); return conf; }
@Test public void testServerSaslNoClientSasl() throws Exception { HdfsConfiguration clusterConf = createSecureConfig( "authentication,integrity,privacy"); // Set short retry timeouts so this test runs faster clusterConf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 10); startCluster(clusterConf); HdfsConfiguration clientConf = new HdfsConfiguration(clusterConf); clientConf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, ""); LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs( LogFactory.getLog(DataNode.class)); try { doTest(clientConf); Assert.fail("Should fail if SASL data transfer protection is not " + "configured or not supported in client"); } catch (IOException e) { GenericTestUtils.assertMatches(e.getMessage(), "could only be replicated to 0 nodes"); } finally { logs.stopCapturing(); } GenericTestUtils.assertMatches(logs.getOutput(), "Failed to read expected SASL data transfer protection " + "handshake from client at"); }
/** * Test that the socket cache can be disabled by setting the capacity to * 0. Regression test for HDFS-3365. * @throws Exception */ @Test public void testDisableCache() throws Exception { HdfsConfiguration confWithoutCache = new HdfsConfiguration(); // Configure a new instance with no peer caching, ensure that it doesn't // cache anything confWithoutCache.setInt( DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY, 0); BlockReaderTestUtil util = new BlockReaderTestUtil(1, confWithoutCache); final Path testFile = new Path("/testConnCache.dat"); util.writeFile(testFile, FILE_SIZE / 1024); FileSystem fsWithoutCache = FileSystem.newInstance(util.getConf()); try { DFSTestUtil.readFile(fsWithoutCache, testFile); assertEquals(0, ((DistributedFileSystem)fsWithoutCache). dfs.getClientContext().getPeerCache().size()); } finally { fsWithoutCache.close(); util.shutdown(); } } }
/** * Creates configuration for starting a secure cluster. * * @param dataTransferProtection supported QOPs * @return configuration for starting a secure cluster * @throws Exception if there is any failure */ protected HdfsConfiguration createSecureConfig( String dataTransferProtection) throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); conf.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal); conf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, keytab); conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal); conf.set(DFS_DATANODE_KEYTAB_FILE_KEY, keytab); conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, spnegoPrincipal); conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, dataTransferProtection); conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10); String keystoresDir = baseDir.getAbsolutePath(); String sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass()); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); return conf; } }
baseConf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); baseConf.set(DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, "localhost:0"); baseConf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10);
conf.setInt(DFS_CLIENT_MMAP_CACHE_SIZE, 0); conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT, CONTEXT + ".1"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
InterruptedException { HdfsConfiguration conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(1)