final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf); try { dfs.setErasureCodingPolicy(p, ecPolicyName); if (ecPolicyName == null){ ecPolicyName = "default";
private void testErasureCodingSetting(boolean defaultBehavior) throws IOException { JobConf jConf = new JobConf(); // don't set to false if EC remains disabled to check default setting if (!defaultBehavior) { jConf.setBoolean(MRJobConfig.MR_AM_STAGING_DIR_ERASURECODING_ENABLED, true); } DistributedFileSystem fs = mock(DistributedFileSystem.class); Path path = new Path("/"); when(fs.makeQualified(any(Path.class))).thenReturn(path); JobResourceUploader uploader = new StubedUploader(fs, true); Job job = Job.getInstance(jConf); uploader.uploadResources(job, new Path("/test")); String replicationPolicyName = SystemErasureCodingPolicies .getReplicationPolicy().getName(); VerificationMode mode = defaultBehavior ? times(1) : never(); verify(fs, mode).setErasureCodingPolicy(eq(path), eq(replicationPolicyName)); }
@Before public void setup() throws Exception { Configuration hdfsConf = new HdfsConfiguration(); hdfsConf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); String namenodeDir = new File(MiniDFSCluster.getBaseDirectory(), "name"). getAbsolutePath(); hdfsConf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeDir); hdfsConf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, namenodeDir); hdfsConf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); cluster = new MiniDFSCluster.Builder(hdfsConf).numDataNodes(15).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy(ecPolicy.getName()); fs.setErasureCodingPolicy(new Path("/"), ecPolicy.getName()); cluster.waitActive(); conf = new Configuration(); submitDir = new Path("/"); testFile = new Path("/testfile"); DFSTestUtil.writeFile(fs, testFile, StripedFileTestUtil.generateBytes(BLOCKSIZE)); conf.set(FileInputFormat.INPUT_DIR, fs.getUri().toString() + testFile.toString()); }
/** * Set the source path to the specified erasure coding policy. * * @param path The source path referring to a directory. * @param ecPolicyName The erasure coding policy name for the directory. * * @throws IOException * @throws HadoopIllegalArgumentException if the specified EC policy is not * enabled on the cluster */ public void setErasureCodingPolicy(final Path path, final String ecPolicyName) throws IOException { dfs.setErasureCodingPolicy(path, ecPolicyName); }
@Override public Void next(final FileSystem fs, final Path p) throws IOException { if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem) fs; myDfs.setErasureCodingPolicy(p, ecPolicyName); return null; } throw new UnsupportedOperationException( "Cannot setErasureCodingPolicy through a symlink to a " + "non-DistributedFileSystem: " + path + " -> " + p); } }.resolve(this, absF);