private static LocalResource createResource(String url) { LocalResource res = mock(LocalResource.class); when(res.getResource()).thenReturn(URL.fromPath(new Path(url))); return res; }
private void makeJarAvailableInSharedCache(Path jar, MyFileUploader fileUploader) throws YarnException, IOException { // copy file to remote file system Path remoteFile = copyToRemote(jar); // prime mocking so that it looks like this file is in the shared cache fileUploader.mockFileInSharedCache(jar, URL.fromPath(remoteFile)); }
@Override public URL getResource() { return URL.fromPath(loc); }
@Override public URL getResource() { return URL.fromPath(loc); }
@Public @Deprecated public static URL getYarnUrlFromPath(Path path) { return URL.fromPath(path); }
/** * Create a {@link LocalResource} record with all the given parameters. */ private static LocalResource createLocalResource(FileSystem fc, Path file, LocalResourceType type, LocalResourceVisibility visibility) throws IOException { FileStatus fstat = fc.getFileStatus(file); URL resourceURL = URL.fromPath(fc.resolvePath(fstat.getPath())); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); }
public static ResourceLocalizationSpec newResourceLocalizationSpec( LocalResource rsrc, Path path) { URL local = URL.fromPath(path); ResourceLocalizationSpec resourceLocalizationSpec = Records.newRecord(ResourceLocalizationSpec.class); resourceLocalizationSpec.setDestinationDirectory(local); resourceLocalizationSpec.setResource(rsrc); return resourceLocalizationSpec; }
public static ResourceLocalizationSpec newResourceLocalizationSpec( LocalResource rsrc, Path path) { URL local = URL.fromPath(path); ResourceLocalizationSpec resourceLocalizationSpec = Records.newRecord(ResourceLocalizationSpec.class); resourceLocalizationSpec.setDestinationDirectory(local); resourceLocalizationSpec.setResource(rsrc); return resourceLocalizationSpec; }
@Test public void testConversion() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.IPC_RECORD_FACTORY_CLASS, RecordFactoryForTest.class.getName()); String[] pathStrs = new String[] {"/", ".", "foo/bar", "foo", "/foo/bar/baz", "moo://bar/baz", "moo://bar:123/baz", "moo:///foo", "moo://foo@bar:123/baz/foo", "moo://foo@bar/baz/foo", "moo://foo@bar", "moo://foo:123"}; for (String s : pathStrs) { Path path = new Path(s); assertEquals(path, URL.fromPath(path, conf).toPath()); } Path p = new Path("/foo/bar#baz"); assertEquals(p, URL.fromPath(p, conf).toPath()); }
@Test public void testConversion() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.IPC_RECORD_FACTORY_CLASS, RecordFactoryForTest.class.getName()); String[] pathStrs = new String[] {"/", ".", "foo/bar", "foo", "/foo/bar/baz", "moo://bar/baz", "moo://bar:123/baz", "moo:///foo", "moo://foo@bar:123/baz/foo", "moo://foo@bar/baz/foo", "moo://foo@bar", "moo://foo:123"}; for (String s : pathStrs) { Path path = new Path(s); assertEquals(path, URL.fromPath(path, conf).toPath()); } Path p = new Path("/foo/bar#baz"); assertEquals(p, URL.fromPath(p, conf).toPath()); }
@Test public void testConvertUrlWithNoPort() throws URISyntaxException { Path expectedPath = new Path("hdfs://foo.com"); URL url = URL.fromPath(expectedPath); Path actualPath = url.toPath(); assertEquals(expectedPath, actualPath); }
@Test public void testConvertUrlWithUserinfo() throws URISyntaxException { Path expectedPath = new Path("foo://username:password@example.com:8042"); URL url = URL.fromPath(expectedPath); Path actualPath = url.toPath(); assertEquals(expectedPath, actualPath); }
@Test public void testConvertUrlWithNoPort() throws URISyntaxException { Path expectedPath = new Path("hdfs://foo.com"); URL url = URL.fromPath(expectedPath); Path actualPath = url.toPath(); assertEquals(expectedPath, actualPath); }
@Test public void testConvertUrlWithUserinfo() throws URISyntaxException { Path expectedPath = new Path("foo://username:password@example.com:8042"); URL url = URL.fromPath(expectedPath); Path actualPath = url.toPath(); assertEquals(expectedPath, actualPath); }
static LocalResource createJar(FileContext files, Path p, LocalResourceVisibility vis) throws IOException { LOG.info("Create jar file " + p); File jarFile = new File((files.makeQualified(p)).toUri()); FileOutputStream stream = new FileOutputStream(jarFile); LOG.info("Create jar out stream "); JarOutputStream out = new JarOutputStream(stream, new Manifest()); LOG.info("Done writing jar stream "); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(URL.fromPath(p)); FileStatus status = files.getFileStatus(p); ret.setSize(status.getLen()); ret.setTimestamp(status.getModificationTime()); ret.setType(LocalResourceType.PATTERN); ret.setVisibility(vis); ret.setPattern("classes/.*"); return ret; }
private Map<String, LocalResource> getLocalResources() throws UnsupportedFileSystemException { FileContext localFS = FileContext.getLocalFSFileContext(); File tmpDir = new File("target"); File scriptFile = new File(tmpDir, "scriptFile.sh"); URL resourceURL = URL.fromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); LocalResource localRes = Records.newRecord(LocalResource.class); localRes.setResource(resourceURL); localRes.setSize(-1); localRes.setVisibility(LocalResourceVisibility.APPLICATION); localRes.setType(LocalResourceType.FILE); localRes.setTimestamp(scriptFile.lastModified()); String destinationFile = "dest_file"; Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put(destinationFile, localRes); return localResources; }
@Test public void testCLCPBImplNullResourceType() throws IOException { RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); try { LocalResource resource = recordFactory.newRecordInstance(LocalResource.class); resource.setResource(URL.fromPath(new Path("."))); resource.setSize(-1); resource.setVisibility(LocalResourceVisibility.APPLICATION); resource.setType(null); resource.setTimestamp(System.currentTimeMillis()); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put("null_type_resource", resource); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(localResources); Assert.fail("Setting an invalid local resource should be an error!"); } catch (NullPointerException e) { Assert.assertTrue(e.getMessage().contains("Null resource type for local resource")); } }
@Test public void testCLCPBImplNullResourceVisibility() throws IOException { RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); try { LocalResource resource = recordFactory.newRecordInstance(LocalResource.class); resource.setResource(URL.fromPath(new Path("."))); resource.setSize(-1); resource.setVisibility(null); resource.setType(LocalResourceType.FILE); resource.setTimestamp(System.currentTimeMillis()); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put("null_visibility_resource", resource); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(localResources); Assert.fail("Setting an invalid local resource should be an error!"); } catch (NullPointerException e) { Assert.assertTrue(e.getMessage().contains("Null resource visibility for local resource")); } } }
static LocalResource createFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException { createFile(files, p, len, r); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(URL.fromPath(p)); ret.setSize(len); ret.setType(LocalResourceType.FILE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(p).getModificationTime()); return ret; }
static LocalResource createFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException { createFile(files, p, len, r); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(URL.fromPath(p)); ret.setSize(len); ret.setType(LocalResourceType.FILE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(p).getModificationTime()); return ret; }