@Override public void onResponse(Call<Page> call, Response<Page> response) { if (!response.isSuccessful()) { System.out.println(call.request().url() + ": failed: " + response.code()); return; } // Print this page's URL and title. Page page = response.body(); HttpUrl base = response.raw().request().url(); System.out.println(base + ": " + page.title); // Enqueue its links for visiting. for (String link : page.links) { HttpUrl linkUrl = base.resolve(link); if (linkUrl != null && fetchedUrls.add(linkUrl)) { crawlPage(linkUrl); } } }
@Override public List<Cookie> loadForRequest(HttpUrl url) { // The RI passes all headers. We don't have 'em, so we don't pass 'em! Map<String, List<String>> headers = Collections.emptyMap(); Map<String, List<String>> cookieHeaders; try { cookieHeaders = cookieHandler.get(url.uri(), headers); } catch (IOException e) { Platform.get().log(WARN, "Loading cookies failed for " + url.resolve("/..."), e); return Collections.emptyList(); } List<Cookie> cookies = null; for (Map.Entry<String, List<String>> entry : cookieHeaders.entrySet()) { String key = entry.getKey(); if (("Cookie".equalsIgnoreCase(key) || "Cookie2".equalsIgnoreCase(key)) && !entry.getValue().isEmpty()) { for (String header : entry.getValue()) { if (cookies == null) cookies = new ArrayList<>(); cookies.addAll(decodeHeaderAsJavaNetCookies(url, header)); } } } return cookies != null ? Collections.unmodifiableList(cookies) : Collections.emptyList(); }
@Override public void saveFromResponse(HttpUrl url, List<Cookie> cookies) { if (cookieHandler != null) { List<String> cookieStrings = new ArrayList<>(); for (Cookie cookie : cookies) { cookieStrings.add(cookie.toString(true)); } Map<String, List<String>> multimap = Collections.singletonMap("Set-Cookie", cookieStrings); try { cookieHandler.put(url.uri(), multimap); } catch (IOException e) { Platform.get().log(WARN, "Saving cookies failed for " + url.resolve("/..."), e); } } }
for (Element element : document.select("a[href]")) { String href = element.attr("href"); HttpUrl link = response.request().url().resolve(href); if (link == null) continue; // URL is either invalid or its scheme isn't http/https. queue.add(link.newBuilder().fragment(null).build());
/** * Inspired by https://github.com/square/okhttp/blob/parent-3.6.0/okhttp/src/main/java/okhttp3/internal/http/RetryAndFollowUpInterceptor.java#L286 */ private Response followPostRedirect(Response response) throws IOException { String location = response.header("Location"); if (location == null) { throw new IllegalStateException(format("Missing HTTP header 'Location' in redirect of %s", response.request().url())); } HttpUrl url = response.request().url().resolve(location); // Don't follow redirects to unsupported protocols. if (url == null) { throw new IllegalStateException(format("Unsupported protocol in redirect of %s to %s", response.request().url(), location)); } Request.Builder redirectRequest = response.request().newBuilder(); redirectRequest.post(response.request().body()); response.body().close(); return okHttpClient.newCall(redirectRequest.url(url).build()).execute(); }
private HttpUrl.Builder prepareUrlBuilder(WsRequest wsRequest) { String path = wsRequest.getPath(); return baseUrl .resolve(path.startsWith("/") ? path.replaceAll("^(/)+", "") : path) .newBuilder(); }
private Response followPostRedirect(Response response, PostRequest postRequest) { String location = response.header("Location"); if (location == null) { throw new IllegalStateException(format("Missing HTTP header 'Location' in redirect of %s", response.request().url())); } HttpUrl url = response.request().url().resolve(location); // Don't follow redirects to unsupported protocols. if (url == null) { throw new IllegalStateException(format("Unsupported protocol in redirect of %s to %s", response.request().url(), location)); } Request.Builder redirectRequest = response.request().newBuilder(); redirectRequest.post(response.request().body()); response.body().close(); return doCall(prepareOkHttpClient(noRedirectOkHttpClient, postRequest), redirectRequest.url(url).build()); }
url = baseUrl.resolve(relativeUrl); if (url == null) { throw new IllegalArgumentException(
/** * Returns a URL for connecting to this server. * * @param path the request path, such as "/". */ public HttpUrl url(String path) { return new HttpUrl.Builder() .scheme(sslSocketFactory != null ? "https" : "http") .host(getHostName()) .port(getPort()) .build() .resolve(path); }
url = baseUrl.resolve(relativeUrl); if (url == null) { throw new IllegalArgumentException(
HttpUrl url = userResponse.request().url().resolve(location);
HttpUrl url = userResponse.request().url().resolve(location);
/** * Returns a string that describes this call. Doesn't include a full URL as that might contain * sensitive information. */ private String toLoggableString() { String string = canceled ? "canceled call" : "call"; HttpUrl redactedUrl = originalRequest.url().resolve("/..."); return string + " to " + redactedUrl; }
/** * Returns a string that describes this call. Doesn't include a full URL as that might contain * sensitive information. */ private String toLoggableString() { String string = canceled ? "canceled call" : "call"; HttpUrl redactedUrl = originalRequest.url().resolve("/..."); return string + " to " + redactedUrl; }
public static HttpUrl testRedirect(final HttpUrl base, final String content) { // check for redirect by http-equiv meta tag header final Matcher mHttpEquiv = P_REDIRECT_HTTP_EQUIV.matcher(content); if (mHttpEquiv.find()) return base.resolve(mHttpEquiv.group(1)); // check for redirect by window.location javascript final Matcher mScript = P_REDIRECT_SCRIPT.matcher(content); if (mScript.find()) return base.resolve(mScript.group(1)); return null; }
public SecretDetailResponse secretDetailsForId(long secretId) throws IOException { String response = httpGet(baseUrl.resolve(format("/admin/secrets/%d", secretId))); return mapper.readValue(response, SecretDetailResponse.class); }
public List<Client> allClients() throws IOException { String httpResponse = httpGet(baseUrl.resolve("/admin/clients/")); return mapper.readValue(httpResponse, new TypeReference<List<Client>>() {}); }
public GroupDetailResponse createGroup(String name, String description, ImmutableMap<String, String> metadata) throws IOException { checkArgument(!name.isEmpty()); String response = httpPost(baseUrl.resolve("/admin/groups"), new CreateGroupRequest(name, description, metadata)); return mapper.readValue(response, GroupDetailResponse.class); }
public Client getClientByName(String name) throws IOException { checkArgument(!name.isEmpty()); String response = httpGet(baseUrl.resolve("/admin/clients").newBuilder() .addQueryParameter("name", name) .build()); return mapper.readValue(response, Client.class); }
/** * Login to the Keywhiz server. * * Future requests made using this client instance will be authenticated. * @param username login username * @param password login password * @throws IOException if a network IO error occurs */ public void login(String username, char[] password) throws IOException { httpPost(baseUrl.resolve("/admin/login"), LoginRequest.from(username, password)); }