private void setupFromConnection(HttpURLConnection conn, Connection.Response previousResponse) throws IOException { method = Method.valueOf(conn.getRequestMethod()); url = conn.getURL(); statusCode = conn.getResponseCode(); statusMessage = conn.getResponseMessage(); contentType = conn.getContentType(); Map<String, List<String>> resHeaders = createHeaderMap(conn); processResponseHeaders(resHeaders); // if from a redirect, map previous response cookies into this response if (previousResponse != null) { for (Map.Entry<String, String> prevCookie : previousResponse.cookies().entrySet()) { if (!hasCookie(prevCookie.getKey())) cookie(prevCookie.getKey(), prevCookie.getValue()); } } }
@Override public List<String> getURLsFromPage(Document doc) { List<String> result = new ArrayList<>(); for (Element el : doc.select("span > span > a.thumb")) { String thumbPageURL = el.attr("href"); Document thumbPage = null; if (checkURL(thumbPageURL)) { try { Connection.Response resp = Http.url(new URL(thumbPageURL)).response(); cookies.putAll(resp.cookies()); thumbPage = resp.parse(); } catch (MalformedURLException e) { LOGGER.info(thumbPageURL + " is a malformed URL"); } catch (IOException e) { LOGGER.info(e.getMessage()); } String imageDownloadUrl = thumbPage.select("a.dev-page-download").attr("href"); if (imageDownloadUrl != null && !imageDownloadUrl.equals("")) { result.add(getImageLinkFromDLLink(imageDownloadUrl)); } } } return result; }
@Override public Document getNextPage(Document page) throws IOException { this.currAlbum.pageIndex++; boolean endOfAlbum = currAlbum.pageIndex > currAlbum.numPages; boolean noMoreSubalbums = albums.isEmpty(); if (endOfAlbum && noMoreSubalbums){ throw new IOException("No more pages"); } try { Thread.sleep(WAIT_BEFORE_NEXT_PAGE); } catch (InterruptedException e) { LOGGER.info("Interrupted while waiting before getting next page"); } if (endOfAlbum){ LOGGER.info("Turning to next album " + albums.get(0).baseURL); return getFirstPage(); } else { LOGGER.info("Turning to page " + currAlbum.pageIndex + " of album " + currAlbum.baseURL); Connection.Response resp = Http.url(currAlbum.getCurrPageURL()).response(); currAlbum.cookies = resp.cookies(); currAlbum.currPage = resp.parse(); return currAlbum.currPage; } }
public Map<String, String> getCookies() { return getResponse().cookies(); }
private Map<String, String> requestNewCookie(String fileName) { Map<String, String> cookies = null; try{ Response response = getConnection(BaseURL).ignoreHttpErrors(true).execute(); cookies = response.cookies(); String gsp = cookies.get("GSP"); cookies.put("GSP", gsp + ":CF=4"); // :CF=4 enables the export to BibTex Link in the result list saveCookies(cookies, fileName); }catch(IOException e){ logger.info(e.getMessage(), e); } return cookies; }
@Override public Document getNextPage(Document doc) throws IOException { page++; Map<String,String> postData = new HashMap<String,String>(); postData.put("cmd", "refreshData"); postData.put("view", "gif"); postData.put("layout", "grid"); postData.put("page", Integer.toString(page)); Response resp = Http.url(this.url) .ignoreContentType() .data(postData) .cookies(cookies) .method(Method.POST) .response(); cookies.putAll(resp.cookies()); Document nextDoc = resp.parse(); if (nextDoc.select("div.gif img").size() == 0) { throw new IOException("No more images found"); } sleep(2000); return nextDoc; }
@Override public Document getFirstPage() throws IOException { Response resp = Http.url(this.url) .ignoreContentType() .response(); cookies = resp.cookies(); Document doc = resp.parse(); if (doc.html().contains("profile is private")) { sendUpdate(STATUS.RIP_ERRORED, "User has private profile"); throw new IOException("User has private profile"); } return doc; }
.cookies(cookies) .response(); cookies.putAll(resp.cookies()); Document doc = resp.parse(); Elements els = doc.select("img.dev-content-full");
username = getUsernameFromURL(url.toExternalForm()); csrf = firstPageJSON.getString("csrf"); pageCookies = res.cookies();
public String getDescription(String page) { try { // Fetch the image page Response resp = Http.url(page) .referrer(this.url) .response(); cookies.putAll(resp.cookies()); // Try to find the description Elements els = resp.parse().select("td[class=alt1][width=\"70%\"]"); if (els.isEmpty()) { LOGGER.debug("No description at " + page); throw new IOException("No description found"); } LOGGER.debug("Description found!"); Document documentz = resp.parse(); Element ele = documentz.select("td[class=alt1][width=\"70%\"]").get(0); // This is where the description is. // Would break completely if FurAffinity changed site layout. documentz.outputSettings(new Document.OutputSettings().prettyPrint(false)); ele.select("br").append("\\n"); ele.select("p").prepend("\\n\\n"); LOGGER.debug("Returning description at " + page); String tempPage = Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)); return documentz.select("meta[property=og:title]").attr("content") + "\n" + tempPage; // Overridden saveText takes first line and makes it the file name. } catch (IOException ioe) { LOGGER.info("Failed to get description " + page + " : '" + ioe.getMessage() + "'"); return null; } } @Override
@Override public Document getFirstPage() throws IOException { if (this.currAlbum == null) { this.albums = getAlbumMetadata(this.url.toExternalForm()); LOGGER.info("Detected " + albums.size() + " albums in total"); } this.currAlbum = this.albums.remove(0); // NOTE: Why not just get media count in the metadata json? // // Because that data might not reflect what the user sees on the page // and can lead to iterating more pages than there actually are. // // An example: // Metadata JSON -> AlbumStats: 146 images + 0 videos -> 146 items/7 pages // http://s1255.photobucket.com/api/user/mimajki/album/Movie%20gifs/get?subAlbums=48&json=1 // Actual item count when looking at the album url: 131 items/6 pages // http://s1255.photobucket.com/user/mimajki/library/Movie%20gifs?sort=6&page=1 Connection.Response resp = Http.url(currAlbum.getCurrPageURL()).response(); this.currAlbum.cookies = resp.cookies(); this.currAlbum.currPage = resp.parse(); JSONObject collectionData = getCollectionData(currAlbum.currPage); int totalNumItems = collectionData.getInt("total"); this.currAlbum.numPages = (int) Math.ceil( (double)totalNumItems / (double)ITEMS_PER_PAGE); this.index = 0; return currAlbum.currPage; }
private void login() throws IOException { Response resp = Http.url(this.url).response(); cookies = resp.cookies(); String ctoken = resp.parse().select("form > input[name=ctoken]").first().attr("value"); Map<String,String> postdata = new HashMap<>(); postdata.put("user[login]", new String(Base64.decode("cmlwbWU="))); postdata.put("user[password]", new String(Base64.decode("cmlwcGVy"))); postdata.put("rememberme", "1"); postdata.put("ctoken", ctoken); resp = Http.url("http://en.2dgalleries.com/account/login") .referrer("http://en.2dgalleries.com/") .cookies(cookies) .data(postdata) .method(Method.POST) .response(); cookies = resp.cookies(); } }
@Override public Document getFirstPage() throws IOException { Connection.Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); Document doc = resp.parse(); String blacklistedTag = RipUtils.checkTags(Utils.getConfigStringArray("tsumino.blacklist.tags"), getTags(doc)); if (blacklistedTag != null) { sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Skipping " + url.toExternalForm() + " as it " + "contains the blacklisted tag \"" + blacklistedTag + "\""); return null; } return doc; }
@Override public Document getFirstPage() throws IOException { Response resp = Http.url(url).response(); cookies = resp.cookies(); return Http.url(url).get(); }
@Override public JSONObject getFirstPage() throws IOException { Connection.Response resp = Http.url(url).response(); LOGGER.info(resp.cookies()); csrftoken = resp.cookie("csrftoken"); Document p = resp.parse(); // Get the query hash so we can download the next page qHash = getQHash(p); return getJSONFromPage(p); }
@Override public Document getFirstPage() throws IOException { if (albumDoc == null) { Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); albumDoc = resp.parse(); } return albumDoc; }
@Override public Document getFirstPage() throws IOException { if (albumDoc == null) { Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); albumDoc = resp.parse(); } return albumDoc; }
@Override public Document getFirstPage() throws IOException { if (albumDoc == null) { Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); albumDoc = resp.parse(); } return albumDoc; }
@Signature public Map<String, String> cookies() { return getWrappedObject().cookies(); } }