@Override public List<String> getURLsFromPage(Document doc) { List<String> result = new ArrayList<>(); for (Element el : doc.select("span > span > a.thumb")) { String thumbPageURL = el.attr("href"); Document thumbPage = null; if (checkURL(thumbPageURL)) { try { Connection.Response resp = Http.url(new URL(thumbPageURL)).response(); cookies.putAll(resp.cookies()); thumbPage = resp.parse(); } catch (MalformedURLException e) { LOGGER.info(thumbPageURL + " is a malformed URL"); } catch (IOException e) { LOGGER.info(e.getMessage()); } String imageDownloadUrl = thumbPage.select("a.dev-page-download").attr("href"); if (imageDownloadUrl != null && !imageDownloadUrl.equals("")) { result.add(getImageLinkFromDLLink(imageDownloadUrl)); } } } return result; }
@Override public Document getNextPage(Document page) throws IOException { this.currAlbum.pageIndex++; boolean endOfAlbum = currAlbum.pageIndex > currAlbum.numPages; boolean noMoreSubalbums = albums.isEmpty(); if (endOfAlbum && noMoreSubalbums){ throw new IOException("No more pages"); } try { Thread.sleep(WAIT_BEFORE_NEXT_PAGE); } catch (InterruptedException e) { LOGGER.info("Interrupted while waiting before getting next page"); } if (endOfAlbum){ LOGGER.info("Turning to next album " + albums.get(0).baseURL); return getFirstPage(); } else { LOGGER.info("Turning to page " + currAlbum.pageIndex + " of album " + currAlbum.baseURL); Connection.Response resp = Http.url(currAlbum.getCurrPageURL()).response(); currAlbum.cookies = resp.cookies(); currAlbum.currPage = resp.parse(); return currAlbum.currPage; } }
@Override public Document getFirstPage() throws IOException { if (this.currAlbum == null) { this.albums = getAlbumMetadata(this.url.toExternalForm()); LOGGER.info("Detected " + albums.size() + " albums in total"); } this.currAlbum = this.albums.remove(0); // NOTE: Why not just get media count in the metadata json? // // Because that data might not reflect what the user sees on the page // and can lead to iterating more pages than there actually are. // // An example: // Metadata JSON -> AlbumStats: 146 images + 0 videos -> 146 items/7 pages // http://s1255.photobucket.com/api/user/mimajki/album/Movie%20gifs/get?subAlbums=48&json=1 // Actual item count when looking at the album url: 131 items/6 pages // http://s1255.photobucket.com/user/mimajki/library/Movie%20gifs?sort=6&page=1 Connection.Response resp = Http.url(currAlbum.getCurrPageURL()).response(); this.currAlbum.cookies = resp.cookies(); this.currAlbum.currPage = resp.parse(); JSONObject collectionData = getCollectionData(currAlbum.currPage); int totalNumItems = collectionData.getInt("total"); this.currAlbum.numPages = (int) Math.ceil( (double)totalNumItems / (double)ITEMS_PER_PAGE); this.index = 0; return currAlbum.currPage; }
URL url = new URL(videoPageurl); Response response = Http.url(url).referrer(hqpornerVideoPageUrl).response(); Document doc = response.parse();
.cookies(cookies) .response(); Document page = res.parse();
public static List<URL> getURLs(URL url) throws IOException{ Response resp = Http.url(url) .ignoreContentType() .response(); Document doc = resp.parse(); List<URL> URLs = new ArrayList<>(); //Pictures Elements imgs = doc.getElementsByTag("img"); for (Element img : imgs) { if (img.hasClass("album-image")) { String imageURL = img.attr("src"); URLs.add(new URL(imageURL)); } } //Videos Elements vids = doc.getElementsByTag("video"); for (Element vid : vids) { if (vid.hasClass("album-video")) { Elements source = vid.getElementsByTag("source"); String videoURL = source.first().attr("src"); URLs.add(new URL(videoURL)); } } return URLs; } }
public String getDescription(String page) { try { // Fetch the image page Response resp = Http.url(page) .referrer(this.url) .response(); cookies.putAll(resp.cookies()); // Try to find the description Elements els = resp.parse().select("td[class=alt1][width=\"70%\"]"); if (els.isEmpty()) { LOGGER.debug("No description at " + page); throw new IOException("No description found"); } LOGGER.debug("Description found!"); Document documentz = resp.parse(); Element ele = documentz.select("td[class=alt1][width=\"70%\"]").get(0); // This is where the description is. // Would break completely if FurAffinity changed site layout. documentz.outputSettings(new Document.OutputSettings().prettyPrint(false)); ele.select("br").append("\\n"); ele.select("p").prepend("\\n\\n"); LOGGER.debug("Returning description at " + page); String tempPage = Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)); return documentz.select("meta[property=og:title]").attr("content") + "\n" + tempPage; // Overridden saveText takes first line and makes it the file name. } catch (IOException ioe) { LOGGER.info("Failed to get description " + page + " : '" + ioe.getMessage() + "'"); return null; } } @Override
private void login() throws IOException { Response resp = Http.url(this.url).response(); cookies = resp.cookies(); String ctoken = resp.parse().select("form > input[name=ctoken]").first().attr("value"); Map<String,String> postdata = new HashMap<>(); postdata.put("user[login]", new String(Base64.decode("cmlwbWU="))); postdata.put("user[password]", new String(Base64.decode("cmlwcGVy"))); postdata.put("rememberme", "1"); postdata.put("ctoken", ctoken); resp = Http.url("http://en.2dgalleries.com/account/login") .referrer("http://en.2dgalleries.com/") .cookies(cookies) .data(postdata) .method(Method.POST) .response(); cookies = resp.cookies(); } }
public Document get() throws IOException { connection.method(Method.GET); return response().parse(); }
public Document post() throws IOException { connection.method(Method.POST); return response().parse(); }
@Override public Document getFirstPage() throws IOException { Connection.Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); Document doc = resp.parse(); String blacklistedTag = RipUtils.checkTags(Utils.getConfigStringArray("tsumino.blacklist.tags"), getTags(doc)); if (blacklistedTag != null) { sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Skipping " + url.toExternalForm() + " as it " + "contains the blacklisted tag \"" + blacklistedTag + "\""); return null; } return doc; }
@Override public Document getFirstPage() throws IOException { if (albumDoc == null) { Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); albumDoc = resp.parse(); } return albumDoc; }
@Override public Document getFirstPage() throws IOException { if (albumDoc == null) { Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); albumDoc = resp.parse(); } return albumDoc; }
@Override public Document getFirstPage() throws IOException { Response resp = Http.url(this.url) .ignoreContentType() .response(); return resp.parse(); }
@Override public JSONObject getFirstPage() throws IOException { Connection.Response resp = Http.url(url).response(); LOGGER.info(resp.cookies()); csrftoken = resp.cookie("csrftoken"); Document p = resp.parse(); // Get the query hash so we can download the next page qHash = getQHash(p); return getJSONFromPage(p); }
@Override public Document getFirstPage() throws IOException { if (albumDoc == null) { Response resp = Http.url(url).response(); cookies.putAll(resp.cookies()); albumDoc = resp.parse(); } return albumDoc; }
@Override public Document getFirstPage() throws IOException { String urlToDownload = this.url.toExternalForm(); Response resp = Http.url(urlToDownload.replace("eroshare.com", "eroshae.com")) .ignoreContentType() .response(); return resp.parse(); }
public Document post() throws IOException { req.method(Method.POST); execute(); return res.parse(); }
public Document get() throws IOException { req.method(Method.GET); execute(); return res.parse(); }