private SyndFeed getSyndFeedFromStream(final InputStream inputStream, final URLConnection connection) throws IOException, IllegalArgumentException, FeedException { final SyndFeed feed = readSyndFeedFromStream(inputStream, connection); fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, connection, feed); return feed; }
protected void retrieveAndCacheFeed(final URL feedUrl, final SyndFeedInfo syndFeedInfo, final HttpURLConnection connection) throws IllegalArgumentException, FeedException, FetcherException, IOException { handleErrorCodes(connection.getResponseCode()); resetFeedInfo(feedUrl, syndFeedInfo, connection); final FeedFetcherCache cache = getFeedInfoCache(); // resetting feed info in the cache // could be needed for some implementations // of FeedFetcherCache (eg, distributed HashTables) if (cache != null) { cache.setFeedInfo(feedUrl, syndFeedInfo); } }
@Override public SyndFeed retrieveFeed(final URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException { return this.retrieveFeed(getUserAgent(), feedUrl); }
final FeedFetcherCache cache = getFeedInfoCache(); if (cache != null) { SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl); setRequestHeaders(connection, syndFeedInfo, userAgent); httpConnection.connect(); try { fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, connection); retrieveAndCacheFeed(feedUrl, syndFeedInfo, httpConnection); } else { retrieveAndCacheFeed(feedUrl, syndFeedInfo, httpConnection); } else { fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, connection); fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, connection); InputStream inputStream = null; setRequestHeaders(connection, null, userAgent); return getSyndFeedFromStream(inputStream, connection); } catch (final java.io.IOException e) { handleErrorCodes(((HttpURLConnection) connection).getResponseCode()); } finally { IO.close(inputStream);
try { inputStream = connection.getInputStream(); SyndFeed syndFeed = getSyndFeedFromStream(inputStream, connection); if (isUsingDeltaEncoding() && imHeader != null && imHeader.contains("feed")) { final FeedFetcherCache cache = getFeedInfoCache(); if (cache != null && connection.getResponseCode() == 226) { syndFeed = combineFeeds(cachedFeed, syndFeed);
private FeedFetcher getRomeFetcher() { FeedFetcherCache feedCache = getRomeFetcherCache(); FeedFetcher feedFetcher; if(feedCache != null) { feedFetcher = new HttpURLFeedFetcher(feedCache); } else { feedFetcher = new HttpURLFeedFetcher(); } // set options feedFetcher.setUsingDeltaEncoding(false); feedFetcher.setUserAgent("RollerPlanetAggregator"); return feedFetcher; }
private SyndFeed readSyndFeedFromStream(final InputStream inputStream, final URLConnection connection) throws IOException, IllegalArgumentException, FeedException { BufferedInputStream is; if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) { // handle gzip encoded content is = new BufferedInputStream(new GZIPInputStream(inputStream)); } else { is = new BufferedInputStream(inputStream); } final XmlReader reader; if (connection.getHeaderField("Content-Type") != null) { reader = new XmlReader(is, connection.getHeaderField("Content-Type"), true); } else { reader = new XmlReader(is, true); } final SyndFeedInput syndFeedInput = new SyndFeedInput(); syndFeedInput.setPreserveWireFeed(isPreserveWireFeed()); syndFeedInput.setAllowDoctypes(isAllowDoctypes()); return syndFeedInput.build(reader); }
/** * <p> * Set appropriate HTTP headers, including conditional get and gzip encoding headers * </p> * * @param connection A URLConnection * @param syndFeedInfo The SyndFeedInfo for the feed to be retrieved. May be null * @param userAgent the name of the user-agent to be placed in HTTP-header. */ protected void setRequestHeaders(final URLConnection connection, final SyndFeedInfo syndFeedInfo, final String userAgent) { if (syndFeedInfo != null) { // set the headers to get feed only if modified // we support the use of both last modified and eTag headers if (syndFeedInfo.getLastModified() != null) { final Object lastModified = syndFeedInfo.getLastModified(); if (lastModified instanceof Long) { connection.setIfModifiedSince((Long) syndFeedInfo.getLastModified()); } } if (syndFeedInfo.getETag() != null) { connection.setRequestProperty("If-None-Match", syndFeedInfo.getETag()); } } // header to retrieve feed gzipped connection.setRequestProperty("Accept-Encoding", "gzip"); connection.addRequestProperty("User-Agent", userAgent); if (isUsingDeltaEncoding()) { connection.addRequestProperty("A-IM", "feed"); } }
public RSSRecordReader(String url) throws MalformedURLException { feedUrl = new URL(url); feedInfoCache = HashMapFeedInfoCache.getInstance(); fetcher = new HttpURLFeedFetcher(feedInfoCache); listener = new FetcherEventListenerImpl(this, LOGGER); fetcher.addFetcherEventListener(listener); }