/** * Log a URIException from deep inside other components to the crawl's * shared log. * * @param e URIException encountered * @param u CrawlURI where problem occurred * @param l String which could not be interpreted as URI without exception */ public void logUriError(URIException e, UURI u, CharSequence l) { Object[] array = {u, l}; uriErrors.log(Level.INFO, e.getMessage(), array); }
protected boolean shouldExtract(CrawlURI uri) { if (getIgnoreUnexpectedHtml()) { try { // HTML was not expected (eg a GIF was expected) so ignore // (as if a soft 404) if (!isHtmlExpectedHere(uri)) { return false; } } catch (URIException e) { logger.severe("Failed expectedHTML test: " + e.getMessage()); // assume it's okay to extract } } String mime = uri.getContentType().toLowerCase(); if (mime.startsWith("text/html") || mime.startsWith("application/xhtml") || mime.startsWith("text/vnd.wap.wml") || mime.startsWith("application/vnd.wap.wml") || mime.startsWith("application/vnd.wap.xhtml")) { return true; } String contentPrefixLC = uri.getRecorder().getContentReplayPrefixString(1000).toLowerCase(); if (contentPrefixLC.contains("<html") || contentPrefixLC.contains("<!doctype html")) { return true; } return false; }
/** * Constructor specifying a URI. * It is responsibility of the caller to ensure that URI elements * (path & query parameters) are properly encoded (URL safe). * * @param uri either an absolute or relative URI. The URI is expected * to be URL-encoded * * @throws IllegalArgumentException when URI is invalid * @throws IllegalStateException when protocol of the absolute URI is not recognised */ public HttpMethodBase(String uri) throws IllegalArgumentException, IllegalStateException { try { // create a URI and allow for null/empty uri values if (uri == null || uri.equals("")) { uri = "/"; } String charset = getParams().getUriCharset(); setURI(new URI(uri, true, charset)); } catch (URIException e) { throw new IllegalArgumentException("Invalid uri '" + uri + "': " + e.getMessage() ); } }
public boolean isPrerequisite(final CrawlURI curi) { boolean result = false; String curiStr = curi.getUURI().toString(); String loginUri = getPrerequisite(curi); if (loginUri != null) { try { UURI uuri = UURIFactory.getInstance(curi.getUURI(), loginUri); if (uuri != null && curiStr != null && uuri.toString().equals(curiStr)) { result = true; if (!curi.isPrerequisite()) { curi.setPrerequisite(true); logger.fine(curi + " is prereq."); } } } catch (URIException e) { logger.severe("Failed to uuri: " + curi + ", " + e.getMessage()); } } return result; }
logger.debug("About to execute '{}'", method.getURI()); } catch (URIException e) { logger.debug("{}", e.getMessage());
req = new FetchHTTPRequest(this, curi); } catch (URIException e) { cleanup(curi, e, e.getMessage(), S_UNFETCHABLE_URI); return;
/** * Log a URIException from deep inside other components to the crawl's * shared log. * * @param e URIException encountered * @param u CrawlURI where problem occurred * @param l String which could not be interpreted as URI without exception */ public void logUriError(URIException e, UURI u, CharSequence l) { Object[] array = {u, l}; uriErrors.log(Level.INFO, e.getMessage(), array); }
public URL toURL() throws MalformedURLException { if(relPath!=null) return null; try { return new URL(httpUrl.getURI()); } catch (URIException e) { throw new MalformedURLException(e.getMessage()); } }
private String getHash(final boolean encoded) { if (hash_ == null || hash_.length() == 0) { return null; } if (encoded) { try { return URIUtil.encodeQuery(hash_); } catch (final URIException e) { LOG.error(e.getMessage(), e); } } return hash_; }
private String getHash(final boolean encoded) { if (hash_ == null || hash_.length() == 0) { return null; } if (encoded) { try { return URIUtil.encodeQuery(hash_); } catch (final URIException e) { LOG.error(e.getMessage(), e); } } return hash_; }
private String getHash(final boolean encoded) { if (hash_ == null || hash_.length() == 0) { return null; } if (encoded) { try { return URIUtil.encodeQuery(hash_); } catch (final URIException e) { LOG.error(e.getMessage(), e); } } return hash_; }
@Override public URI getUri() throws URISyntaxException { try { return new URI(postMethod.getURI().toString()); } catch (URIException ex) { throw new URISyntaxException("", ex.getMessage()); } }
public URI getUri() throws URISyntaxException { try { return new URI(postMethod.getURI().toString()); } catch (URIException ex) { throw new URISyntaxException("", ex.getMessage()); } }
public URI getURI() { try { return URI.create(this.httpMethod.getURI().getEscapedURI()); } catch (URIException ex) { throw new IllegalStateException("Could not get HttpMethod URI: " + ex.getMessage(), ex); } }
@Override public URI getUri() throws URISyntaxException { try { return new URI(postMethod.getURI().toString()); } catch (URIException ex) { throw new URISyntaxException("", ex.getMessage()); } }
@Override public URI getUri() throws URISyntaxException { try { return new URI(postMethod.getURI().toString()); } catch (URIException ex) { throw new URISyntaxException("", ex.getMessage()); } }
private String normalize(String uri) throws URISyntaxException { try { URI normalized = new URI(uri, DefaultHTTPClient.isUrlEncoded(uri)); normalized.normalize(); return normalized.toString(); } catch (URIException e) { LOG.warn("Invalid uri: {}", uri); LOG.error("Can not convert URL", e); throw new URISyntaxException(uri, e.getMessage()); } }
@Override protected void buildMessage(HttpMessage msg, String query) { try { // ZAP: encoding has been decided before inside the VariantAbstractQuery // implementation so now we have only to set a raw query string msg.getRequestHeader().getURI().setEscapedQuery(query); } catch (URIException e) { log.error(e.getMessage() + query, e); } } }
@Override public void addAuthentication(HttpMethod forMethod, HttpClient forClient) { try { forMethod.addRequestHeader("Authorization", "token " + accessToken); String url = forMethod.getURI().toString(); String separator = url.contains("?") ? "&" : "?"; url += separator + "access_token=" + getAccessToken(); forMethod.setURI(new URI(url, true)); } catch (URIException e) { throw new SourceControlException("Failed to decode/encode given URI. " + e.getMessage(), e); } }
protected void fixClientHost(HttpMethod method) { try { // update client host, workaround for Exchange 2003 mailbox with an Exchange 2007 frontend URI currentUri = method.getURI(); if (currentUri != null && currentUri.getHost() != null && currentUri.getScheme() != null) { httpClient.getHostConfiguration().setHost(currentUri.getHost(), currentUri.getPort(), currentUri.getScheme()); } } catch (URIException e) { LOGGER.warn("Unable to update http client host:" + e.getMessage(), e); } }