public static HttpResult httpPost(String url, List<String> headers, Map<String, String> paramValues, String encoding) { try { HttpPost httpost = new HttpPost(url); RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(5000).setConnectTimeout(5000).setSocketTimeout(5000).setRedirectsEnabled(true).setMaxRedirects(5).build(); httpost.setConfig(requestConfig); List<NameValuePair> nvps = new ArrayList<NameValuePair>(); for (Map.Entry<String, String> entry : paramValues.entrySet()) { nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); } httpost.setEntity(new UrlEncodedFormEntity(nvps, encoding)); HttpResponse response = postClient.execute(httpost); HttpEntity entity = response.getEntity(); String charset = encoding; if (entity.getContentType() != null) { HeaderElement[] headerElements = entity.getContentType().getElements(); if (headerElements != null && headerElements.length > 0 && headerElements[0] != null && headerElements[0].getParameterByName("charset") != null) { charset = headerElements[0].getParameterByName("charset").getValue(); } } return new HttpResult(response.getStatusLine().getStatusCode(), IOUtils.toString(entity.getContent(), charset), Collections.<String, String>emptyMap()); } catch (Throwable e) { return new HttpResult(500, e.toString(), Collections.<String, String>emptyMap()); } }
protected void configureHttpGet(HttpGet httpGet, long timeout) { httpGet.setConfig(RequestConfig.custom() .setConnectTimeout((int) timeout) .setSocketTimeout((int) timeout) .setConnectionRequestTimeout((int) timeout) .setRedirectsEnabled(true) .setCircularRedirectsAllowed(true) .setMaxRedirects(20) .setCookieSpec(CookieSpecs.STANDARD) .build() ); }
public static CloseableHttpClient createHttpClient(final int maxRedirects) throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { final Registry<ConnectionSocketFactory> socketFactoryRegistry = createSocketFactoryConfigration(); final BasicCookieStore cookieStore = new BasicCookieStore(); return HttpClientBuilder.create() .setConnectionManager(new PoolingHttpClientConnectionManager(socketFactoryRegistry)) .setRedirectStrategy(new LaxRedirectStrategy()) .setDefaultRequestConfig(RequestConfig.custom().setCookieSpec(CookieSpecs.DEFAULT).setMaxRedirects(maxRedirects).build()) .setDefaultCookieStore(cookieStore) .setRetryHandler(new StandardHttpRequestRetryHandler()) .build(); }
@SuppressWarnings("deprecation") public static RequestConfig.Builder copy(final RequestConfig config) { return new Builder() .setExpectContinueEnabled(config.isExpectContinueEnabled()) .setProxy(config.getProxy()) .setLocalAddress(config.getLocalAddress()) .setStaleConnectionCheckEnabled(config.isStaleConnectionCheckEnabled()) .setCookieSpec(config.getCookieSpec()) .setRedirectsEnabled(config.isRedirectsEnabled()) .setRelativeRedirectsAllowed(config.isRelativeRedirectsAllowed()) .setCircularRedirectsAllowed(config.isCircularRedirectsAllowed()) .setMaxRedirects(config.getMaxRedirects()) .setAuthenticationEnabled(config.isAuthenticationEnabled()) .setTargetPreferredAuthSchemes(config.getTargetPreferredAuthSchemes()) .setProxyPreferredAuthSchemes(config.getProxyPreferredAuthSchemes()) .setConnectionRequestTimeout(config.getConnectionRequestTimeout()) .setConnectTimeout(config.getConnectTimeout()) .setSocketTimeout(config.getSocketTimeout()) .setDecompressionEnabled(config.isDecompressionEnabled()) .setContentCompressionEnabled(config.isContentCompressionEnabled()); }
@SuppressWarnings("deprecation") public static RequestConfig.Builder copy(final RequestConfig config) { return new Builder() .setExpectContinueEnabled(config.isExpectContinueEnabled()) .setProxy(config.getProxy()) .setLocalAddress(config.getLocalAddress()) .setStaleConnectionCheckEnabled(config.isStaleConnectionCheckEnabled()) .setCookieSpec(config.getCookieSpec()) .setRedirectsEnabled(config.isRedirectsEnabled()) .setRelativeRedirectsAllowed(config.isRelativeRedirectsAllowed()) .setCircularRedirectsAllowed(config.isCircularRedirectsAllowed()) .setMaxRedirects(config.getMaxRedirects()) .setAuthenticationEnabled(config.isAuthenticationEnabled()) .setTargetPreferredAuthSchemes(config.getTargetPreferredAuthSchemes()) .setProxyPreferredAuthSchemes(config.getProxyPreferredAuthSchemes()) .setConnectionRequestTimeout(config.getConnectionRequestTimeout()) .setConnectTimeout(config.getConnectTimeout()) .setSocketTimeout(config.getSocketTimeout()) .setDecompressionEnabled(config.isDecompressionEnabled()) .setContentCompressionEnabled(config.isContentCompressionEnabled()); }
@SuppressWarnings("deprecation") public static RequestConfig.Builder copy(final RequestConfig config) { return new Builder() .setExpectContinueEnabled(config.isExpectContinueEnabled()) .setProxy(config.getProxy()) .setLocalAddress(config.getLocalAddress()) .setStaleConnectionCheckEnabled(config.isStaleConnectionCheckEnabled()) .setCookieSpec(config.getCookieSpec()) .setRedirectsEnabled(config.isRedirectsEnabled()) .setRelativeRedirectsAllowed(config.isRelativeRedirectsAllowed()) .setCircularRedirectsAllowed(config.isCircularRedirectsAllowed()) .setMaxRedirects(config.getMaxRedirects()) .setAuthenticationEnabled(config.isAuthenticationEnabled()) .setTargetPreferredAuthSchemes(config.getTargetPreferredAuthSchemes()) .setProxyPreferredAuthSchemes(config.getProxyPreferredAuthSchemes()) .setConnectionRequestTimeout(config.getConnectionRequestTimeout()) .setConnectTimeout(config.getConnectTimeout()) .setSocketTimeout(config.getSocketTimeout()) .setDecompressionEnabled(config.isDecompressionEnabled()) .setContentCompressionEnabled(config.isContentCompressionEnabled()); }
@SuppressWarnings("deprecation") public static RequestConfig.Builder copy(final RequestConfig config) { return new Builder() .setExpectContinueEnabled(config.isExpectContinueEnabled()) .setProxy(config.getProxy()) .setLocalAddress(config.getLocalAddress()) .setStaleConnectionCheckEnabled(config.isStaleConnectionCheckEnabled()) .setCookieSpec(config.getCookieSpec()) .setRedirectsEnabled(config.isRedirectsEnabled()) .setRelativeRedirectsAllowed(config.isRelativeRedirectsAllowed()) .setCircularRedirectsAllowed(config.isCircularRedirectsAllowed()) .setMaxRedirects(config.getMaxRedirects()) .setAuthenticationEnabled(config.isAuthenticationEnabled()) .setTargetPreferredAuthSchemes(config.getTargetPreferredAuthSchemes()) .setProxyPreferredAuthSchemes(config.getProxyPreferredAuthSchemes()) .setConnectionRequestTimeout(config.getConnectionRequestTimeout()) .setConnectTimeout(config.getConnectTimeout()) .setSocketTimeout(config.getSocketTimeout()) .setDecompressionEnabled(config.isDecompressionEnabled()) .setContentCompressionEnabled(config.isContentCompressionEnabled()); }
private static RequestConfig prepareConfig(HttpClientRequestOptions httpClientRequestOptions) { RequestConfig.Builder configBuilder = RequestConfig.custom(); configBuilder.setAuthenticationEnabled(httpClientRequestOptions.isAuthenticationEnabled()) .setCircularRedirectsAllowed(httpClientRequestOptions.isCircularRedirectsAllowed()) .setContentCompressionEnabled(httpClientRequestOptions.isContentCompressionEnabled()) .setExpectContinueEnabled(httpClientRequestOptions.isExpectContinueEnabled()) .setRedirectsEnabled(httpClientRequestOptions.isRedirectsEnabled()) .setRelativeRedirectsAllowed(httpClientRequestOptions.isRelativeRedirectsAllowed()) .setMaxRedirects(httpClientRequestOptions.getMaxRedirects()) .setConnectionRequestTimeout(httpClientRequestOptions.getConnectionRequestTimeout()) .setConnectTimeout(httpClientRequestOptions.getConnectTimeout()).setSocketTimeout(httpClientRequestOptions.getSocketTimeout()) .setCookieSpec(httpClientRequestOptions.getCookieSpec()) .setProxyPreferredAuthSchemes(httpClientRequestOptions.getProxyPreferredAuthSchemes()) .setTargetPreferredAuthSchemes(httpClientRequestOptions.getTargetPreferredAuthSchemes()); if ((httpClientRequestOptions.getProxyHost() != null) && (httpClientRequestOptions.getProxyPort() != 0)) { configBuilder.setProxy(new HttpHost(httpClientRequestOptions.getProxyHost(), httpClientRequestOptions.getProxyPort())); } RequestConfig config = configBuilder.build(); return config; }
private static RequestConfig prepareConfig(HttpClientRequestOptions httpClientRequestOptions) { RequestConfig.Builder configBuilder = RequestConfig.custom(); configBuilder.setAuthenticationEnabled(httpClientRequestOptions.isAuthenticationEnabled()) .setCircularRedirectsAllowed(httpClientRequestOptions.isCircularRedirectsAllowed()) .setContentCompressionEnabled(httpClientRequestOptions.isContentCompressionEnabled()) .setExpectContinueEnabled(httpClientRequestOptions.isExpectContinueEnabled()) .setRedirectsEnabled(httpClientRequestOptions.isRedirectsEnabled()) .setRelativeRedirectsAllowed(httpClientRequestOptions.isRelativeRedirectsAllowed()) .setMaxRedirects(httpClientRequestOptions.getMaxRedirects()) .setConnectionRequestTimeout(httpClientRequestOptions.getConnectionRequestTimeout()) .setConnectTimeout(httpClientRequestOptions.getConnectTimeout()).setSocketTimeout(httpClientRequestOptions.getSocketTimeout()) .setCookieSpec(httpClientRequestOptions.getCookieSpec()) .setProxyPreferredAuthSchemes(httpClientRequestOptions.getProxyPreferredAuthSchemes()) .setTargetPreferredAuthSchemes(httpClientRequestOptions.getTargetPreferredAuthSchemes()); if ((httpClientRequestOptions.getProxyHost() != null) && (httpClientRequestOptions.getProxyPort() != 0)) { configBuilder.setProxy(new HttpHost(httpClientRequestOptions.getProxyHost(), httpClientRequestOptions.getProxyPort())); } RequestConfig config = configBuilder.build(); return config; }
public static RequestConfig.Builder copy(final RequestConfig config) { return new Builder() .setExpectContinueEnabled(config.isExpectContinueEnabled()) .setProxy(config.getProxy()) .setLocalAddress(config.getLocalAddress()) .setStaleConnectionCheckEnabled(config.isStaleConnectionCheckEnabled()) .setCookieSpec(config.getCookieSpec()) .setRedirectsEnabled(config.isRedirectsEnabled()) .setRelativeRedirectsAllowed(config.isRelativeRedirectsAllowed()) .setCircularRedirectsAllowed(config.isCircularRedirectsAllowed()) .setMaxRedirects(config.getMaxRedirects()) .setAuthenticationEnabled(config.isAuthenticationEnabled()) .setTargetPreferredAuthSchemes(config.getTargetPreferredAuthSchemes()) .setProxyPreferredAuthSchemes(config.getProxyPreferredAuthSchemes()) .setConnectionRequestTimeout(config.getConnectionRequestTimeout()) .setConnectTimeout(config.getConnectTimeout()) .setSocketTimeout(config.getSocketTimeout()); }
public static RequestConfig.Builder copy(final RequestConfig config) { return new Builder() .setExpectContinueEnabled(config.isExpectContinueEnabled()) .setProxy(config.getProxy()) .setLocalAddress(config.getLocalAddress()) .setStaleConnectionCheckEnabled(config.isStaleConnectionCheckEnabled()) .setCookieSpec(config.getCookieSpec()) .setRedirectsEnabled(config.isRedirectsEnabled()) .setRelativeRedirectsAllowed(config.isRelativeRedirectsAllowed()) .setCircularRedirectsAllowed(config.isCircularRedirectsAllowed()) .setMaxRedirects(config.getMaxRedirects()) .setAuthenticationEnabled(config.isAuthenticationEnabled()) .setTargetPreferredAuthSchemes(config.getTargetPreferredAuthSchemes()) .setProxyPreferredAuthSchemes(config.getProxyPreferredAuthSchemes()) .setConnectionRequestTimeout(config.getConnectionRequestTimeout()) .setConnectTimeout(config.getConnectTimeout()) .setSocketTimeout(config.getSocketTimeout()); }
/** * 针对每次Http请求需要设置的一些参数 * @param spiderParams * @return */ public RequestConfig setRequestConfig(SpiderParams spiderParams) { RequestConfig requestConfig = RequestConfig.copy(setGlobalRequestConfig()) //连接成功后,读取IO超时或者两次读写之间的间隔时间超时 .setSocketTimeout(spiderParams.getHttpReadTimeout()) //一个请求到与服务器建立连接的超时时间 .setConnectTimeout(spiderParams.getHttpConnectionTimeout()) .setConnectionRequestTimeout(5000) .setRedirectsEnabled(spiderParams.isRedirect()) .setCircularRedirectsAllowed(true) .setMaxRedirects(100) .build(); return requestConfig; }
HttpBuilderBase() { credsProvider = new BasicCredentialsProvider(); config.setMaxRedirects(20); }
HttpBuilderBase() { credsProvider = new BasicCredentialsProvider(); config.setMaxRedirects(20); }
private CloseableHttpClient buildClient(int timeoutInMillis, URL url) { RequestConfig config = RequestConfig.custom().setConnectionRequestTimeout(timeoutInMillis) .setConnectTimeout(timeoutInMillis).setRedirectsEnabled(true).setMaxRedirects(MAX_REDIRECTS) .setProxy(this.proxyFactory.getProxy(url)).build(); return HttpClientBuilder.create().setDefaultRequestConfig(config) .setDefaultConnectionConfig( ConnectionConfig .custom().setMessageConstraints(MessageConstraints.custom() .setMaxHeaderCount(MAX_HEADER_COUNT).setMaxLineLength(MAX_LINE_LENGTH).build()) .build()) .build(); }
/** * 设置重定向的次数 * * @param maxRedirects * @return */ public HttpUtils maxRedirects(int maxRedirects){ config.setMaxRedirects(maxRedirects); return this; }
public static RequestConfig toRequestConfig(DynamicPoolingHttpClientProviderConfig config) { Objects.requireNonNull(config, "config is null"); return RequestConfig.custom().setAuthenticationEnabled(config.getAuthenticationEnabled()) .setCircularRedirectsAllowed(config.getCircularRedirectsAllowed()) .setConnectionRequestTimeout(config.getConnectionRequestTimeout()) .setConnectTimeout(config.getConnectTimeout()) .setContentCompressionEnabled(config.getContentCompressionEnabled()) .setExpectContinueEnabled(config.getExpectContinueEnabled()).setMaxRedirects(config.getMaxRedirects()) .setRedirectsEnabled(config.getRedirectsEnabled()) .setRelativeRedirectsAllowed(config.getRelativeRedirectsAllowed()) .setSocketTimeout(config.getSocketTimeout()).build(); }
private CloseableHttpClient buildClient(int timeoutInMillis, URL url) { RequestConfig config = RequestConfig.custom().setConnectionRequestTimeout(timeoutInMillis) .setConnectTimeout(timeoutInMillis).setRedirectsEnabled(true).setMaxRedirects(MAX_REDIRECTS) .setProxy(this.proxyFactory.getProxy(url)).build(); return HttpClientBuilder.create().setDefaultRequestConfig(config) .setDefaultConnectionConfig( ConnectionConfig .custom().setMessageConstraints(MessageConstraints.custom() .setMaxHeaderCount(MAX_HEADER_COUNT).setMaxLineLength(MAX_LINE_LENGTH).build()) .build()) .build(); }
private RequestConfig createRequestConfig(HttpConfig config) { return RequestConfig.custom() .setConnectionRequestTimeout(config.getConnectTimeoutMillis()) .setConnectTimeout(config.getConnectTimeoutMillis()) .setSocketTimeout(config.getRequestTimeoutMillis()) .setRedirectsEnabled(config.isFollowRedirects()) .setMaxRedirects(config.getMaxRedirects()) .setRelativeRedirectsAllowed(config.isRejectRelativeRedirects()) .build(); }
public static void requestConfig() { reqConf = RequestConfig.custom() .setSocketTimeout(5000) .setConnectTimeout(5000) .setConnectionRequestTimeout(2000) .setRedirectsEnabled(false) .setMaxRedirects(0) .build(); standardHandler = new StandardHttpRequestRetryHandler(3, true); }