123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158 |
- package com.nosum.common.util;
- import org.apache.http.HttpException;
- import org.apache.http.HttpRequest;
- import org.apache.http.HttpRequestInterceptor;
- import org.apache.http.client.CookieStore;
- import org.apache.http.config.Registry;
- import org.apache.http.config.RegistryBuilder;
- import org.apache.http.config.SocketConfig;
- import org.apache.http.conn.socket.ConnectionSocketFactory;
- import org.apache.http.conn.socket.PlainConnectionSocketFactory;
- import org.apache.http.conn.ssl.DefaultHostnameVerifier;
- import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
- import org.apache.http.impl.client.BasicCookieStore;
- import org.apache.http.impl.client.CloseableHttpClient;
- import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
- import org.apache.http.impl.client.HttpClientBuilder;
- import org.apache.http.impl.client.HttpClients;
- import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
- import org.apache.http.impl.cookie.BasicClientCookie;
- import org.apache.http.protocol.HttpContext;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import us.codecraft.webmagic.Site;
- import us.codecraft.webmagic.downloader.CustomRedirectStrategy;
- import javax.net.ssl.SSLContext;
- import javax.net.ssl.TrustManager;
- import javax.net.ssl.X509TrustManager;
- import java.io.IOException;
- import java.security.KeyManagementException;
- import java.security.NoSuchAlgorithmException;
- import java.security.cert.CertificateException;
- import java.security.cert.X509Certificate;
- import java.util.Map;
- /**
- * @author: sumbytes
- * @date: 2019/8/3 14:57
- * @modified:
- */
- public class HttpClientGenerator {
- private transient Logger logger = LoggerFactory.getLogger(getClass());
- private PoolingHttpClientConnectionManager connectionManager;
- public HttpClientGenerator() {
- Registry<ConnectionSocketFactory> reg = RegistryBuilder.<ConnectionSocketFactory>create()
- .register("http", PlainConnectionSocketFactory.INSTANCE)
- .register("https", buildSSLConnectionSocketFactory())
- .build();
- connectionManager = new PoolingHttpClientConnectionManager(reg);
- connectionManager.setDefaultMaxPerRoute(100);
- }
- private SSLConnectionSocketFactory buildSSLConnectionSocketFactory() {
- try {
- return new SSLConnectionSocketFactory(createIgnoreVerifySSL(), new String[]{"SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"},
- null,
- new DefaultHostnameVerifier()); // 优先绕过安全证书
- } catch (KeyManagementException e) {
- logger.error("ssl connection fail", e);
- } catch (NoSuchAlgorithmException e) {
- logger.error("ssl connection fail", e);
- }
- return SSLConnectionSocketFactory.getSocketFactory();
- }
- private SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
- // 实现一个X509TrustManager接口,用于绕过验证,不用修改里面的方法
- X509TrustManager trustManager = new X509TrustManager() {
- @Override
- public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
- }
- @Override
- public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
- }
- @Override
- public X509Certificate[] getAcceptedIssuers() {
- return null;
- }
- };
- SSLContext sc = SSLContext.getInstance("SSLv3");
- sc.init(null, new TrustManager[]{trustManager}, null);
- return sc;
- }
- public HttpClientGenerator setPoolSize(int poolSize) {
- connectionManager.setMaxTotal(poolSize);
- return this;
- }
- public CloseableHttpClient getClient(Site site) {
- return generateClient(site);
- }
- private CloseableHttpClient generateClient(Site site) {
- HttpClientBuilder httpClientBuilder = HttpClients.custom();
- httpClientBuilder.setConnectionManager(connectionManager);
- if (site.getUserAgent() != null) {
- httpClientBuilder.setUserAgent(site.getUserAgent());
- } else {
- httpClientBuilder.setUserAgent("");
- }
- if (site.isUseGzip()) {
- httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
- @Override
- public void process(
- HttpRequest request,
- HttpContext context) throws HttpException, IOException {
- if (!request.containsHeader("Accept-Encoding")) {
- request.addHeader("Accept-Encoding", "gzip");
- }
- }
- });
- }
- //解决post/redirect/post 302跳转问题
- httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy());
- SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
- socketConfigBuilder.setSoKeepAlive(true).setTcpNoDelay(true);
- socketConfigBuilder.setSoTimeout(site.getTimeOut());
- SocketConfig socketConfig = socketConfigBuilder.build();
- httpClientBuilder.setDefaultSocketConfig(socketConfig);
- connectionManager.setDefaultSocketConfig(socketConfig);
- httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
- generateCookie(httpClientBuilder, site);
- return httpClientBuilder.build();
- }
- private void generateCookie(HttpClientBuilder httpClientBuilder, Site site) {
- if (site.isDisableCookieManagement()) {
- httpClientBuilder.disableCookieManagement();
- return;
- }
- CookieStore cookieStore = new BasicCookieStore();
- for (Map.Entry<String, String> cookieEntry : site.getCookies().entrySet()) {
- BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());
- cookie.setDomain(site.getDomain());
- cookieStore.addCookie(cookie);
- }
- for (Map.Entry<String, Map<String, String>> domainEntry : site.getAllCookies().entrySet()) {
- for (Map.Entry<String, String> cookieEntry : domainEntry.getValue().entrySet()) {
- BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());
- cookie.setDomain(domainEntry.getKey());
- cookieStore.addCookie(cookie);
- }
- }
- httpClientBuilder.setDefaultCookieStore(cookieStore);
- }
- }
|