使用HttpClient的PoolingHttpClientConnectionManager实现Http请求连接池

ゝ一世哀愁。 2022-06-05 22:28 418阅读 0赞

[http://blog.csdn.net/catoop/article/details/50352334][http_blog.csdn.net_catoop_article_details_50352334]

只有一个httpclient的实例，你可以看看CloseableHttpClient和PoolingHttpClientConnectionManager的源码,你会发现httpclient实例通过execute执行get或post获取连接的时候，会通过实例关联的connectionManager的connect()来建立连接，这里的connectionManager是一个连接池的实现, 因此connect方法调用的是PoolingHttpClientConnectionManager重写的实现，这就是一个连接池建立连接的实现，具体还是看源码吧

两个主机建立连接的过程是很复杂的一个过程，涉及到多个数据包的交换，并且也很耗时间。Http连接需要的三次握手开销很大，这一开销对于比较小的http消息来说更大。但是如果我们直接使用已经建立好的http连接，这样花费就比较小，吞吐率更大。

传统的HttpURLConnection并不支持连接池，如果要实现连接池的机制，还需要自己来管理连接对象。对于网络请求这种底层相对复杂的操作，个人以为如果有可用的其他方案，也没有必要自己去管理连接对象。

除了HttpURLConnection，大家肯定还知道HttpClient。一般情况下，普通使用HttpClient已经能满足我们的需求，不过有时候，在我们需要高并发大量的请求网络的时候，还是用“连接池”这样的概念能提升吞吐量。

我们来看下怎么使用 org.apache.httpcomponents.httpclient（版本4.4）提供的连接池来实现我们的高并发网络请求。

使用到的jar包：   
org\\apache\\httpcomponents\\httpclient\\4.4-beta1\\httpclient-4.4-beta1.jar   
org\\apache\\httpcomponents\\httpclient-cache\\4.4-beta1\\httpclient-cache-4.4-beta1.jar   
org\\apache\\httpcomponents\\httpcore\\4.4-beta1\\httpcore-4.4-beta1.jar

下面代码实例中主要使用到 PoolingHttpClientConnectionManager

package com.hvgroup.zhuhai10086.jms.utils; import java.io.IOException; import java.io.InterruptedIOException; import java.io.UnsupportedEncodingException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import javax.net.ssl.SSLException; import javax.net.ssl.SSLHandshakeException; import org.apache.http.HttpEntity; import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.NameValuePair; import org.apache.http.NoHttpResponseException; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ConnectTimeoutException; import org.apache.http.conn.routing.HttpRoute; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.LayeredConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; /** * HttpClient工具类 * * @return * @author SHANHY * @create 2015年12月18日 */ public class HttpClientUtil { static final int timeOut = 10 * 1000; private static CloseableHttpClient httpClient = null; private final static Object syncLock = new Object(); private static void config(HttpRequestBase httpRequestBase) { // 设置Header等 // httpRequestBase.setHeader("User-Agent", "Mozilla/5.0"); // httpRequestBase // .setHeader("Accept", // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); // httpRequestBase.setHeader("Accept-Language", // "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");// "en-US,en;q=0.5"); // httpRequestBase.setHeader("Accept-Charset", // "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7"); // 配置请求的超时设置 RequestConfig requestConfig = RequestConfig.custom() .setConnectionRequestTimeout(timeOut) .setConnectTimeout(timeOut).setSocketTimeout(timeOut).build(); httpRequestBase.setConfig(requestConfig); } /** * 获取HttpClient对象 * * @return * @author SHANHY * @create 2015年12月18日 */ public static CloseableHttpClient getHttpClient(String url) { String hostname = url.split("/")[2]; int port = 80; if (hostname.contains(":")) { String[] arr = hostname.split(":"); hostname = arr[0]; port = Integer.parseInt(arr[1]); } if (httpClient == null) { synchronized (syncLock) { if (httpClient == null) { httpClient = createHttpClient(200, 40, 100, hostname, port); } } } return httpClient; } /** * 创建HttpClient对象 * * @return * @author SHANHY * @create 2015年12月18日 */ public static CloseableHttpClient createHttpClient(int maxTotal, int maxPerRoute, int maxRoute, String hostname, int port) { ConnectionSocketFactory plainsf = PlainConnectionSocketFactory .getSocketFactory(); LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory .getSocketFactory(); Registry<ConnectionSocketFactory> registry = RegistryBuilder .<ConnectionSocketFactory> create().register("http", plainsf) .register("https", sslsf).build(); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager( registry); // 将最大连接数增加 cm.setMaxTotal(maxTotal); // 将每个路由基础的连接增加 cm.setDefaultMaxPerRoute(maxPerRoute); HttpHost httpHost = new HttpHost(hostname, port); // 将目标主机的最大连接数增加 cm.setMaxPerRoute(new HttpRoute(httpHost), maxRoute); // 请求重试处理 HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() { public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { if (executionCount >= 5) { // 如果已经重试了5次，就放弃 return false; } if (exception instanceof NoHttpResponseException) { // 如果服务器丢掉了连接，那么就重试 return true; } if (exception instanceof SSLHandshakeException) { // 不要重试SSL握手异常 return false; } if (exception instanceof InterruptedIOException) { // 超时 return false; } if (exception instanceof UnknownHostException) { // 目标服务器不可达 return false; } if (exception instanceof ConnectTimeoutException) { // 连接被拒绝 return false; } if (exception instanceof SSLException) { // SSL握手异常 return false; } HttpClientContext clientContext = HttpClientContext .adapt(context); HttpRequest request = clientContext.getRequest(); // 如果请求是幂等的，就再次尝试 if (!(request instanceof HttpEntityEnclosingRequest)) { return true; } return false; } }; CloseableHttpClient httpClient = HttpClients.custom() .setConnectionManager(cm) .setRetryHandler(httpRequestRetryHandler).build(); return httpClient; } private static void setPostParams(HttpPost httpost, Map<String, Object> params) { List<NameValuePair> nvps = new ArrayList<NameValuePair>(); Set<String> keySet = params.keySet(); for (String key : keySet) { nvps.add(new BasicNameValuePair(key, params.get(key).toString())); } try { httpost.setEntity(new UrlEncodedFormEntity(nvps, "UTF-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } } /** * GET请求URL获取内容 * * @param url * @return * @author SHANHY * @throws IOException * @create 2015年12月18日 */ public static String post(String url, Map<String, Object> params) throws IOException { HttpPost httppost = new HttpPost(url); config(httppost); setPostParams(httppost, params); CloseableHttpResponse response = null; try { response = getHttpClient(url).execute(httppost, HttpClientContext.create()); HttpEntity entity = response.getEntity(); String result = EntityUtils.toString(entity, "utf-8"); EntityUtils.consume(entity); return result; } catch (Exception e) { // e.printStackTrace(); throw e; } finally { try { if (response != null) response.close(); } catch (IOException e) { e.printStackTrace(); } } } /** * GET请求URL获取内容 * * @param url * @return * @author SHANHY * @create 2015年12月18日 */ public static String get(String url) { HttpGet httpget = new HttpGet(url); config(httpget); CloseableHttpResponse response = null; try { response = getHttpClient(url).execute(httpget, HttpClientContext.create()); HttpEntity entity = response.getEntity(); String result = EntityUtils.toString(entity, "utf-8"); EntityUtils.consume(entity); return result; } catch (IOException e) { e.printStackTrace(); } finally { try { if (response != null) response.close(); } catch (IOException e) { e.printStackTrace(); } } return null; } public static void main(String[] args) { // URL列表数组 String[] urisToGet = { "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497" }; long start = System.currentTimeMillis(); try { int pagecount = urisToGet.length; ExecutorService executors = Executors.newFixedThreadPool(pagecount); CountDownLatch countDownLatch = new CountDownLatch(pagecount); for (int i = 0; i < pagecount; i++) { HttpGet httpget = new HttpGet(urisToGet[i]); config(httpget); // 启动线程抓取 executors .execute(new GetRunnable(urisToGet[i], countDownLatch)); } countDownLatch.await(); executors.shutdown(); } catch (InterruptedException e) { e.printStackTrace(); } finally { System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成，开始进入下一步！"); } long end = System.currentTimeMillis(); System.out.println("consume -> " + (end - start)); } static class GetRunnable implements Runnable { private CountDownLatch countDownLatch; private String url; public GetRunnable(String url, CountDownLatch countDownLatch) { this.url = url; this.countDownLatch = countDownLatch; } @Override public void run() { try { System.out.println(HttpClientUtil.get(url)); } finally { countDownLatch.countDown(); } } } } 
     
      
       1
       2
       3
       4
       5
       6
       7
       8
       9
       10
       11
       12
       13
       14
       15
       16
       17
       18
       19
       20
       21
       22
       23
       24
       25
       26
       27
       28
       29
       30
       31
       32
       33
       34
       35
       36
       37
       38
       39
       40
       41
       42
       43
       44
       45
       46
       47
       48
       49
       50
       51
       52
       53
       54
       55
       56
       57
       58
       59
       60
       61
       62
       63
       64
       65
       66
       67
       68
       69
       70
       71
       72
       73
       74
       75
       76
       77
       78
       79
       80
       81
       82
       83
       84
       85
       86
       87
       88
       89
       90
       91
       92
       93
       94
       95
       96
       97
       98
       99
       100
       101
       102
       103
       104
       105
       106
       107
       108
       109
       110
       111
       112
       113
       114
       115
       116
       117
       118
       119
       120
       121
       122
       123
       124
       125
       126
       127
       128
       129
       130
       131
       132
       133
       134
       135
       136
       137
       138
       139
       140
       141
       142
       143
       144
       145
       146
       147
       148
       149
       150
       151
       152
       153
       154
       155
       156
       157
       158
       159
       160
       161
       162
       163
       164
       165
       166
       167
       168
       169
       170
       171
       172
       173
       174
       175
       176
       177
       178
       179
       180
       181
       182
       183
       184
       185
       186
       187
       188
       189
       190
       191
       192
       193
       194
       195
       196
       197
       198
       199
       200
       201
       202
       203
       204
       205
       206
       207
       208
       209
       210
       211
       212
       213
       214
       215
       216
       217
       218
       219
       220
       221
       222
       223
       224
       225
       226
       227
       228
       229
       230
       231
       232
       233
       234
       235
       236
       237
       238
       239
       240
       241
       242
       243
       244
       245
       246
       247
       248
       249
       250
       251
       252
       253
       254
       255
       256
       257
       258
       259
       260
       261
       262
       263
       264
       265
       266
       267
       268
       269
       270
       271
       272
       273
       274
       275
       276
       277
       278
       279
       280
       281
       282
       283
       284
       285
       286
       287
       288
       289
       290
       291
       292
       293
       294
       295
       296
       297
       298
       299
       300
       301
       302
       303
       304
       305
       306
       307
       308
       309
       310
       311
       312
       313
       314
       315
       316
       317
       318
       319
       320
       321
       322
       323
       324
       325
       326
       327
       328
       329
       330
       331
       332

Post使用方法

// 其中 params 为 Map<String, Object> params String ret = HttpClientUtil.post(url, params); jsonRet = new JSONObject(ret); 
     
      
       1
       2
       3

一开始我是使用传统的 HttpURLConnection 来做网络请求的，查了很多资料，有不少说 HttpURLConnection 效率高的。可是经过我修改实现方法后，HttpClient 连接池版本的网络请求相对比较稳定。这也说明，我们并不请尽信他人解说，凡事还是要寻找适合自己的方法，真正的解决自己的问题，才是王道。

在使用 HttpURLConnection 的时候，大并发对外做网络请求的时候，前期请求耗时还好，后面耗时越来越高。下面是我之前的实现代码：

@Deprecated protected JSONObject callRestfulOld(String url, Map<String, Object> params) { String temp; String ret=""; JSONObject jsonRet=null; String sign = generateSign("POST", url, params);// 对参数进行加密签名 if(sign.isEmpty()) return new JSONObject("{\"ret_code\":-1,\"err_msg\":\"generateSign error\"}"); params.put("sign", sign); try{ URL u = new URL(url); HttpURLConnection conn = (HttpURLConnection)u.openConnection(); conn.setRequestMethod("POST"); conn.setConnectTimeout(10000); conn.setDoOutput(true); conn.setDoInput(true); conn.setUseCaches(false); StringBuffer param = new StringBuffer(); for (String key: params.keySet()) { param.append(key).append("=").append(URLEncoder.encode(params.get(key).toString(), "UTF-8")).append("&"); } conn.getOutputStream().write(param.toString().getBytes("UTF-8")); //System.out.println(param); conn.getOutputStream().flush(); conn.getOutputStream().close(); InputStreamReader isr = new InputStreamReader(conn.getInputStream()); BufferedReader br = new BufferedReader(isr); while((temp = br.readLine()) != null){ ret += temp; } br.close(); isr.close(); conn.disconnect(); //System.out.println(ret); jsonRet = new JSONObject(ret); } catch(java.net.SocketTimeoutException e) { //e.printStackTrace(); jsonRet = new JSONObject("{\"ret_code\":-1,\"err_msg\":\"call restful timeout\"}"); } catch(Exception e) { //e.printStackTrace(); jsonRet = new JSONObject("{\"ret_code\":-1,\"err_msg\":\"call restful error\"}"); } return jsonRet; } 
     
      
       1
       2
       3
       4
       5
       6
       7
       8
       9
       10
       11
       12
       13
       14
       15
       16
       17
       18
       19
       20
       21
       22
       23
       24
       25
       26
       27
       28
       29
       30
       31
       32
       33
       34
       35
       36
       37
       38
       39
       40
       41
       42
       43
       44
       45
       46
       47

[http_blog.csdn.net_catoop_article_details_50352334]: http://blog.csdn.net/catoop/article/details/50352334