httpclient 4.3 连接池卡住问题

Question

static{
try{
SSLContextBuilderbuilder=newSSLContextBuilder();
builder.loadTrustMaterial(null,newTrustSelfSignedStrategy());
SSLConnectionSocketFactorysslsf=newSSLConnectionSocketFactory(builder.build());
CookieSpecProvidereasySpecProvider=newCookieSpecProvider(){
publicCookieSpeccreate(HttpContextcontext){
returnnewBrowserCompatSpec(){
@Override
publicvoidvalidate(Cookiecookie,CookieOriginorigin)throwsMalformedCookieException{
//Oh,Iameasy
}
};
}
};
Registryr=RegistryBuilder.create()
.register(CookieSpecs.BEST_MATCH,newBestMatchSpecFactory())
.register(CookieSpecs.BROWSER_COMPATIBILITY,newBrowserCompatSpecFactory())
.register("easy",easySpecProvider).build();
//5秒超时
RequestConfigrequestConfig=RequestConfig.custom().setConnectionRequestTimeout(5000)
.setSocketTimeout(10000).setConnectTimeout(10000).setCookieSpec("easy").setRedirectsEnabled(false)
.build();
PoolingHttpClientConnectionManagercm=newPoolingHttpClientConnectionManager();
cm.setMaxTotal(100);//连接池最大并发连接数
cm.setDefaultMaxPerRoute(10);//单路由最大并发数
client=HttpClients.custom().setConnectionManager(cm).setDefaultCookieSpecRegistry(r)
.setSSLSocketFactory(sslsf).setDefaultRequestConfig(requestConfig).build();
}catch(Exceptione){
logger.error("httpclient初始化失败!",e);
}
}
publicstaticStringexecute(HttpRequesthttpRequest){
CloseableHttpResponseresponse=null;
HttpGethttpGet=null;
HttpEntityhttpEntity=null;
try{
httpGet=newHttpGet(httpRequest.getUrl());
httpGet.setHeader("Connection","close");//短链接
if(httpRequest.isUseGzip()){
httpGet.addHeader("Accept-Encoding","gzip,deflate,sdch");
}
if(!StringUtils.isEmpty(httpRequest.getContentType())){
httpRequest.setContentType(httpRequest.getContentType());
}
httpGet.addHeader("User-Agent",
"Mozilla/5.0(WindowsNT6.1)AppleWebKit/537.36(KHTML,likeGecko)Chrome/31.0.1650.63");
response=client.execute(httpGet);
httpEntity=response.getEntity();
//不管状态如何，response都需要读取
byte[]bytes=null;
try{
bytes=EntityUtils.toByteArray(httpEntity);
}catch(Exceptione){
returnnull;
}
if(response.getStatusLine().getStatusCode()!=200){
logger.warn("请求异常!StatusCode:"+response.getStatusLine().getStatusCode()+",url:"
+httpRequest.getUrl());
returnnull;
}
//确认网页编码
@SuppressWarnings("deprecation")
Stringcharset=EntityUtils.getContentCharSet(httpEntity);
if(StringUtils.isEmpty(charset)){
Matchermatch=charsetPatterm.matcher(newString(bytes));
if(match.find()){
charset=match.group(1);
}
}
if(!StringUtils.isEmpty(charset)){
StringstrUtf8=newString(newString(bytes,charset).getBytes(),GlobalConfig.ENCODING);
returnStringEscapeUtils.unescapeHtml4(strUtf8);
}
}catch(Exceptione){
logger.error("抓取页面异常!url["+httpRequest.getUrl()+"]",e);
}finally{
try{
if(httpEntity!=null){
EntityUtils.consume(httpEntity);
}
if(response!=null){
response.close();
}
if(httpGet!=null){
httpGet.abort();
}
}catch(Exceptione){
//ignore
}
}
returnnull;
}
这个是httpclient4.3使用的代码，程序跑了一段时间使用jstack查看线程状态发现。
"pool-1-thread-10"prio=10tid=0x00007f7168003000nid=0x3e4dwaitingoncondition[0x00007f717c398000]
java.lang.Thread.State:WAITING(parking)
atsun.misc.Unsafe.park(NativeMethod)
-parkingtowaitfor<0x00000000e69d7350>(ajava.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
atjava.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
atjava.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
atorg.apache.http.pool.PoolEntryFuture.await(PoolEntryFuture.java:133)
atorg.apache.http.pool.AbstractConnPool.getPoolEntryBlocking(AbstractConnPool.java:282)
atorg.apache.http.pool.AbstractConnPool.access$000(AbstractConnPool.java:64)
atorg.apache.http.pool.AbstractConnPool$2.getPoolEntry(AbstractConnPool.java:177)
atorg.apache.http.pool.AbstractConnPool$2.getPoolEntry(AbstractConnPool.java:170)
atorg.apache.http.pool.PoolEntryFuture.get(PoolEntryFuture.java:102)
atorg.apache.http.impl.conn.PoolingHttpClientConnectionManager.leaseConnection(PoolingHttpClientConnectionManager.jav
a:244)
atorg.apache.http.impl.conn.PoolingHttpClientConnectionManager$1.get(PoolingHttpClientConnectionManager.java:231)
atorg.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:173)
atorg.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:195)
atorg.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:86)
atorg.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)
atorg.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
atorg.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
atorg.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
其中一些线程一直卡在这，这是为什么原因呢？

Answer 1

很不幸，我遇到了和你一样的问题。请问你现在解决了吗？我没有找到原因，我用一个map来标记每一个httpclient，发现这个httpclient处理时间过长就主动close掉，虽然比较low但是目前只有这个办法。

Answer 2

是Apache这个httpclient的关闭释放方法有点小问题，导致很多http处在CLOSE_WAIT的状态。然后他的调用方式也有问题，肯定是并发高，但是他设置是最大连接数太小了。把最大连接数设置到100，才比较合理。在服务器上执行以下这个命令：netstat-n|awk'/^tcp/{++S[$NF]}END{for(ainS)printa,S[a]}'看是不是有很多处在CLOSE_WAIT的tcp连接如果是的话那就对了。