参考文档后总结：
//www.greatytc.com/p/bd7bea8eb487
https://blog.csdn.net/chenpeng19910926/article/details/71482347
pom文件
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.6</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
简单get和post请求:
public static void main(String[] args) throws Exception{
try {
//此为百度一篇文章
String url = "https://mbd.baidu.com/newspage/data/landingsuper?context=%7B%22nid%22%3A%22news_15984470154821203735%22%7D&n_type=0&p_from=1";
// 使用默认配置创建httpclient的实例
CloseableHttpClient client = HttpClients.createDefault();
// HttpPost post = new HttpPost(url);
HttpGet get = new HttpGet(url);
// CloseableHttpResponse response = client.execute(post);
CloseableHttpResponse response = client.execute(get);
// 服务器返回码
System.out.println(response.getStatusLine().toString());
System.out.println(response.getStatusLine().getStatusCode());
System.out.println(response.getStatusLine().getReasonPhrase());
// 服务器返回内容
String respStr = null;
HttpEntity entity = response.getEntity();
if(entity != null) {
respStr = EntityUtils.toString(entity, "UTF-8");
}
System.out.println("respStr = " + respStr);
// 释放资源
EntityUtils.consume(entity);
} catch (Exception e) {
e.printStackTrace();
}
}
流式(Stream)：内容从流(Stream)中接收，或者在运行中产生(generated on the fly)。
自我包含(self-contained)：内容在内存中或通过独立的连接或其他实体中获得。
包装(wrapping)：内容是从其他实体(Entity)中获得的。
模拟登陆
public static void main(String[] args) {
try {
String url = "http://localhost:9090";
// 使用默认配置创建httpclient的实例
CloseableHttpClient client = HttpClients.createDefault();
HttpPost post = new HttpPost(url);
/**
* 设置参数，常用的有StringEntity,UrlEncodedFormEntity,MultipartEntity
* 具体看org.apache.http.entity包
*/
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("username", "张三"));
params.add(new BasicNameValuePair("password", "123456"));
UrlEncodedFormEntity e = new UrlEncodedFormEntity(params, "UTF-8");
post.setEntity(e);
CloseableHttpResponse response = client.execute(post);
// 服务器返回码
int status_code = response.getStatusLine().getStatusCode();
System.out.println(status_code);
// 服务器返回内容
String respStr = null;
HttpEntity entity = response.getEntity();
if(entity != null) {
respStr = EntityUtils.toString(entity, "UTF-8");
}
System.out.println("respStr = " + respStr);
// 释放资源
EntityUtils.consume(entity);
} catch (Exception e) {
e.printStackTrace();
}
}
官方实例：
import java.net.URI;
import java.util.List;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
/**
* A example that demonstrates how HttpClient APIs can be used to perform
* form-based logon.
*/
public class ClientFormLogin {
public static void main(String[] args) throws Exception {
BasicCookieStore cookieStore = new BasicCookieStore();
CloseableHttpClient httpclient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
try {
HttpGet httpget = new HttpGet("https://someportal/");
CloseableHttpResponse response1 = httpclient.execute(httpget);
try {
HttpEntity entity = response1.getEntity();
System.out.println("Login form get: " + response1.getStatusLine());
EntityUtils.consume(entity);
System.out.println("Initial set of cookies:");
List<Cookie> cookies = cookieStore.getCookies();
if (cookies.isEmpty()) {
System.out.println("None");
} else {
for (int i = 0; i < cookies.size(); i++) {
System.out.println("- " + cookies.get(i).toString());
}
}
} finally {
response1.close();
}
HttpUriRequest login = RequestBuilder.post().setUri(new URI("https://someportal/"))
.addParameter("IDToken1", "username").addParameter("IDToken2", "password").build();
CloseableHttpResponse response2 = httpclient.execute(login);
try {
HttpEntity entity = response2.getEntity();
System.out.println("Login form get: " + response2.getStatusLine());
EntityUtils.consume(entity);
System.out.println("Post logon cookies:");
List<Cookie> cookies = cookieStore.getCookies();
if (cookies.isEmpty()) {
System.out.println("None");
} else {
for (int i = 0; i < cookies.size(); i++) {
System.out.println("- " + cookies.get(i).toString());
}
}
} finally {
response2.close();
}
} finally {
httpclient.close();
}
}
}
public static void main(String[] args) {
try {
String url = "http://www.baidu.com";
/**
* 请求参数配置
* connectionRequestTimeout:
* 从连接池中获取连接的超时时间，超过该时间未拿到可用连接，
* 会抛出org.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection from pool
* connectTimeout:
* 连接上服务器(握手成功)的时间，超出该时间抛出connect timeout
* socketTimeout:
* 服务器返回数据(response)的时间，超过该时间抛出read timeout
*/
CloseableHttpClient client = HttpClients.custom().setDefaultRequestConfig(RequestConfig.custom()
.setConnectionRequestTimeout(2000).setConnectTimeout(2000).setSocketTimeout(2000).build()).build();
HttpPost post = new HttpPost(url);
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
设置header
HttpGet get = new HttpGet(url);
get.setHeader("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36");
get.setHeader("Connection", "keep-alive");
get.setHeader("charset","utf-8");
待验证
### 1.1.3\. HTTP报文 headers
一个HTTP报文可能包含很多个header来表述报文的属性例如： content length, content type等等. HttpClient提供了对http报文head进行查询、添加、移除、枚举的方法。
**[java]** [view plain](https://blog.csdn.net/chenpeng19910926/article/details/71482347# "view plain")[copy](https://blog.csdn.net/chenpeng19910926/article/details/71482347# "copy")
<embed id="ZeroClipboardMovie_5" src="https://csdnimg.cn/public/highlighter/ZeroClipboard.swf" loop="false" menu="false" quality="best" bgcolor="#ffffff" width="16" height="16" name="ZeroClipboardMovie_5" align="middle" allowscriptaccess="always" allowfullscreen="false" type="application/x-shockwave-flash" pluginspage="http://www.macromedia.com/go/getflashplayer" flashvars="id=5&width=16&height=16" wmode="transparent" style="box-sizing: border-box;">
1. //在服务器中
2. HttpResponse response=new BasicHttpResponse(org.apache.http.HttpVersion.HTTP_1_1,org.apache.http.HttpStatus.SC_OK, "OK");
3. response.addHeader("Set-Cookie","c1=a; path=/; domain=localhost");
4. response.addHeader("Set-Cookie", "c2=b; path=\"/\", c3=c; domain=\"localhost\"");
5. //同样可以进行拆箱的工作，获取Header
6. Header h1 = response.getFirstHeader("Set-Cookie");
7. System.out.println(h1);
8. Header h2 = response.getLastHeader("Set-Cookie");
9. System.out.println(h2);
10. Header[] hs = response.getHeaders("Set-Cookie");
11. System.out.println(hs.length);
最高效的获取所有header的方法是使用 `HeaderIterator接口`.
**[java]** [view plain](https://blog.csdn.net/chenpeng19910926/article/details/71482347# "view plain")[copy](https://blog.csdn.net/chenpeng19910926/article/details/71482347# "copy")
<embed id="ZeroClipboardMovie_6" src="https://csdnimg.cn/public/highlighter/ZeroClipboard.swf" loop="false" menu="false" quality="best" bgcolor="#ffffff" width="16" height="16" name="ZeroClipboardMovie_6" align="middle" allowscriptaccess="always" allowfullscreen="false" type="application/x-shockwave-flash" pluginspage="http://www.macromedia.com/go/getflashplayer" flashvars="id=6&width=16&height=16" wmode="transparent" style="box-sizing: border-box;">
1. //相当于只是取到了header
2. HeaderIterator it = response.headerIterator("Set-Cookie");
3. while (it.hasNext()) {
4. System.out.println(it.next());
5. }
当然也提供了非常方便的解析header elements部分报文的方法.
**[java]** [view plain](https://blog.csdn.net/chenpeng19910926/article/details/71482347# "view plain")[copy](https://blog.csdn.net/chenpeng19910926/article/details/71482347# "copy")
<embed id="ZeroClipboardMovie_7" src="https://csdnimg.cn/public/highlighter/ZeroClipboard.swf" loop="false" menu="false" quality="best" bgcolor="#ffffff" width="16" height="16" name="ZeroClipboardMovie_7" align="middle" allowscriptaccess="always" allowfullscreen="false" type="application/x-shockwave-flash" pluginspage="http://www.macromedia.com/go/getflashplayer" flashvars="id=7&width=16&height=16" wmode="transparent" style="box-sizing: border-box;">
1. HeaderElementIterator hei=new BasicHeaderElementIterator(response.headerIterator("Set-Cookie"));
2. while (it.hasNext()) {
3. HeaderElement elem = hei.nextElement();
4. System.out.println(elem.getName() + " = " + elem.getValue());
5. //当然可以以键值对的形式读取
6. NameValuePair[] params = elem.getParameters();
7. for (int i = 0; i < params.length; i++) {
8. System.out.println(" " + params[i]);
9. }
10. }
StringEntity entity=new StringEntity("important message",ContentType.create("text/plain","utf-8"));
//Entity中可以包含的信息Content-Type
entity.getContentType();
//Content-Length
entity.getContentLength();
//支持的编码：Content-Encoding;Content-Type: text/plain; charset=utf-8
entity.getContentEncoding();
//显示entity的content ：important message
EntityUtils.toString(entity);
//将entity转化为Byte字节数组
EntityUtils.toByteArray(entity);
//将其想象成建一个浏览器的过程，HttpClients我个人感觉可以类比Collection和Collections的关系，提供HTTPClient的工具
CloseableHttpClient httpclient=HttpClients.createDefault();
//可以想象为用什么方法去访问服务，就像表单提交时候选择Get还是Post
HttpGet httpget=new HttpGet("www.baidu.com");
//可以想象为点击鼠标的过程，或者是提交表单的过程。有返回值。。。。。
CloseableHttpResponse response=httpclient.execute(httpget);
try {
//业务处理层的东西
HttpEntity entity=response.getEntity();
if(entity!=null) {
InputStream is=entity.getContent();
try {
//dosomething
}finally {
//关闭entity的输入流
is.close();
}
}
}finally{
//关闭响应的流
response.close();
}
}
public class PoolTest {
private static void config(HttpRequestBase httpRequestBase) {
httpRequestBase.setHeader("User-Agent", "Mozilla/5.0");
httpRequestBase.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
httpRequestBase.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");//"en-US,en;q=0.5");
httpRequestBase.setHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7");
// 配置请求的超时设置
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(3000)
.setConnectTimeout(3000)
.setSocketTimeout(3000)
.build();
httpRequestBase.setConfig(requestConfig);
}
public static void main(String[] args) {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", sslsf)
.build();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
// 将最大连接数增加到200
cm.setMaxTotal(200);
// 将每个路由基础的连接增加到20
cm.setDefaultMaxPerRoute(20);
// 将目标主机的最大连接数增加到50
HttpHost localhost = new HttpHost("http://blog.csdn.net/gaolu",80);
cm.setMaxPerRoute(new HttpRoute(localhost), 50);
//请求重试处理
HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception,int executionCount, HttpContext context) {
if (executionCount >= 5) {// 如果已经重试了5次，就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的，就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
};
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(cm)
.setRetryHandler(httpRequestRetryHandler)
.build();
// URL列表数组
String[] urisToGet = {
"http://blog.csdn.net/gaolu/article/details/48466059",
"http://blog.csdn.net/gaolu/article/details/48243103",
"http://blog.csdn.net/gaolu/article/details/47656987",
"http://blog.csdn.net/gaolu/article/details/47055029",
"http://blog.csdn.net/gaolu/article/details/46400883",
"http://blog.csdn.net/gaolu/article/details/46359127",
"http://blog.csdn.net/gaolu/article/details/46224821",
"http://blog.csdn.net/gaolu/article/details/45305769",
"http://blog.csdn.net/gaolu/article/details/43701763",
"http://blog.csdn.net/gaolu/article/details/43195449",
"http://blog.csdn.net/gaolu/article/details/42915521",
"http://blog.csdn.net/gaolu/article/details/41802319",
"http://blog.csdn.net/gaolu/article/details/41045233",
"http://blog.csdn.net/gaolu/article/details/40395425",
"http://blog.csdn.net/gaolu/article/details/40047065",
"http://blog.csdn.net/gaolu/article/details/39891877",
"http://blog.csdn.net/gaolu/article/details/39499073",
"http://blog.csdn.net/gaolu/article/details/39314327",
"http://blog.csdn.net/gaolu/article/details/38820809",
"http://blog.csdn.net/gaolu/article/details/38439375",
};
long start = System.currentTimeMillis();
try {
int pagecount = urisToGet.length;
ExecutorService executors = Executors.newFixedThreadPool(pagecount);
CountDownLatch countDownLatch = new CountDownLatch(pagecount);
for(int i = 0; i< pagecount;i++){
HttpGet httpget = new HttpGet(urisToGet[i]);
config(httpget);
//启动线程抓取
executors.execute(new GetRunnable(httpClient,httpget,countDownLatch));
}
countDownLatch.await();
executors.shutdown();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成，开始进入下一步！");
}
long end = System.currentTimeMillis();
System.out.println("consume -> " + (end - start));
}
static class GetRunnable implements Runnable {
private CountDownLatch countDownLatch;
private final CloseableHttpClient httpClient;
private final HttpGet httpget;
public GetRunnable(CloseableHttpClient httpClient, HttpGet httpget, CountDownLatch countDownLatch){
this.httpClient = httpClient;
this.httpget = httpget;
this.countDownLatch = countDownLatch;
}
@Override
public void run() {
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpget,HttpClientContext.create());
HttpEntity entity = response.getEntity();
System.out.println(EntityUtils.toString(entity, "utf-8")) ;
EntityUtils.consume(entity);
} catch (IOException e) {
e.printStackTrace();
} finally {
countDownLatch.countDown();
try {
if(response != null)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
HttpUtils工具类：
package com.springboot.utils;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public class HttpUtils {
public static String doGet(String url, Map<String, String> param) {
// 创建Httpclient对象
CloseableHttpClient httpclient = HttpClients.createDefault();
String resultString = "";
CloseableHttpResponse response = null;
try {
// 创建uri
URIBuilder builder = new URIBuilder(url);
if (param != null) {
for (String key : param.keySet()) {
builder.addParameter(key, param.get(key));
}
}
URI uri = builder.build();
// 创建http GET请求
HttpGet httpGet = new HttpGet(uri);
// 执行请求
response = httpclient.execute(httpGet);
// 判断返回状态是否为200
if (response.getStatusLine().getStatusCode() == 200) {
resultString = EntityUtils.toString(response.getEntity(), "UTF-8");
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (response != null) {
response.close();
}
httpclient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return resultString;
}
public static String doGet(String url) {
return doGet(url, null);
}
public static String doPost(String url, Map<String, String> param) {
// 创建Httpclient对象
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse response = null;
String resultString = "";
try {
// 创建Http Post请求
HttpPost httpPost = new HttpPost(url);
// 创建参数列表
if (param != null) {
List<NameValuePair> paramList = new ArrayList<>();
for (String key : param.keySet()) {
paramList.add(new BasicNameValuePair(key, param.get(key)));
}
// 模拟表单
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(paramList, "utf-8");
httpPost.setEntity(entity);
}
// 执行http请求
response = httpClient.execute(httpPost);
if (response.getStatusLine().getStatusCode() == 200) {
resultString = EntityUtils.toString(response.getEntity(), "utf-8");
return resultString;
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return resultString;
}
public static String doPost(String url) {
return doPost(url, null);
}
/**
*
*/
public static String doPostJson(String url, String json) {
// 创建Httpclient对象
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse response = null;
String resultString = "";
try {
// 创建Http Post请求
HttpPost httpPost = new HttpPost(url);
// 创建请求内容
ContentType contentType;
StringEntity entity = new StringEntity(json, ContentType.APPLICATION_JSON);
httpPost.setEntity(entity);
// 执行http请求
response = httpClient.execute(httpPost);
resultString = EntityUtils.toString(response.getEntity(), "utf-8");
return resultString;
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return resultString;
}
}
response.getFirstHeader("key");// 得到第一个名字为key的header
response.getHeaders("key");// 得到名字为key的所有header，返回一个数组
response.getLastHeader("key");
InputStream inputStream = response.getEntity().getContent();
// 发送请求，返回响应
HttpResponse response = httpClient.execute(request);
HttpEntity entity=response.getEntity();
// 取出服务器返回的数据流
InputStream stream = entity.getContent();
// ByteArrayOutputStream baos = new ByteArrayOutputStream();
// int i=-1;
// while((i=stream.read())!=-1){
// baos.write(i);
// }System.out.println(baos);
对于参数的URLEncoding处理，HttpClient程序包为我们准备了另一个工具类：URLEncodedUtils。通过它，我们可以直观的（但是比较复杂）生成URI，如
List params = new ArrayList();
params.add(new BasicNameValuePair("param1", "中国"));
params.add(new BasicNameValuePair("param2", "value2"));
String param = URLEncodedUtils.format(params, "UTF-8");
URI uri = URIUtils.createURI("http", "localhost", 8080,
"/sshsky/index.html", param, null);
System.out.println(uri);
上例的打印结果如下：
http://localhost/index.html?param1=%E4%B8%AD%E5%9B%BD¶m2=value2
