爬虫基础一

超、凢脫俗 2022-03-28 14:58 204阅读 0赞

# httpClient 第一个爬虫测试 #

/**
     * 第一个爬虫测试
     * Created by DuFei on 2017/7/27.
     * 来源 https://www.datalearner.com/blog/1051501160659926
     */
    public class FirstTest {
        public static void main(String[] args) {
            //建立一个新的请求客户端
            CloseableHttpClient httpClient = HttpClients.createDefault();
            //使用HttpGet方式请求网址
            //请求百度首页会有乱码问题http://www.baidu.com"
    
            HttpGet httpGet = new HttpGet("http://www.datalearner.com/blog");
            //获取网址的返回结果
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpGet);
            } catch (IOException e) {
                e.printStackTrace();
            }
            //获取返回结果中的实体
            HttpEntity entity = response.getEntity();
            //将返回的实体输出
            try {
                System.out.println(EntityUtils.toString(entity));
                EntityUtils.consume(entity);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

注意的是，爬虫一般都是模拟浏览器的行为，大都get请求

# HttpClient详细使用方法 #

[https://www.datalearner.com/blog/1051505270990649][https_www.datalearner.com_blog_1051505270990649]  
在实际中，有很多网页的请求需要附带许多参数设置。主要包括请求的Header设置以及路径参数。在HttpClient 4.3及以上的版本中，这个过程主要包含如下步骤：

使用List<NameValuePair>添加路径参数（请求参数）
    使用URI对请求路径及其参数进行设置
    使用List<Header>设置请求的头部
    初始化自定义的HttpClient客户端，并设置头部
    使用HttpUriRequest设置请求
    使用HttpClient请求上述步骤中的HttpUriRequest对象

我们看一个代码示例

import com.google.common.collect.Lists;
    import org.apache.http.Header;
    import org.apache.http.HttpHeaders;
    import org.apache.http.HttpResponse;
    import org.apache.http.NameValuePair;
    import org.apache.http.client.HttpClient;
    import org.apache.http.client.methods.HttpUriRequest;
    import org.apache.http.client.methods.RequestBuilder;
    import org.apache.http.client.utils.URIBuilder;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.message.BasicHeader;
    import org.apache.http.message.BasicNameValuePair;
    import java.io.IOException;
    import java.net.URI;
    import java.net.URISyntaxException;
    import java.util.List;
    /************
     * HttpClient 使用示例
     * ***********/
    public class HttpClientTest {
      public static void main(String[] args) throws URISyntaxException, IOException {
        String url = "";    //请求路径
        //构造路径参数
        List<NameValuePair> nameValuePairList = Lists.newArrayList();
        nameValuePairList.add(new BasicNameValuePair("username","test"));
        nameValuePairList.add(new BasicNameValuePair("password","password"));
        //构造请求路径，并添加参数
        URI uri = new URIBuilder(url).addParameters(nameValuePairList).build();
        //构造Headers
        List<Header> headerList = Lists.newArrayList();
        headerList.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING,"gzip, deflate"));
        headerList.add(new BasicHeader(HttpHeaders.CONNECTION, "keep-alive"));
        //构造HttpClient
        HttpClient httpClient = HttpClients.custom().setDefaultHeaders(headerList).build();
        //构造HttpGet请求
        HttpUriRequest httpUriRequest = RequestBuilder.get().setUri(uri).build();
        //获取结果
        HttpResponse httpResponse = httpClient.execute(httpUriRequest);
        //获取返回结果中的实体
        HttpEntity entity = httpResponse.getEntity();
        //查看页面内容结果
        String rawHTMLContent = EntityUtils.toString(entity);
        System.out.println(rawHTMLContent);
        //关闭HttpEntity流
        EntityUtils.consume(entity);
      }
    }

[https_www.datalearner.com_blog_1051505270990649]: https://www.datalearner.com/blog/1051505270990649