基于WebMagic爬虫

╰半橙微兮° 2022-07-15 14:07 260阅读 0赞

## 基于WebMagic爬虫 ##

#### 一、WebMagic简介 ####

> WebMagic是一个简单灵活的爬虫框架。基于WebMagic，你可以快速开发出一个高效、易维护的爬虫。
> 
> 特性：  
>   
> 1. 简单的API，可快速上手  
>   
> 2. 模块化的结构，可轻松扩展  
>   
> 3. 提供多线程和分布式支持  
>

*  项目地址：[http://webmagic.io/][http_webmagic.io]
 *  API中文地址：[http://webmagic.io/docs/zh/][http_webmagic.io_docs_zh]

#### 二、示列代码 ####

*  1) Maven依赖

<dependency>
        <groupId>us.codecraft</groupId>
        <artifactId>webmagic-core</artifactId>
        <version>0.5.3</version>
    </dependency>
    
    <dependency>
        <groupId>us.codecraft</groupId>
        <artifactId>webmagic-extension</artifactId>
        <version>0.5.3</version>
    </dependency>

*  2) Java代码

爬取地址：[http://blog.csdn.net/zhengyong15984285623][http_blog.csdn.net_zhengyong15984285623] 下面所有该用户发表的文章标题和创建时间

package crawl.webmagic.csdn;
    
    import org.apache.commons.collections.CollectionUtils;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    
    import us.codecraft.webmagic.Page;
    import us.codecraft.webmagic.Site;
    import us.codecraft.webmagic.processor.PageProcessor;
    import us.codecraft.webmagic.selector.Html;
    
    import java.util.List;
    
    /** * csdn 爬取控制 */
    public class CsdnProcessor implements PageProcessor { 
    
        public static Logger       logger   = LoggerFactory.getLogger(CsdnProcessor.class);
    
        private Site               site     = Site.me().setSleepTime(0).setCycleRetryTimes(3);
    
        /** * csdn uri 后缀 */
        public static final String CSDN_URI = "zhengyong15984285623";
    
        @Override
        public void process(Page page) {
            List pagenation = page.getHtml().links().regex("/" + CSDN_URI + "/article/list/\\d*").all();
            page.addTargetRequests(pagenation);
            // 文章列表页面只捕捉每篇文章url，并将其加入爬取队列
            if (CollectionUtils.isNotEmpty(pagenation)) {
                List<String> titleList = page.getHtml().xpath("//div[@id='article_list']/div[@class=list_item]").all();
                for (String titleHtml : titleList) {
                    page.addTargetRequests(new Html(titleHtml).links().regex("/" + CSDN_URI
                                                                             + "/article/details/\\d*").all());
                }
                page.setSkip(true);
            } else { // csdn具体文章页面
                String title = page.getHtml().xpath("//div[@class=article_title]/h1/span/a/text()").toString();
                String createTime = page.getHtml().xpath("//div[@class=article_r]/span[@class=link_postdate]/text()").toString();
                page.putField("title", title);
                page.putField("createTime", createTime);
            }
        }
    
        @Override
        public Site getSite() {
            return site;
        }
    
    }

package crawl.webmagic.csdn;
    
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    
    import us.codecraft.webmagic.Spider;
    import us.codecraft.webmagic.pipeline.ConsolePipeline;
    
    /** * 基于webmagic爬取csdn */
    public class CsdnMain { 
    
        public static Logger logger = LoggerFactory.getLogger(CsdnMain.class);
    
        public static void main(String[] args) {
    
            Spider.create(new CsdnProcessor())
                  // 从url开始抓
                  .addUrl("http://blog.csdn.net/" + CsdnProcessor.CSDN_URI)
                  // 设置Scheduler，使用File来管理URL队列
                  // .setScheduler(new FileCacheQueueScheduler("/Users/zhengyong/queue"))
                  // 设置Pipeline，将结果以console方式输出到控制台
                  .addPipeline(new ConsolePipeline())
                  // 开启5个线程同时执行
                  .thread(5)
                  // 启动爬虫
                  .run();
        }
    }

*  3) 运行结果

get page: http://blog.csdn.net/zhengyong15984285623/article/details/52775883
    title:   google guava Joiner 示列 
    createTime: 2016-10-10 11:01
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/52345931
    title:   elastic-job 构建 
    createTime: 2016-08-28 15:20
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/52769537
    title:   Google Guava Cache 示列 
    createTime: 2016-10-09 18:56
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/52276636
    title:   CountDownLatch使用 
    createTime: 2016-08-22 14:36
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/52260396
    title:   ElasticSearch-2.3.5学习笔记 
    createTime: 2016-08-20 11:43
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/52054638
    title:   kafka 0.10.0.0 版本 
    createTime: 2016-07-28 13:56
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51769135
    title:   git配置用户名和邮箱 
    createTime: 2016-06-27 16:21
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51784731
    title:   java身份证格式强校验 
    createTime: 2016-06-29 17:37
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51700245
    title:   jdk动态代理例子 
    createTime: 2016-06-17 15:26
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51684339
    title:   spring ioc 源码解析（二） 
    createTime: 2016-06-15 18:13
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51672539
    title:   图解https 
    createTime: 2016-06-14 16:10
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51682276
    title:   spring ioc 源码解析（一） 
    createTime: 2016-06-15 14:48
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51690969
    title:   ClassPathXmlApplicationContext 源码解析 
    createTime: 2016-06-16 12:15
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51611721
    title:   ThreadPoolExecutor源码分析 
    createTime: 2016-06-08 14:29
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51482737
    title:   JVM底层又是如何实现synchronized 
    createTime: 2016-05-23 16:34
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51437532
    title:   jstat查看内存 
    createTime: 2016-05-17 17:01
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51611216
    title:   ThreadPoolExecutor 分析 
    createTime: 2016-06-08 10:10
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51405003
    title:   java中volatile关键字的含义 
    createTime: 2016-05-14 11:40
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51388614
    title:   mysql 行锁 
    createTime: 2016-05-12 23:39
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51361119
    title:   设计模式 
    createTime: 2016-05-10 10:28
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51396375
    title:   mysql 优化 
    createTime: 2016-05-13 16:22
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51367279
    title:   mysql 事务级别 
    createTime: 2016-05-10 22:19
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51276591
    title:   https客户端证书.p12maven打包后tomcat启动不正确 
    createTime: 2016-04-28 18:16
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51331002
    title:   java NIO详解 
    createTime: 2016-05-06 13:48
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51281760
    title:   jvm学习 
    createTime: 2016-04-29 14:44
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51172649
    title:   redis持久化机制 
    createTime: 2016-04-17 12:20
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51169582
    title:   redis配置说明 
    createTime: 2016-04-16 19:10
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51275616
    title:   dobbo配置 
    createTime: 2016-04-28 16:41
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51169380
    title:   mac 下安装wget 
    createTime: 2016-04-16 18:15
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51253868
    title:   logback.xml文件配置 
    createTime: 2016-04-26 20:42
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/49866373
    title:   linux 启动或停止jar shell脚本 
    createTime: 2015-11-16 15:19
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/51169355
    title:   mac下安装redis 
    createTime: 2016-04-16 18:13
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50984880
    title:   google guava 测试 
    createTime: 2016-03-26 00:27
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/49684219
    title:   spring batch + spring boot 配置 
    createTime: 2015-11-06 16:08
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50984665
    title:   google工具包 
    createTime: 2016-03-25 23:24
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/49692943
    title:   redis命令学习笔记 
    createTime: 2015-11-07 00:49
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/49366849
    title:   mybatis 批量插入oracle与mysql 
    createTime: 2015-10-23 22:10
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/49273057
    title:   linux项目发布常见命令 
    createTime: 2015-10-20 09:56
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50969099
    title:   http方式调用webservice 
    createTime: 2016-03-24 10:01
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50779106
    title:   http post请求 
    createTime: 2016-03-02 13:17
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/48732523
    title:   mac 配置maven环境变量 
    createTime: 2015-09-25 14:03
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/49131093
    title:   本地git上传至远程git仓库 
    createTime: 2015-10-14 21:26
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50779071
    title:   http get请求 
    createTime: 2016-03-02 13:09
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/48732373
    title:   maven 提交oracle jar 包 
    createTime: 2015-09-25 13:47
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/48311255
    title:   mybatis foreach 熟悉 
    createTime: 2015-09-09 09:41
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/46785229
    title:   maven 利用axis2插件配置webservice 
    createTime: 2015-07-07 09:46
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/48729815
    title:   使用 git 进行项目同步开发步骤 
    createTime: 2015-09-25 11:27
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/47721533
    title:   MQ开启密码访问平台服务步骤 
    createTime: 2015-08-17 10:28
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/48729763
    title:   mac 文件显示与隐藏 
    createTime: 2015-09-25 10:34
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/46741017
    title:   jvm 内存回收finalize如何在垃圾清除前工作原理 
    createTime: 2015-07-03 14:07
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/44014901
    title:   maven项目修改jsp代码不用打包就能运行 
    createTime: 2015-03-02 09:39
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/44019831
    title:   观察者模式 
    createTime: 2015-03-02 16:33
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/43193967
    title:   postgre循环插入模拟数据 
    createTime: 2015-01-27 13:52
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/42678061
    title:   事件通知 
    createTime: 2015-01-13 16:17
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/40371545
    title:   maven批处理命令 
    createTime: 2014-10-22 09:49
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/39296751
    title:   java反射 
    createTime: 2014-09-15 19:20
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/39399303
    title:   CAS单点登录配置 
    createTime: 2014-09-19 16:01
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/39338447
    title:   maven学习笔记 
    createTime: 2014-09-17 10:05
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880811
    title:   struts2 表单标签属性 
    createTime: 2014-08-27 23:10
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/39382973
    title:   cas单点登录 
    createTime: 2014-09-19 00:16
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880761
    title:   struts2 数据标签 
    createTime: 2014-08-27 23:09
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880845
    title:   struts2注解 
    createTime: 2014-08-27 23:11
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880697
    title:   struts2 控制标签 
    createTime: 2014-08-27 23:06
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38909139
    title:   集合操作 
    createTime: 2014-08-28 23:52
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50765287
    title:   异步线程池 
    createTime: 2016-02-29 14:26
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50765217
    title:   字符串大写字母转下划线 
    createTime: 2016-02-29 14:17
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880679
    title:   OGNL 中的集合操作 
    createTime: 2014-08-27 23:05
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880635
    title:   struts2中的数据校验文件配置方法 
    createTime: 2014-08-27 23:04
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50315739
    title:   nginx 配置说明 
    createTime: 2015-12-15 15:07
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50420355
    title:   使用BlockingQueue创建生产者消费者模式 
    createTime: 2015-12-28 17:39
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50395936
    title:   spring mvc 线程安全问题说明 
    createTime: 2015-12-24 17:46
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50125637
    title:   [解决异常] spring batch 报错 ORA-08177: 无法连续访问此事务处理 
    createTime: 2015-12-01 10:04
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50191029
    title:   linux操作oracle命令 
    createTime: 2015-12-06 00:14
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/50040183
    title:   memcache查看数据命令 
    createTime: 2015-11-25 17:46
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/49909489
    title:   Python学习指南 
    createTime: 2015-11-18 16:49
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880583
    title:   struts2结果类型 
    createTime: 2014-08-27 23:02
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880611
    title:   struts2拦截器 
    createTime: 2014-08-27 23:03
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880415
    title:   struts2输出国际信息 
    createTime: 2014-08-27 22:57
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880555
    title:   Action访问Servlet API 
    createTime: 2014-08-27 23:01
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880505
    title:   struts2拦截器配置 
    createTime: 2014-08-27 23:01
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880371
    title:   struts2标签 
    createTime: 2014-08-27 22:57
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/37968297
    title:   日志 
    createTime: 2014-07-19 20:17
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/12749205
    title:   快速排序算法 
    createTime: 2013-10-15 14:12
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38010949
    title:   培训总结 
    createTime: 2014-07-21 09:48
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/38880425
    title:   struts2常量配置 
    createTime: 2014-08-27 22:58
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9303017
    title:   12-jbpm工作流之 分配任务给一个"组的成员"GroupTask 
    createTime: 2013-07-11 17:52
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9301875
    title:   11-jbpm工作流之"事件处理Event" 
    createTime: 2013-07-11 16:24
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9300305
    title:   09-jbpm工作流之"分支聚合Join-Fork" 
    createTime: 2013-07-11 14:21
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9300977
    title:   10-jbpm工作流之"自定义活动Custom" 
    createTime: 2013-07-11 15:12
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728199
    title:   05 CheckBox 
    createTime: 2013-03-27 20:08
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9296173
    title:   08-jbpm工作流之Decision流程决策(判断活动执行方向) 
    createTime: 2013-07-11 00:52
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728194
    title:   04 RadioGroup 
    createTime: 2013-03-27 20:08
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728180
    title:   03 EditText 
    createTime: 2013-03-27 20:05
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728176
    title:   01 TextView 
    createTime: 2013-03-27 20:04
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728066
    title:   06 绝对定位布局管理器 
    createTime: 2013-03-27 19:33
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728062
    title:   05 布局管理器的嵌套 
    createTime: 2013-03-27 19:33
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728178
    title:   02 Button 
    createTime: 2013-03-27 20:04
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728061
    title:   04 相对布局管理器 
    createTime: 2013-03-27 19:32
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728060
    title:   03 表格布局管理器 
    createTime: 2013-03-27 19:31
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728043
    title:   Activity 初步 
    createTime: 2013-03-27 19:27
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728048
    title:   01 线性布局管理器 
    createTime: 2013-03-27 19:28
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728049
    title:   02 框架布局管理器 
    createTime: 2013-03-27 19:29
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728030
    title:   Android项目文件夹解析 
    createTime: 2013-03-27 19:22
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728010
    title:   AndroidManifest.xml解析 
    createTime: 2013-03-27 19:16
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728021
    title:   main.xml解析 
    createTime: 2013-03-27 19:19
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9294173
    title:   07-jbpm工作流之根据流程变量分配任务Task 
    createTime: 2013-07-10 19:52
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9291577
    title:   06-jbpm工作流之状态活动State 
    createTime: 2013-07-10 15:43
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9290735
    title:   05-jbpm工作流的流转Transition 
    createTime: 2013-07-10 14:37
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9284027
    title:   03-jbpm工作流管理方法扩展 
    createTime: 2013-07-09 17:46
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9283165
    title:   02-jbpm工作流管理 
    createTime: 2013-07-09 16:33
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9289231
    title:   04-jbpm工作流执行变量Variable 
    createTime: 2013-07-10 11:36
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9282047
    title:   01-jbpm工作流实现 
    createTime: 2013-07-09 15:10
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9090409
    title:   向字符串对象中追加replaceAll方法 
    createTime: 2013-06-13 22:21
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9067319
    title:   获取登录用户IP地址 
    createTime: 2013-06-09 19:35
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9142311
    title:   根据tree文件菜单的path,拼接文件夹路径 
    createTime: 2013-06-21 09:15
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9087247
    title:   EasyUI的from表单，根据皮肤变换 改变表单颜色 
    createTime: 2013-06-13 15:47
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/9028551
    title:   初试.net使用ajax调用后台方法 
    createTime: 2013-06-05 10:34
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8997073
    title:   myelipse 10版本破解 
    createTime: 2013-05-30 22:49
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8990035
    title:   .net 如何连接mysql 
    createTime: 2013-05-29 16:40
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8977729
    title:   集合按某个属性排序 
    createTime: 2013-05-26 21:57
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8944242
    title:   EasyUI+Struts2整合KindEditor 
    createTime: 2013-05-18 16:15
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8870841
    title:   kaptcha验证码使用配置 
    createTime: 2013-04-30 22:30
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8867549
    title:   定时器-----每天定时删除临时文件 
    createTime: 2013-04-29 18:14
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8816594
    title:   html5绘图铺满整个屏幕 
    createTime: 2013-04-17 22:52
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8835784
    title:   canvas像素化video 
    createTime: 2013-04-22 18:09
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8798233
    title:   .net 当GridView编辑状态获取新值时，往往获取的是修改前的值。 
    createTime: 2013-04-13 21:44
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8789932
    title:   HTML5 之 notifications 
    createTime: 2013-04-11 19:38
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8789756
    title:   Html5 之 Google地图 
    createTime: 2013-04-11 18:41
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8784637
    title:   CSS3 动画之animation 
    createTime: 2013-04-10 19:00
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8784565
    title:   CSS3 过渡之transition 
    createTime: 2013-04-10 18:39
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8784524
    title:   CSS3 3D 转换 
    createTime: 2013-04-10 18:29
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8784401
    title:   CSS3 2D 转换 
    createTime: 2013-04-10 18:02
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8782853
    title:   CSS3 border-radius 属性 
    createTime: 2013-04-10 14:39
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8779509
    title:   html5视频播放 
    createTime: 2013-04-09 20:25
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8783425
    title:   CSS3 border-image 属性 
    createTime: 2013-04-10 15:56
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8783624
    title:   CSS3 box-shadow 属性 
    createTime: 2013-04-10 16:21
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8780133
    title:   html5之绘图板 
    createTime: 2013-04-09 21:45
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8778863
    title:   html5中canvas基本应用 
    createTime: 2013-04-09 17:40
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8779037
    title:   html5之canvas动画 
    createTime: 2013-04-09 18:18
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8764192
    title:   easyui扁平Json生成树形菜单 
    createTime: 2013-04-06 15:40
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8730910
    title:   14 AutoCompleteTextView 
    createTime: 2013-03-28 13:05
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8760347
    title:   easyui动态创建一个dialog 
    createTime: 2013-04-05 02:07
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8730904
    title:   13 Dialog 
    createTime: 2013-03-28 13:04
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8730896
    title:   11 ScrollView 
    createTime: 2013-03-28 13:03
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8730900
    title:   12 ListView 
    createTime: 2013-03-28 13:04
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8730891
    title:   10 DatePicker 
    createTime: 2013-03-28 13:02
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728227
    title:   07 ImageView 
    createTime: 2013-03-27 20:16
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728233
    title:   09 TimePicker 
    createTime: 2013-03-27 20:17
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728231
    title:   08 ImageButton 
    createTime: 2013-03-27 20:16
    get page: http://blog.csdn.net/zhengyong15984285623/article/details/8728201
    title:   06 Spinner 
    createTime: 2013-03-27 20:09

#### 三、 注意事项 ####

WebMagic使用log4j打印日志

<?xml version="1.0" encoding="UTF-8" ?>
    <!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
    
    <log4j:configuration>
    
        <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
            <layout class="org.apache.log4j.PatternLayout">
                <param name="ConversionPattern" value="%d - %c -%-4r [%t] %-5p %x - %m%n" />
            </layout>
    
            
            <filter class="org.apache.log4j.varia.LevelRangeFilter">
                <param name="LevelMax" value="ERROR"/>
                <param name="LevelMin" value="TRACE"/>
            </filter>
        </appender>
    
        <appender name="FILE" class="org.apache.log4j.FileAppender">
            <param name="File" value="/Users/zhengyong/log/crawl.log"/>
            <layout class="org.apache.log4j.PatternLayout">
                <param name="ConversionPattern" value="%d - %c -%-4r [%t] %-5p %x - %m%n" />
            </layout>
        </appender>
    
        <root>
            <priority value="info" />
            <appender-ref ref="CONSOLE" />
            <appender-ref ref="FILE" />
        </root>
    
    
    </log4j:configuration>

Java中main方法使用

import org.apache.log4j.xml.DOMConfigurator;
    
    DOMConfigurator.configure("/Users/zhengyong/crawl/log4j.xml");//加载.xml文件

[http_webmagic.io]: http://webmagic.io/
[http_webmagic.io_docs_zh]: http://webmagic.io/docs/zh/
[http_blog.csdn.net_zhengyong15984285623]: http://blog.csdn.net/zhengyong15984285623