webmagic原始实例

曾经终败给现在 2022-07-24 03:16 118阅读 0赞

import java.io.IOException;  
import java.net.MalformedURLException;  
import java.net.URL;  
import java.net.URLConnection;  
import java.text.SimpleDateFormat;  
import java.util.Date;  
import java.util.Iterator;  
import java.util.List;  
  
  
import us.codecraft.webmagic.Page;  
import us.codecraft.webmagic.Request;  
import us.codecraft.webmagic.ResultItems;  
import us.codecraft.webmagic.Site;  
import us.codecraft.webmagic.Spider;  
import us.codecraft.webmagic.processor.PageProcessor;  
import us.codecraft.webmagic.selector.Html;  
public class ganjitest implements PageProcessor\{  
  
  
private Site site=Site.me().setSleepTime(3000).setRetryTimes(2);  
  
  
List<Request> list=null; //取得当前内存中存放的相关的网页链接  
int num=0;  //取得所有的链接  
public void Show(Page page)\{  
  
String addressUrl=null;  
String url=page.getUrl().toString();  
if(url.trim()!="http://www.xinhuanet.com/" || !url.equals("http://www.xinhuanet.com/"))\{  
try\{  
page.putField("address",addressUrl=page.getUrl().regex("(http://news\\\\.xinhuanet\\\\.com/politics/.\*)").toString());  
\}catch(Exception e)\{  
System.out.println("url not contains");  
\}  
\}else\{  
page.putField("address", addressUrl=page.getUrl().toString());  
\}  
page.putField("name", page.getHtml().xpath("//title/text()").toString());  
//提取meta中的content的值  
String str=page.getHtml().xpath("//meta\[@name='keywords'\]").toString();  
int a=str.lastIndexOf("content=\\"");  
int b=str.lastIndexOf("\\"");  
page.putField("meta", str.substring(a+9, b));  
//提取最后刷新时间  
URL hp = null;  
URLConnection hpCon;  
try \{  
   SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");   
hp = new URL(addressUrl);  
hpCon = hp.openConnection();  
//转化为本地时间  
page.putField("lasttimes",sdf.format(new Date(hpCon.getLastModified())));   
\} catch (Exception e) \{  
e.printStackTrace();  
\}   
  
  
if(page.getResultItems().get("name")==null)\{  
page.setSkip(true);  
\}  
\}  
public void process(Page page) \{  
  
//存放到内存中的网页  
page.addTargetRequests(page.getHtml().links().regex("(http://news.xinhuanet.com/politics/.\*)").all());  
list= page.getTargetRequests();  
num+=list.size();  
System.out.println(num);  
if(num>100)\{  
return ;  
\}else\{  
Show(page);   
\}   
  
\}   
public Site getSite() \{  
return site;  
\}  
public static void main(String\[\] args) \{  
//初始化进入的首页  
System.out.println("开始");  
//从哪里开始抓  
Spider.create(new ganjitest()).addUrl("http://www.xinhuanet.com/").thread(5).run();  
System.out.println("结束");  
  
\}  
\}