sina网页新闻小偷原理及源代码（java版） .

落日映苍穹つ 2022-06-11 07:56 219阅读 0赞

在网上看到网页小偷，有偷笑话的有偷天气的。。。最可笑的是GOOGLE上有个人发布了一篇文章，说是js版本的，我打开一看，language=vbscript.......还被很多人引用，一搜网页小偷，都是那一篇文章，真让人无语。更让人无语的就是那个所谓的超级经典的笑话小偷。引用了很多网站上的js，根本看不到js代码。下载下来源代码，更令人吐血，说是用js实现的，结果里面的js都是0KB，就是空文件。

我今天写了个偷新闻的，由于是“偷”所以程序得基于要偷得网站的代码。

简单的说，就是打开网页，看他的源文件。然后使用java的URL类获取网页内容，使用正则表达式过滤想要的内容，我写的是偷新浪新闻的。还有很多地方没有完善，不过基本功能实现了。打开页面，有个按钮，点开始，就会开始下载和过滤。然后将新闻存入数据库中。

以下代码：

callJsp.jsp:

<%@ page language="java" import="java.util.\*" pageEncoding="utf-8"%>

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"  
"[http://www.w3.org/TR/html4/loose.dtd][http_www.w3.org_TR_html4_loose.dtd] ">  
<html>  
<head>  
<meta http-equiv="Content-Type" content="text/html; charset=gb2312">  
<title>callJsp</title>  
</head>

<body>  
<form action="CallServlet">  
<button type="submit">start</button>  
</form>  
</body>  
</html>

CallServlet.java:

package servlet;

import java.io.IOException;

import javax.servlet.ServletException;  
import javax.servlet.http.HttpServlet;  
import javax.servlet.http.HttpServletRequest;  
import javax.servlet.http.HttpServletResponse;

import bo.GetContent;

public class CallServlet extends HttpServlet \{  
private static final long serialVersionUID = 1L;

public void doGet(HttpServletRequest request, HttpServletResponse response)  
throws ServletException, IOException \{  
System.out.println("called");  
GetContent gc = new GetContent();  
gc.getContent("[http://news.sina.com.cn][http_news.sina.com.cn] ");  
gc.savaToDB();  
\}

public void doPost(HttpServletRequest request, HttpServletResponse response)  
throws ServletException, IOException \{

this.doGet(request, response);  
\}  
\}

实体类：

package dto;

public class New \{

private String title;  
private int id;  
public int getId() \{  
return id;  
\}  
public void setId(int id) \{  
this.id = id;  
\}  
private String content;  
public String getTitle() \{  
return title;  
\}  
public void setTitle(String title) \{  
this.title = title;  
\}  
public String getContent() \{  
return content;  
\}  
public void setContent(String content) \{  
this.content = content;  
\}  
\}

逻辑类（关键代码）：

package bo;

import java.io.BufferedReader;  
import java.io.IOException;  
import java.io.InputStream;  
import java.io.InputStreamReader;

import java.net.MalformedURLException;  
import java.net.URL;  
import java.util.ArrayList;  
import java.util.List;  
import java.util.regex.Matcher;  
import java.util.regex.Pattern;

import dao.NewsDAO;  
import dto.New;

public class GetContent \{  
ArrayList<String> newsList;  
Pattern newsPat;  
Pattern mainPat;  
List<New> news;  
NewsDAO newsDAO = new NewsDAO();  
int i = 1;

public GetContent() \{  
String newsReg = "<li(.\*)<a(.\*)([http://news\[/.\]sina\[/.\]com\[/.\]cn(.\*)\[/.\]shtml)+(.\*)>.\*</a></li][http_news_._sina_._com_._cn_._._shtml_._._a_li] >";  
String mainReg = "<p>(.\*)</p>";  
newsList = new ArrayList<String>();  
newsPat = Pattern.compile(newsReg);  
mainPat = Pattern.compile(mainReg);  
news = new ArrayList<New>();  
\}

public void getContent(String URLSTR) \{  
try \{  
URL url = new URL(URLSTR);  
InputStream o = url.openStream();  
BufferedReader br = new BufferedReader(new InputStreamReader(o,  
"gbk"));  
String line = null;  
while ((line = br.readLine()) != null) \{  
Matcher m = newsPat.matcher(line);  
if (m.matches()) \{  
newsList.add(line);  
System.out.println(line);  
getMain(line);  
\}  
\}  
\} catch (IOException e) \{  
e.printStackTrace();  
\}  
\}

public void getMain(String line) throws MalformedURLException \{  
String newsContent = null;

int indexOfHrefBegin = line.indexOf("http");  
int indexOfHrefEnd = line.indexOf("shtml");

int indexOfTitleBegin = line.indexOf(">", indexOfHrefBegin);  
int indexOfTitleEnd = line.indexOf("</a>");  
String title = line.substring(indexOfTitleBegin + 1, indexOfTitleEnd);  
System.out.println(title);

String hrefStr = line.substring(indexOfHrefBegin, indexOfHrefEnd + 5);  
URL u = new URL(hrefStr);  
try \{  
InputStream o = u.openStream();  
BufferedReader br = new BufferedReader(new InputStreamReader(o,  
"gbk"));  
String content = null;  
StringBuffer sf = new StringBuffer();  
while ((content = br.readLine()) != null) \{  
Matcher m = mainPat.matcher(content);  
if (m.matches()) \{  
sf.append(content);  
newsContent = sf.toString();  
\}  
\}  
if (sf != null) \{  
New n = new New();  
n.setId(i);  
i++;  
n.setContent(newsContent);  
n.setTitle(title);  
news.add(n);  
\}  
\} catch (IOException e) \{  
e.printStackTrace();  
\}  
\}  
  
public void savaToDB() \{  
newsDAO.saveNews(news);  
\}  
\}

持久类：

package dao;

import java.sql.Connection;  
import java.sql.PreparedStatement;  
import java.sql.ResultSet;  
import java.sql.SQLException;  
import java.util.List;

import dto.New;

public class NewsDAO \{  
Connection con = DBConnection.getConnection();  
PreparedStatement pst = null;  
ResultSet rs = null;

public void saveNews(List<New> news) \{  
try \{  
String sql = "insert into news values(?,?,?)";  
pst = con.prepareStatement(sql);

for (New n : news) \{  
pst.setInt(1, n.getId());  
pst.setString(2, n.getTitle());  
pst.setString(3, n.getContent());  
pst.executeUpdate();  
\}

\} catch (SQLException e) \{  
e.printStackTrace();  
\} finally \{  
try \{  
pst.close();  
con.close();  
\} catch (SQLException e) \{  
e.printStackTrace();  
\}

\}  
\}  
\}

数据库链接类：

package dao;

import java.sql.Connection;  
import java.sql.DriverManager;

public class DBConnection \{  
private static final String DBDRIVER = "com.mysql.jdbc.Driver";  
private static final String DBURL = "jdbc:mysql://localhost:3306/newsstealing?useUnicode=true&characterEncoding=GBK";  
private static final String DBUSER = "root";//用户名  
private static final String PASSWORD = "pass";//密码  
private static Connection connection = null;  
static\{  
try\{  
Class.forName(DBDRIVER);  
\}catch(ClassNotFoundException e)\{  
e.printStackTrace();  
\}  
\}  
  
public static Connection getConnection()\{  
try\{  
connection = DriverManager.getConnection(DBURL,DBUSER,PASSWORD);  
\}catch(Exception e)\{  
e.printStackTrace();  
\}  
return connection;  
\}  
\}

所有的类都在这里了。根据类名和包名，自己构建一个web project。把mysql的驱动程序放到lib下面。在数据库中创建个数据库，和表（数据库链接类中和dao中可以找到名字）。可以根据自己的条件使用不同的数据库。因为功能还不完善，所有创建表的时候，除了ID，其它的要允许为空值，否则插入的时候有的新闻内容没有获取到，就会出错。

[http_www.w3.org_TR_html4_loose.dtd]: http://www.w3.org/TR/html4/loose.dtd
[http_news.sina.com.cn]: http://news.sina.com.cn/
[http_news_._sina_._com_._cn_._._shtml_._._a_li]: http://news[/.]sina[/.]com[/.]cn%28.*%29[/.]shtml%29+%28.*%29%3E.*%3C/a%3E%3C/li