SSM整点定时抓取新浪网IT新闻数据

青旅半醒 2022-05-26 10:43 171阅读 0赞

之前做过一些网站的数据抓取功能,不过是手动操作抓取,现在做的是定时任务下的数据爬取.每天7点到23点,后台自动先清空原数据表中的数据,再将新爬取的数据写入数据库

1.数据库建表

CREATE TABLE news (
      id int(10) not null auto_increment primary key,
      title varchar(200) NOT NULL,
      url varchar(150) NOT NULL,
      imageUrl varchar(150) DEFAULT NULL,
      describes varchar(500) DEFAULT NULL,
      readCount int(10) DEFAULT '0',
      createTime timestamp NULL DEFAULT CURRENT_TIMESTAMP
    )

2.配置Task环境

<?xml version="1.0" encoding="UTF-8"?>
    <beans xmlns="http://www.springframework.org/schema/beans"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xmlns:mvc="http://www.springframework.org/schema/mvc"
        xmlns:context="http://www.springframework.org/schema/context"
        xmlns:task="http://www.springframework.org/schema/task"
        xsi:schemaLocation="
            http://www.springframework.org/schema/beans
            http://www.springframework.org/schema/beans/spring-beans-4.2.xsd
            http://www.springframework.org/schema/mvc
            http://www.springframework.org/schema/mvc/spring-mvc-4.2.xsd     
            http://www.springframework.org/schema/context
            http://www.springframework.org/schema/context/spring-context-4.2.xsd
            http://www.springframework.org/schema/task
            http://www.springframework.org/schema/task/spring-task-4.2.xsd">
            
        
        <context:component-scan base-package="com.ytdx.action"/>
        
        <mvc:annotation-driven/>
        
        <mvc:default-servlet-handler/>
     
        
         <bean id="viewResolver"
              class="org.springframework.web.servlet.view.InternalResourceViewResolver"> 
            
            <property name="prefix">
                <value>/jsp/</value>
            </property>
            
            <property name="suffix">
                <value>.jsp</value>
            </property>
        </bean>
    	
        
        <context:annotation-config></context:annotation-config>
        
        <context:component-scan base-package="com.ytdx.task">
        </context:component-scan>
        
        <task:annotation-driven scheduler="qbScheduler" mode="proxy"/>
        <task:scheduler id="qbScheduler" pool-size="10"/>
    </beans>

3.代码

package com.ytdx.dao;
    
    import org.apache.ibatis.annotations.Insert;
    import org.apache.ibatis.annotations.Update;
    
    import com.ytdx.entity.News;
    
    public interface NewsDao {
    	@Insert("insert into news(title,url,imageUrl,describes) values(#{title},#{url},#{imageUrl},#{describes})")
    	void AddNews(News news);
    	
    	@Update("TRUNCATE TABLE news")
    	void truncateNews();
    }

package com.ytdx.service;
    
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.stereotype.Service;
    
    import com.ytdx.dao.NewsDao;
    import com.ytdx.entity.News;
    
    @Service("newsBiz")
    public class NewsBiz {
    	@Autowired
    	private NewsDao newsDao;
    	
    	public void AddNews(News news){
    		newsDao.AddNews(news);
    	}
    	
    	public void truncateNews(){
    		newsDao.truncateNews();
    	}
    }

package com.ytdx.task;
    
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.beans.factory.annotation.Qualifier;
    import org.springframework.scheduling.annotation.Scheduled;
    import org.springframework.stereotype.Component;
    
    import com.ytdx.entity.News;
    import com.ytdx.service.NewsBiz;
    
    @Component("NewsJob")
    public class NewsTask {
    	@Autowired
    	@Qualifier("newsBiz")
    	private NewsBiz newsBiz;
    	
    	@Scheduled(cron = "0 0 7-23 * * ?")       //7点到23点，每整点抓取一次数据
    	public void TimeTask() throws Exception{
    		 AddNews();
    		 System.out.println("系统抓取新闻数据成功!");
    	}
    	
    	public void AddNews() throws Exception{
    		Document doc = Jsoup.connect("http://tech.sina.com.cn/it/").get();
    		Element e = doc.getElementsByClass("seo_data_list").first(); 
    		Elements es = e.children(); 
    		newsBiz.truncateNews();        //先清空之前的新闻表记录
    		for (int i = 0; i < es.size(); i++) {
    			Element nodes = es.get(i);
    			Element content = nodes.getElementsByTag("a").first();
    			String title = content.text(); // 获取链接和题目
    			String url = content.attr("href");
    			String imageUrl = nodes.getElementsByTag("a").get(1).getElementsByTag("img").attr("src");
    			String describe = nodes.getElementsByTag("a").get(2).text();
    			
    			News news = new News();
    			news.setTitle(title);
    			news.setUrl(url);
    			news.setImageUrl(imageUrl);
    			news.setDescribes(describe);
    			newsBiz.AddNews(news);
    		}
    	}
    }

3.结果

![20180430000121965][]

4.再在前端将数据读取,即可实现一个即时的新闻客户端.这里不做了.

[20180430000121965]: /images/20220526/e8c9a4b0d15f45afb4cd146fa784ab46.png