爬取猫眼电影榜单的top100并输出csv文件

水深无声 2022-05-12 09:26 272阅读 0赞

#!/usr/bin/env python 
 # -*- coding:utf-8 -*-
 #encoding = utf-8
 import json
 import pandas as pd
 import requests
 from gevent.pool import Pool
 from requests.exceptions import RequestException
 import re
 import csv
 
 
 def get_one_page(url):
 try:
 hd = {
 'user-agent':'Chrome/10'}
 response = requests.get(url, headers=hd)
 if response.status_code==200:
 return response.text
 return None
 except RequestException:
 return None
 
 def parse_one_page(html):
 pattern = re.compile('<dd>.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name"><a'
 +'.*?>(.*?)</a>.*?star">(.*?).*?releasetime">(.*?)'
 +'.*?integer">(.*?).*?fraction">(.*?).*?</dd>', re.S)
 items = re.findall(pattern, html)
 for item in items:
 yield {
 'id':item[0],
 'image':item[1],
 'title':item[2],
 'actor':item[3].strip()[3:],
 'time':item[4].strip()[5:],
 'score':item[5]+item[6]
 }
 
 def write_to_file(content):
 with open('result.txt', 'a', encoding='utf-8') as f:
 f.write(json.dumps(content, ensure_ascii=False) + '\n')
 
 def main():
 con_list = []
 for i in range(10):
 url = 'http://maoyan.com/board/4?offset=' + str(i*10)
 html = get_one_page(url)
 # parse_one_page(html)
 # print(html)
 for item in parse_one_page(html):
 con_list.append(item)
 # print(item)
 # write_to_file(item) #写入到txt文件中
 print(con_list)#测试是否为[{}{}...]文件
 df = pd.DataFrame(con_list, columns=['id', 'image', 'title', 'actor', 'time', 'score'])
 # print(df)# 输出pandas结果集
 df.to_csv('./myfir.csv', index=False, encoding='utf_8_sig')
 
 if __name__ == '__main__':
 main()
 # for i in range(10):
 # pool =Pool()
 # pool.map(main, [i*10 for i in range(10)])
 # 将数组中的每个元素提取出来当作函数的参数，创建一个个进程，放进进程池中
 # 第一个参数是函数，第二个参数是一个迭代器，将迭代器中的数字作为参数依次传入函数中