from urllib.request import urlopen, Request
from lxml import etree
#构造函数，抓取第i页信息
# 构造第i页的网址
def crow(i):
url = 'https://movie.douban.com/top250?start=%s&filter='
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
ret = Request(url, headers=headers)
res = urlopen(ret)
html = res.read().decode('utf-8')
print(i)
print("开始")
html = etree.HTML(html)
datas = html.xpath('//ol[@class="grid_view"]/li')
a=0
for data in datas:
data_title=data.xpath('div/div[2]/div[@class="hd"]/a/span[1]/text()')
data_info=data.xpath('div/div[2]/div[@class="bd"]/p[1]/text()')
data_quote=data.xpath('div/div[2]/div[@class="bd"]/p[2]/span/text()')
data_score=data.xpath('div/div[2]/div[@class="bd"]/div/span[@class="rating_num"]/text()')
data_num=data.xpath('div/div[2]/div[@class="bd"]/div/span[4]/text()')
data_picurl=data.xpath('div/div[1]/a/img/@src')
print("No: "+str(i*25+a+1))
print(data_title)
#保存电影信息到txt文件，下载封面图片
with open('douban250.txt','a',encoding='utf-8')as f:
#封面图片保存路径和文件名
picname='F:/top250/'+str(i*25+a+1)+'.jpg'
f.write("No: "+str(i*25+a+1)+'\n')
f.write(data_title[0]+'\n')
f.write(str(data_info[0]).strip()+'\n')
f.write(str(data_info[1]).strip()+'\n')
#因为发现有几部电影没有quote，所以这里加个判断，以免报错
if data_quote:
f.write(data_quote[0]+'\n')
f.write(data_score[0]+'\n')
f.write(data_num[0]+'\n')
f.write('\n'*3)
#下载封面图片到本地，路径为picname
a+=1
for i in range(100):
crow(i)
