自动化爬虫-爬取文本言论正负面识别-存数据库

# coding:utf-8 from selenium import webdriver from sqlalchemy import create_engine import pandas as pd import lxml.html, queue, logging import time, json, datetime import requests import copy # browser = webdriver.Firefox() # headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0"} driver = webdriver.Firefox() tieba = [] # 当前时间点 TIME = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 上周这个时间点 # LAST_WEEK = datetime.datetime.strptime((datetime.datetime.now()-datetime.timedelta(hours=168)).strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S') LAST_WEEK = datetime.datetime.strptime('1990-01-01 00:00', '%Y-%m-%d %H:%M') App_Key = '' Secret_Key = '' logfile = "result.log" logging.basicConfig(level=logging.DEBUG, format='%(asctime)s line:%(lineno)d %(levelname)s : %(message)s', datefmt=' %Y-%m-%d %H:%M:%S', filename='result.log', filemode='a+') # 等待加载页面，滚动页面显示页面数据 def scroll_page(driver): time.sleep(10) driver.execute_script('var q=document.documentElement.scrollTop=0') for i in range(40): js = 'window.scrollBy(0,400)' driver.execute_script(js) time.sleep(0.5) # 调取情感识别接口 def getToken(): # client_id 为官网获取的AK， client_secret 为官网获取的SK host = 'https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=' + App_Key + '&client_secret=' + Secret_Key response = requests.get(host) if response.status_code == 200: # 响应成功 info = json.loads(response.text) # 将字符串转成字典 access_token = info['access_token'] # 解析数据到access_token return access_token return '' AccessToken = getToken() # 情感识别 def getEmotion(inputText, access_token): url = 'https://aip.baidubce.com/rpc/2.0/nlp/v1/sentiment_classify?access_token=' + access_token # url = 'https://aip.baidubce.com/rpc/2.0/nlp/v2/comment_tag?access_token=' + access_token header = {'Content-Type ': 'application/json'} body = {'text': inputText} requests.packages.urllib3.disable_warnings() res = requests.post(url=url, data=json.dumps(body), headers=header, verify=False) if res.status_code == 200: info = json.loads(res.text) print(info) # 打印接口返回信息，如果报错方便查看，也可以忽略报错继续执行 if 'items' in info and len(info['items']) > 0: sentiment = info['items'][0]['sentiment'] if sentiment == 2: type = '正面' return type elif sentiment == 1: type = '中性' return type else: type = '负面' return type # 获取所有评论信息 def get_all_reply(page_info): parser = page_info reply_lt = parser.xpath('//div[starts-with(@class,"l_post j_l_post l_post_bright ")]') if reply_lt: reply_list = reply_lt else: reply_list = parser.xpath('//div[starts-with(@class,"l_post l_post_bright j_l_post clearfix ")]') for element in reply_list: data = {} data['tieba_name'] = tieba_name data['title'] = subject name = element.xpath('.//li[@class="d_name"]/a[starts-with(@class,"p_author_name")]/text()') if len(name) > 1: lt_name = [] for i in range(len(name)): lt_name.append(name[i]) str_name = ''.join(lt_name) else: str_name = name[0] # 回复内容 try: content = '' content1 = element.xpath('.//div[@class="d_post_content j_d_post_content clearfix"]/text()') if len(content1) > 1: for i in range(len(content1)): content += content1[i] else: content = content1[0] except: try: content = element.xpath('.//div[starts-with(class, "d_post_content j_d_post_content")]/text()')[ 0].strip() except: content = element.xpath( './/div[@class="d_post_content j_d_post_content d_post_content_bold clearfix"]/text()')[ 0].strip() # 内容类型 # data['content_type'] = getEmotion(content, AccessToken) try: reply_time = json.loads(element.xpath('@data-field')[0])['content']['date'] except: reply_time = element.xpath('.//div[@class="post-tail-wrap"]/span[4]/text()')[0] data['author_name'] = str_name data['content'] = content data['public_time'] = reply_time # 二级回复 lt_reply2 = element.xpath( './div[@class="d_post_content_main"]//div[@class="j_lzl_c_b_a core_reply_content"]/ul/li') if lt_reply2: data['content_type'] = getEmotion(content, AccessToken) time.sleep(3) reply2(data, lt_reply2) else: pb_time = datetime.datetime.strptime(reply_time, '%Y-%m-%d %H:%M') if pb_time > LAST_WEEK: # 内容类型 data['content_type'] = getEmotion(content, AccessToken) tieba.append(data) else: continue # 二级回复信息 def reply2(data, lt_reply2): for reply_detail2 in lt_reply2: # 二级回复内容 cp_data = copy.deepcopy(data) reply_content = reply_detail2.xpath('.//span[@class="lzl_content_main"]//text()') if len(reply_content) > 1: reply_msg = reply_content[0] for i in range(1, len(reply_content)): reply_msg += reply_content[i] cp_data['reply_content'] = reply_msg # cp_data['reply_type'] = getEmotion(reply_msg, AccessToken) time.sleep(3) else: try: cp_data['reply_content'] = reply_content[0] # cp_data['reply_type'] = getEmotion(reply_content[0], AccessToken) time.sleep(3) except: pass # 二级回复人 reply_name = reply_detail2.xpath('./div/a[@class="at j_user_card "]/text()') if reply_name: cp_data['reply_name'] = reply_name[0] # else: # cp_data['reply_name'] = '' # 二级回复时间 reply2_time = reply_detail2.xpath('.//span[@class="lzl_time"]/text()') if reply2_time: reply_time1 = ' '.join(reply2_time[0].split()) cp_data['reply_time'] = reply_time1 # else: # cp_data['reply_time'] = '' try: r_time = datetime.datetime.strptime(cp_data['reply_time'], '%Y-%m-%d %H:%M') p_time = datetime.datetime.strptime(cp_data['public_time'], '%Y-%m-%d %H:%M') if r_time > LAST_WEEK or p_time > LAST_WEEK: # content = cp_data['content'] reply2_content = cp_data['reply_content'] # cp_data['content_type'] = getEmotion(content, AccessToken) cp_data['reply_type'] = getEmotion(reply2_content, AccessToken) tieba.append(cp_data) else: continue except Exception as e: logging.info(e) class DownloadItem(object): """ 下载类型对象 """ def __init__(self, url_str, type): """ :param url_str: 下载地址 :param type: 0:列表页，1：详情页 """ self.url = url_str self.type = type # 创建一个队列 download_queue = queue.Queue() download_item = DownloadItem( "百度贴吧根据关键词全爬取全贴吧相关信息（此处放置搜索后第一页的结果，下面元素获取是原搜索信息不方便透露）例如：https://tieba.baidu.com/f/search/res?isnew=1&kw=&qw=lol&rn=10&un=&only_thread=0&sm=1&sd=&ed=&pn=1", 0) download_queue.put(download_item) # 如果下载队列不为空就一直循环 while not download_queue.empty(): item = download_queue.get() driver.get(item.url) time.sleep(3) parser = lxml.html.fromstring(driver.page_source) if item.type == 0: # 翻页信息 pages = parser.xpath('.//div[@class="pager pager-search"]/a[@class="last"]/@href') page_num = pages[0].split('=')[-1] print(page_num) # for i in range(76, int(page_num) + 1): for i in range(1, 2): url = 'https://tieba.baidu.com/f/search/res?isnew=1&kw=&qw=lol&rn=10&un=&only_thread=0&sm=1&sd=&ed=&pn={}'.format( 1 + i) result = requests.get(url) # time.sleep(2) parser = lxml.html.fromstring(result.text) detail_url = parser.xpath('//span[@class="p_title"]/a/@href') for url in detail_url: real_url = 'https://tieba.baidu.com/' + url detail_item = DownloadItem(real_url, 1) download_queue.put(detail_item) break # 解析下一页地址 # next_item = DownloadItem('https:' + parser.xpath('//a[@class="next pagination-item "]/@href')[0], 0) # print(next_item) # download_queue.put(next_item) else: # 解析详情页内容 scroll_page(driver) parser = lxml.html.fromstring(driver.page_source) tieba_name = driver.find_element_by_xpath('//div[@class="card_title "]/a').text # 文章主题 try: subject = parser.xpath('//h1/@title')[0] except: # subject = driver.find_element_by_xpath('//h3[@class="core_title_txt pull-left text-overflow ")').text subject = parser.xpath('//h3/@title')[0] # 多评论也问题 get_page = parser.xpath('//li[@class="l_reply_num"]/span[2]/text()')[0] reply_urls = [] if int(get_page) > 1: reply_url = \ parser.xpath('//ul[@class="l_posts_num"]/li[@class="l_pager pager_theme_4 pb_list_pager"]/a[1]/@href')[ 0].split( "=")[0] for i in range(1, int(get_page) + 1): reply_urls.append('https://tieba.baidu.com' + reply_url + '=' + str(i)) if reply_urls: for item in reply_urls: driver.get(item) scroll_page(driver) page_info = lxml.html.fromstring(driver.page_source) get_all_reply(page_info) else: page_info = parser get_all_reply(page_info) conndb = create_engine('mysql+pymysql://root:123456@localhost:3306/hj_bi') for item in tieba: item['public_media'] = '贴吧' item['project_name'] = 'craw' insert_data = pd.DataFrame(tieba) try: pd.io.sql.to_sql(insert_data, 'spider_copy2', conndb, if_exists='append', index=False) except Exception as e: logging.info(e)