有道翻译的爬虫
import requests import json #引入json字符串处理模块 #分析请求 获取requests headers里相关参数 #Request URL: http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule 这里将translate_o改成translate 否则请求失败 返回ErrorCode 50错误 url = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule" query = input("请输入要翻译的中文：") #经过测试 data里面必须的四个参数 data = { "i": query, "from": "AUTO", "to": "AUTO", "doctype": "json" } #模拟headers headers = { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36", "referer": "http://fanyi.youdao.com/", } response = requests.post(url, data=data, headers=headers) html_str = response.content.decode() # print(type(html_str)) 返回json字符串类型 dict_str = json.loads(html_str)#将json字符串转为python字典dict print(dict_str) print(dict_str['translateResult'][0][0]['tgt']) #获取翻译结果 print(type(dict_str)) #dict豆瓣电影的爬取以及写入本地文件
import requests import json url = "https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start=0" headers = { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36", "referer": "https://movie.douban.com/explore", } response = requests.get(url, headers=headers) json_str = response.content.decode() dict_str = json.loads(json_str) with open("douban.text", "w") as f: f.write(json.dumps(dict_str, ensure_ascii=False, indent=2)) #json.dumps 将python字典型转化为字符串 #ensure_asclii 编码参数 fasle不以 asclii编码写入文件 #indent参数 换行 空两个字符