爬虫基础 之深度优先，广度优先策略

1.深度优先递归方式;

import re import requests headers = { 'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36 Edg/85.0.564.41" } def get_html(url): try: res= requests.get(url,headers=headers) return res.text except: return "" def get_son_url(url): # 获取 html = get_html(url) html_re = '<a.*?href="(.*?)".*?>' href_list = re.findall(html_re,html,re.S) return href_list def deep_path(url): if deepdict[url] > 3: return print("\t"*deepdict[url],"当前层级:%d" % deepdict[url]) # 获取子url列表 sonurl_list = get_son_url(url) #返回的是一个列表 #遍历所有的子url for sonurl in sonurl_list: if sonurl.startswith('https') or sonurl.startswith('http'): if sonurl not in deepdict: deepdict[sonurl] = deepdict[url]+1 deep_path(sonurl) if __name__ == '__main__': url = "https://www.baidu.com/s?wd=%E6%AD%A6%E6%B1%89%E5%85%89%E8%B0%B7" # 控制层级 deepdict = {} deepdict[url] = 1 deep_path(url)

2.广度优先策略之队列方法：

import re import requests headers = { 'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36 Edg/85.0.564.41" } #获取网页源代码 def get_html(url): try: res= requests.get(url,headers=headers) return res.text except: return "" #获取子url列表 def get_son_url(url): # 获取 html = get_html(url) html_re = '<a.*?href="(.*?)".*?>' href_list = re.findall(html_re,html,re.S) return href_list #广度爬取 def vast_path(url): #队列方法 先进先出 #append 入队列 pop 出队列 用列表 模拟队列 url_queue = [] url_queue.append(url) #默认先把第一个放进来 while len(url_queue)>0: #出队列 每次取出一个 url = url_queue.pop(0) print("\t" * deepdict[url],'当前层级:%d'%deepdict[url]) if deepdict[url]<3: #获取子url列表 sonurl_list = get_son_url(url) for sonurl in sonurl_list: #过滤出有效链接 if sonurl.startswith('https') or sonurl.startswith('http'): if sonurl not in deepdict: #过滤重复url deepdict[sonurl] = deepdict[url]+1 #入队列 url_queue.append(sonurl) if __name__ == '__main__': url = "https://www.baidu.com/s?wd=%E6%AD%A6%E6%B1%89%E5%85%89%E8%B0%B7" # 控制层级 deepdict = {} #控制层级 deepdict[url] = 1 # 默认第一级 vast_path(url)

3.深度优先策略之栈方法：

import re import requests headers = { 'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36 Edg/85.0.564.41" } #获取网页源代码 def get_html(url): try: res= requests.get(url,headers=headers) return res.text except: return "" #获取子url列表 def get_son_url(url): # 获取 html = get_html(url) html_re = '<a.*?href="(.*?)".*?>' href_list = re.findall(html_re,html,re.S) return href_list #广度爬取 def vast_path(url): #队列方法 先进先出 #append 入栈 pop 出栈 用列表 模拟栈 url_queue = [] url_queue.append(url) #默认先把第一个放进来 while len(url_queue)>0: #出栈 每次取出最后一个 url = url_queue.pop() print("\t" * deepdict[url],'当前层级:%d'%deepdict[url]) if deepdict[url]<3: #获取子url列表 sonurl_list = get_son_url(url) for sonurl in sonurl_list: #过滤出有效链接 if sonurl.startswith('https') or sonurl.startswith('http'): if sonurl not in deepdict: #过滤重复url deepdict[sonurl] = deepdict[url]+1 #子url相比父url层级+1 #入队列 url_queue.append(sonurl) if __name__ == '__main__': url = "https://www.baidu.com/s?wd=%E6%AD%A6%E6%B1%89%E5%85%89%E8%B0%B7" # 控制层级 deepdict = {} #控制层级 deepdict[url] = 1 # 默认第一级 vast_path(url)

万水千山总是情，点个关注行不行。