jackfrued
/
Python-100-Days
mirror of https://github.com/jackfrued/Python-100-Days.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

hello crawler

pull/6/head
jackfrued 2018-05-30 11:42:00 +08:00
parent b8725569cd
commit 79f297a466
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Day66-75/code/example06.py
View File

@ -17,20 +17,22 @@ def main():
seed_url = urljoin(base_url, 'explore') seed_url = urljoin(base_url, 'explore')
# 创建Redis客户端 # 创建Redis客户端
client = Redis(host='1.2.3.4', port=6379, password='1qaz2wsx') client = Redis(host='1.2.3.4', port=6379, password='1qaz2wsx')
# 设置用户代理 # 设置用户代理(否则访问会被拒绝)
headers = {'user-agent': 'Baiduspider'} headers = {'user-agent': 'Baiduspider'}
# 通过requests模块发送GET请求并指定用户代理 # 通过requests模块发送GET请求并指定用户代理
resp = requests.get(seed_url, headers=headers) resp = requests.get(seed_url, headers=headers)
# 创建BeautifulSoup对象并指定使用lxml作为解析器 # 创建BeautifulSoup对象并指定使用lxml作为解析器
soup = BeautifulSoup(resp.text, 'lxml') soup = BeautifulSoup(resp.text, 'lxml')
href_regex = re.compile(r'^/question') href_regex = re.compile(r'^/question')
# 将URL处理成SHA1摘要(长度固定更简短)
hasher_proto = sha1()
# 查找所有href属性以/question打头的a标签 # 查找所有href属性以/question打头的a标签
for a_tag in soup.find_all('a', {'href': href_regex}): for a_tag in soup.find_all('a', {'href': href_regex}):
# 获取a标签的href属性值并组装完整的URL # 获取a标签的href属性值并组装完整的URL
href = a_tag.attrs['href'] href = a_tag.attrs['href']
full_url = urljoin(base_url, href) full_url = urljoin(base_url, href)
# 将URL处理成SHA1摘要(长度固定更简短) # 传入URL生成SHA1摘要
hasher = sha1() hasher = hasher_proto.copy()
hasher.update(full_url.encode('utf-8')) hasher.update(full_url.encode('utf-8'))
field_key = hasher.hexdigest() field_key = hasher.hexdigest()
# 如果Redis的键'zhihu'对应的hash数据类型中没有URL的摘要就访问页面并缓存 # 如果Redis的键'zhihu'对应的hash数据类型中没有URL的摘要就访问页面并缓存