update code in python_spider
parent
54a152483b
commit
8be49fcbfd
|
@ -1,8 +1,12 @@
|
||||||
# LearnPython
|
# LearnPython
|
||||||
以撸代码的形式学习Python
|
以撸代码的形式学习Python
|
||||||
|
|
||||||
|
============================================================
|
||||||
### python_base.py: 千行代码入门Python
|
### python_base.py: 千行代码入门Python
|
||||||
|
|
||||||
### python_visual.py: 15张图入门Matplotlib
|
### python_visual.py: 15张图入门Matplotlib
|
||||||
|
|
||||||
### python_spider.py: 一个很“水”的Python爬虫入门代码文件
|
### python_spider.py: 一个很“水”的Python爬虫入门代码文件
|
||||||
|
============================================================
|
||||||
|
|
||||||
|
### 您可以fork该项目,并在修改后提交Pull request
|
||||||
|
|
|
@ -9,79 +9,88 @@ import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
import http.cookiejar
|
import http.cookiejar
|
||||||
|
|
||||||
|
# 首先定义下边可能需要的变量
|
||||||
|
url = "https://www.baidu.com"
|
||||||
|
headers = {"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"}
|
||||||
|
|
||||||
# 最简单的方式
|
# 最简单的网页抓取方式
|
||||||
response = urllib.request.urlopen("http://www.baidu.com", timeout=10)
|
response = urllib.request.urlopen(url, timeout=10)
|
||||||
html = response.read().decode("utf-8")
|
html = response.read().decode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
# 使用Request类
|
# 使用Request实例代替url
|
||||||
request = urllib.request.Request("http://www.baidu.com/")
|
request = urllib.request.Request(url, data=None, headers={})
|
||||||
response = urllib.request.urlopen(request, timeout=10)
|
response = urllib.request.urlopen(request, timeout=10)
|
||||||
|
|
||||||
|
|
||||||
# 发送数据,即在urlopen()或者Request()中添加data参数
|
# 发送数据,即在Request()中添加data参数
|
||||||
url = "http://localhost/login.php"
|
|
||||||
data = urllib.parse.urlencode({"act": "login", "email": "xianhu@qq.com", "password": "123456"})
|
data = urllib.parse.urlencode({"act": "login", "email": "xianhu@qq.com", "password": "123456"})
|
||||||
request1 = urllib.request.Request(url, data) # POST方法
|
request1 = urllib.request.Request(url, data=data) # POST方法
|
||||||
request2 = urllib.request.Request(url + "?%s" % data) # GET方法
|
request2 = urllib.request.Request(url+"?%s" % data) # GET方法
|
||||||
response = urllib.request.urlopen(request, timeout=10)
|
response = urllib.request.urlopen(request, timeout=10)
|
||||||
|
|
||||||
|
|
||||||
# 发送Header,即在urlopen()或者Request()中添加headers参数
|
# 发送Header,即在Request()中添加headers参数
|
||||||
headers = {"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"}
|
|
||||||
request = urllib.request.Request(url, data=data, headers=headers) # 参数中添加header参数
|
request = urllib.request.Request(url, data=data, headers=headers) # 参数中添加header参数
|
||||||
request.add_header("Referer", "http://www.baidu.com") # add_header函数,另一种添加header的方法
|
request.add_header("Referer", "http://www.baidu.com") # 另一种添加header的方式,添加Referer是为了应对"反盗链"
|
||||||
response = urllib.request.urlopen(request, timeout=10)
|
response = urllib.request.urlopen(request, timeout=10)
|
||||||
|
|
||||||
|
|
||||||
# 对付"反盗链":所谓的反盗链设置,就是检查header里的referer站点是不是他自己。所以只需要像把headers的referer改成某个网站自己即可
|
# 网页抓取引发异常:urllib.error.HTTPError, urllib.error.URLError, 两者存在继承关系
|
||||||
headers = {"Referer": "http://www.baidu.com/"}
|
|
||||||
|
|
||||||
|
|
||||||
# 引发异常:urllib.error.HTTPError, urllib.error.URLError, 两者存在继承关系
|
|
||||||
try:
|
try:
|
||||||
urllib.request.urlopen(request, timeout=10)
|
urllib.request.urlopen(request, timeout=10)
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
print(e.code, e.reason)
|
print(e.code, e.reason)
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
print(e.errno, e.reason)
|
||||||
|
|
||||||
|
|
||||||
# 使用代理,以防止IP被封或IP次数受限:
|
# 使用代理,以防止IP被封或IP次数受限:
|
||||||
proxy = urllib.request.ProxyHandler({"http": "111.123.76.12:8080"})
|
proxy_handler = urllib.request.ProxyHandler(proxies={"http": "111.123.76.12:8080"})
|
||||||
|
|
||||||
opener = urllib.request.build_opener(proxy) # 利用代理创建opener实例(OpenerDirector实例)
|
opener = urllib.request.build_opener(proxy_handler) # 利用代理创建opener实例
|
||||||
response = opener.open("https://www.baidu.com/") # 直接利用opener实例打开url
|
response = opener.open(url) # 直接利用opener实例打开url
|
||||||
|
|
||||||
urllib.request.install_opener(opener) # 安装、设置全局的opener,然后利用urlopen打开url
|
urllib.request.install_opener(opener) # 安装全局opener,然后利用urlopen打开url
|
||||||
response = urllib.request.urlopen("https://www.baidu.com/")
|
response = urllib.request.urlopen(url)
|
||||||
|
|
||||||
|
|
||||||
|
# 使用cookie和cookiejar,应对服务器检查
|
||||||
|
cookie_jar = http.cookiejar.CookieJar()
|
||||||
|
cookie_jar_handler = urllib.request.HTTPCookieProcessor(cookiejar=cookie_jar)
|
||||||
|
opener = urllib.request.build_opener(cookie_jar_handler)
|
||||||
|
response = opener.open(url)
|
||||||
|
|
||||||
|
|
||||||
|
# 发送在浏览器中获取的cookie,两种方式:
|
||||||
|
# (1)直接放到headers里
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)",
|
||||||
|
"Cookie": "PHPSESSID=btqkg9amjrtoeev8coq0m78396; USERINFO=n6nxTHTY%2BJA39z6CpNB4eKN8f0KsYLjAQTwPe%2BhLHLruEbjaeh4ulhWAS5RysUM%2B; "
|
||||||
|
}
|
||||||
|
request = urllib.request.Request(url, headers=headers)
|
||||||
|
|
||||||
|
# (2)构建cookie,添加到cookiejar中
|
||||||
|
cookie = http.cookiejar.Cookie(name="xx", value="xx", domain="xx", ...)
|
||||||
|
cookie_jar.set_cookie(cookie)
|
||||||
|
response = opener.open(url)
|
||||||
|
|
||||||
|
|
||||||
|
# 同时使用代理和cookiejar
|
||||||
|
opener = urllib.request.build_opener(cookie_jar_handler)
|
||||||
|
opener.add_handler(proxy_handler)
|
||||||
|
response = opener.open("https://www.baidu.com/")
|
||||||
|
|
||||||
|
|
||||||
# 抓取网页中的图片:同样适用于抓取网络上的文件。右击鼠标,找到图片属性中的地址,然后进行保存。
|
# 抓取网页中的图片:同样适用于抓取网络上的文件。右击鼠标,找到图片属性中的地址,然后进行保存。
|
||||||
url = "http://ww3.sinaimg.cn/large/7d742c99tw1ee7dac2766j204q04qmxq.jpg"
|
response = urllib.request.urlopen("http://ww3.sinaimg.cn/large/7d742c99tw1ee7dac2766j204q04qmxq.jpg", timeout=120)
|
||||||
response = urllib.request.urlopen(url, timeout=120)
|
|
||||||
with open("test.jpg", "wb") as file_img:
|
with open("test.jpg", "wb") as file_img:
|
||||||
file_img.write(response.read())
|
file_img.write(response.read())
|
||||||
|
|
||||||
|
|
||||||
# 使用cookie和cookiejar
|
# HTTP认证:即HTTP身份验证
|
||||||
cookie_jar = http.cookiejar.CookieJar()
|
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() # 创建一个PasswordMgr
|
||||||
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookiejar=cookie_jar))
|
password_mgr.add_password(realm=None, uri=url, user='username', passwd='password') # 添加用户名和密码
|
||||||
response = opener.open("https://www.baidu.com/")
|
handler = urllib.request.HTTPBasicAuthHandler(password_mgr) # 创建HTTPBasicAuthHandler
|
||||||
for cookie in cookie_jar:
|
opener = urllib.request.build_opener(handler) # 创建opner
|
||||||
print(cookie)
|
response = opener.open(url, timeout=10) # 获取数据
|
||||||
|
|
||||||
|
|
||||||
# 发送在浏览器中获取的cookie,两种方式:(1)直接放到headers里,(2)构建cookie,添加到cookiejar中
|
|
||||||
headers = {"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)",
|
|
||||||
"Cookie": "PHPSESSID=btqkg9amjrtoeev8coq0m78396; USERINFO=n6nxTHTY%2BJA39z6CpNB4eKN8f0KsYLjAQTwPe%2BhLHLruEbjaeh4ulhWAS5RysUM%2B; "}
|
|
||||||
request = urllib.request.Request(url, headers=headers)
|
|
||||||
|
|
||||||
cookie = http.cookiejar.Cookie(name="xx", value="xx", domain="xx")
|
|
||||||
cookie_jar.set_cookie(cookie)
|
|
||||||
response = opener.open("https://www.baidu.com/")
|
|
||||||
|
|
||||||
|
|
||||||
# 同时使用代理和cookiejar
|
|
||||||
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookiejar=cookie_jar))
|
|
||||||
opener.add_handler(urllib.request.ProxyHandler(proxies={"http": "http://www.example.com:8888/"}))
|
|
||||||
response = opener.open("https://www.baidu.com/")
|
|
||||||
|
|
Loading…
Reference in New Issue