直接上代码:
import time
import requests
from bs4 import BeautifulSoup
def get_content(url):
try:
user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'
response = requests.get(url, headers={'User-Agent': user_agent})
response.raise_for_status() # 如果返回的状态码不是200， 则抛出异常;
response.encoding = response.apparent_encoding # 判断网页的编码格式， 便于respons.text知道如何解码;
except Exception as e:
print("爬取错误")
else:
print(response.url)
print("爬取成功!")
return response.content
def save_img(img_src):
if img_src is None:
return
try:
print(img_src)
urlArr = img_src.split('?')
if len(urlArr) == 2:
url = urlArr[1]
else:
url = urlArr[0]
headers = {"User-Agent": 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0'}
# 注意 verify 参数设置为 False ，不验证网站证书
requests.packages.urllib3.disable_warnings()
Related Keywords