1. 普通写法
from urllib import parse
from urllib import request
name = input(' 选择您要查看的主题： ')
start = int(input(' 选择起始页： '))
end = int(input(' 选择结束页： '))
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko)"
" Chrome/90.0.4430.85 Safari/537.36 Edg/90.0.818.46"
}
base_url = 'https://tieba.baidu.com/f?kw='
for i in range(start, end + 1):
num = (i - 1) * 50
url = base_url + parse.quote(name) + '&ie=utf-8&pn=' + str(num)
req = request.Request(url, headers=headers)
response = request.urlopen(req)
html = response.read().decode('utf-8')
file_name = ' 第 ' + str(i) + ' 页内容 .html'
with open(file_name, 'w', encoding='utf-8') as file_obj:
print(' 正在爬取第 %d 页 ' % i)
file_obj.write(html)
2. 函数式写法
from urllib import parse
from urllib import request
# 获取数据
def read_url(url):
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 "
"(KHTML, like Gecko)"
" Chrome/90.0.4430.85 Safari/537.36 Edg/90.0.818.46"
}
req = request.Request(url, headers=headers)
response = request.urlopen(req)
html = response.read().decode('utf-8')
return html
# 写入数据
def write_page(file_name, html):
with open(file_name, 'w', encoding='utf-8') as file_obj:
file_obj.write(html)
print(" 写入成功 ")
# 主函数，其他的都写入其中
def main():
name = input('外汇跟单gendan5.com 选择您要查看的主题： ')
start = int(input(' 选择起始页： '))
end = int(input(' 选择结束页： '))
base_url = 'https://tieba.baidu.com/f?kw='
for i in range(start, end + 1):
num = (i - 1) * 50
url = base_url + parse.quote(name) + '&ie=utf-8&pn=' + str(num)
file_name = ' 第 ' + str(i) + ' 页内容 .html'
html = read_url(url)
write_page(file_name, html)
if __name__ == '__main__':
main()
3. 面向对象写法
from urllib import parse
from urllib import request
class BaiduSpider:
def __init__(self):
self.headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko)"
" Chrome/90.0.4430.85 Safari/537.36 Edg/90.0.818.46"
}
self.base_url = 'https://tieba.baidu.com/f?kw='
def read_page(self, url):
req = request.Request(url, headers=self.headers)
response = request.urlopen(req)
html = response.read().decode('utf-8')
return html
def write_page(self, file_name, html):
with open(file_name, 'w', encoding='utf-8') as file_obj:
file_obj.write(html)
print(" 写入成功 ")
def main(self):
name = input(' 选择您要查看的主题： ')
start = int(input(' 选择起始页： '))
end = int(input(' 选择结束页： '))
for i in range(start, end + 1):
num = (i - 1) * 50
url = self.base_url + parse.quote(name) + '&ie=utf-8&pn=' + str(num)
file_name = ' 第 ' + str(i) + ' 页内容 .html'
html = self.read_page(url)
self.write_page(file_name, html)
if __name__ == '__main__':
yes = BaiduSpider()
yes.main()