import random

from time import sleep

import pymysql

from bs4 import BeautifulSoup

import re # 正则表达式

import urllib.request, urllib.error # 定制 url 获取网页数据

def main():

parser_url_save()

def parser_url_save():

print("doing parser url ....")

# 猫眼 https://maoyan.com/board/4?offset=10

# 豆瓣 https://movie.douban.com/top250?start=25

urls = ['https://maoyan.com/board/4?offset=','https://movie.douban.com/top250?start=']

for url in urls:

if url.find('https://maoyan.com') !=-1 :

datalist = parser_html(url)

# datalist = getData_maoyan(html)

# print(datalist)

saveData_maoyan(datalist)

else :

# parser_DOUBAN(url)

print("parser douban ...")

# 正则列表

# 猫眼标题

find_maoyan_title = re.compile(r'.*?title="(.*?)"')

# 猫眼链接

find_maoyan_link = re.compile(r'.*?href="(.*?)"')

# 猫眼图片

find_maoyan_pic = re.compile(r'.*?

# 猫眼评分

find_maoyan_score1 = re.compile(r'

(.*?)<')

find_maoyan_score2 = re.compile(r'(.*?)<')

# 主演

find_maoyan_star = re.compile(r'.* 主演： (.*)')

# 上映时间

find_maoyan_date = re.compile(r' 上映时间： (.*)<')

def parser_html (url):

     cookie = '###'

     # agent=random.choice(user_agent)

     agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"

     data_maoyan_list=[]

     for i in range (0,10) :

         sleep(3)

         url_tmp = url+str(i*10)

         headers = {

             "User-Agent":agent

             ,"Cookie":cookie

         }

         req = urllib.request.Request(url_tmp, headers=headers)

         response = urllib.request.urlopen(req)

         html = response.read().decode("utf-8")

         data_maoyan_list = getData_maoyan(html,data_maoyan_list)

     return data_maoyan_list

def getData_maoyan (html,data_maoyan_list):

     html_parser = BeautifulSoup(html, "html.parser")

     base_url = 'https://maoyan.com/'

     item_list = html_parser.find_all('dd')

     for item in item_list:

         sleep(1) # 延时访问

         # 单个电影的数据集

         data = []

         item_a = str(item.a)

         # 取标题

         title = re.findall(find_maoyan_title, item_a)[0]

         # 取链接

         curr_url = base_url + str(re.findall(find_maoyan_link, item_a)[0])

         # 取图片链接

         pic = re.findall(find_maoyan_pic, item_a)[0]

         # 评分

         item_p = item.select("p[class='score']")

         # if i * 10 == 20:

         #     print(item_p)

         score = "0.0" # 存在没有评分的重置 0.0

         if len(re.findall(find_maoyan_score1, str(item_p))) > 0:

             score = float(str(re.findall(find_maoyan_score1, str(item_p))[0]) + str(

                 re.findall(find_maoyan_score2, str(item_p))[0]))

         # 主演

         # ’
‘

         item_star = item.select("p[class='star']")

         # print(str(item_star))

         star = re.findall(find_maoyan_star, str(item_star))[0]

         # 上映时间

         item_releasetime = item.select("p[class='releasetime']")

         releasetime = re.findall(find_maoyan_date, str(item_releasetime))[0]

         # 外汇跟单gendan5.com 添加到数据集中 ,title,curr_url,pic,score,star,releasetime

         data.append(title)

         data.append(curr_url)

         data.append(pic)

         data.append(score)

         data.append(star)

         data.append(releasetime)

         data_maoyan_list.append(data)

     return data_maoyan_list

def saveData_maoyan(data_list):

     conn = pymysql.connect(

             host='xxx.xx.xx.xx',# host

             port = 80, # 默认端口，根据实际修改

             user='root',# 用户名

             passwd='123456', # 密码

             db ='luke_db', # DB name

             )

     cur=conn.cursor()

     print(conn)

     # 获取了数据列表

     for id in range(0,len(data_list)):

         #      取得字段

         ind_id = str(id);

         title = '"'+str(data_list[id][0])+'"' # 标题

         link = '"'+str(data_list[id][1])+'"' # 连接

         pic_link = '"'+str(data_list[id][2])+'"' # 图片连接

         score = str(data_list[id][3]) # 评分

         actor = '"'+str(data_list[id][4])+'"' # 主演

         pub_date = '"'+str(data_list[id][5])+'"' # 上映时间

         arr=[ind_id,title,link,pic_link,score,actor,pub_date]

         sql='''

             insert into luke_db.t_movie_top100_maoyan (xh,m_title,m_link,m_pic,m_score,m_actor,m_pubdate)

              values(%s)'''%",".join(arr)

         print(sql)

         print(cur.execute(sql))

         conn.commit() # 插入数据

     cur.close()

     conn.close()

if __name__== '__main__':

     main()

python爬虫日记01