Python爬虫笔记（04）：数据解析

from openpyxl import Workbookwb = Workbook()  # 1. 创建一个新的工作簿 Workbook首字母大写ws = wb.active   # 2. 获取当前活动的工作表（默认第一个）ws.title = "数据"  #3.  设置工作表名称​# 4：使用append添加一行数据   每次使用append都是一行数据ws.append(["学号", "姓名", "成绩", "是否及格"])  # 表头ws.append([1001, "张三", 85, "是"]) ws.append([1002, "李四", 92, "是"])ws.append([1003, "王五", 58, "否"])wb.save("example.xlsx") # 5. 设置文件名称与位置,以及将数据保存至这个文件中

# 完整版import requestsimport json, timefrom jsonpath import jsonpathfrom openpyxl import Workbook​wb = Workbook()  # 创建一个xlsx表格文件ws = wb.active  # 获取创建的表格文件ws.title = "新闻数据"  # 设置工作表格名称ws.append(["序号", "标题", "时间", "地址", "详情"])  # 表头def spider(url, headers, data):    res = requests.post(url, headers=headers, json=data)    return res.json()    pass​def save_data(result):    title = jsonpath(result, "$.data..title")  # 标题    id = jsonpath(result, "$.data..id")  # 序号    publish_time = jsonpath(result, "$.data..publish_time")  # 时间    long_summary = jsonpath(result, "$.data..long_summary")  # 详情    user_address = jsonpath(result, "$.data..user_address")  # 地址    for title1, id1, publish_time1, long_summary1, user_address1 in zip(title, id, publish_time, long_summary, user_address):        ws.append([id1, title1, publish_time1,user_address1, long_summary1])        print(id1, title1, publish_time1,user_address1, long_summary1)    # 设置文件名称与位置    wb.save("./data/example.xlsx")    pass​if __name__ == "__main__":    url = "https://i.news.qq.com/web_feed/getHotModuleList"    headers = {        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"    }    for i in range(1,5):        print(f"第{i}页")        data = {            "base_req": {                "from": "pc"            },            "forward": "2",            "qimei36": "0_RpPpK7EBAwAEp",            "device_id": "0_RpPpK7EBAwAEp",            "flush_num": i,            "channel_id": "news_news_top",            "item_count": 20        }        result = spider(url, headers, data)        save_data(result)

from bs4 import BeautifulSoup  #导包 # 创建BeautifulSoup对象soup = BeautifulSoup(res.text, 'lxml')  # 或使用   # 常用解析器比较 Python内置，无需额外安装soup = BeautifulSoup(res.text, 'html.parser')  ​# 速度快，需要 pip install lxml    				soup = BeautifulSoup(res.text, 'lxml')         ​# 容错性最好，但速度慢 pip install html5lib soup = BeautifulSoup(res.text, 'html5lib')  ​# 安装对应模块之后BeautifulSoup会自动使用

# 选择器示例soup.select('ul')           # 所有<ul> ←→ find_all('ul')soup.select('.list')        # class="list" ←→ find_all(class_="list")soup.select('#fruits')      # id="fruits" ←→ find_all(id="fruits")soup.select('ul#fruits')    # id="fruits"的<ul>soup.select('ul > li')      # 直接子元素 ←→ find_all('ul').find_all('li', recursive=False)soup.select('ul li:first-child')  # 每个<ul>的第一个<li>

​tag['属性名']  	  #  不存在会报错tag.attrs['属性名']  #  不存在会报错tag.get("属性名")    # 不存在返回None​tag.attrs 	# 获取所有属性tag.name     # 获取标签名称

//html/body/div/*  # 选择div元素的所有子元素节点​//html/body/div/@*  # 选择div元素的所有属性​/html/body//*[@class] #获取后代具有class类名的元素​/html/body/div/node() # 获取div子级的所有节点​# | 或tree.xpath('//p[@class="intro"] | //div[@class="content"]')

# 存在空格是无法匹配成功/html/body/a[text()="你好"] # 获取a文字为你好的标签​#  清除a标签文字的空格 进行匹配/html/body/div//a[normalize-space(text())="百度"]​# 获取对应的属性/html/body/div//a[normalize-space(text())="百度"]/@href

 # 获取a包含 文字为百度的标签  存在空格也没事/html/body//a[contains(text(),"百度")]​# 获取对应的属性/html/body//a[contains(text(),"百度")]/@href​# 获取a标签包含n的类名 标签/html/body//a[contains(@class,"n")]

import requestsfrom lxml import etreeimport reimport time​def spider(url):    global index    headers = {}    resp = requests.request("GET", url, headers=headers)    if resp.status_code == 200:        xml = etree.HTML(resp.text)        elements = xml.xpath('//div[@class="bot-content"]')        page_next = xml.xpath('//*[@id="__layout"]/div/div[3]/div[1]/div[4]/div/div[2]/div/div[3]/a[2]/@href')​        for ele in elements:            href = ele.xpath('./@data-href')[0]            price = ele.xpath('./div[@class="price"]//text()')[0]  # 获取价格            title = ele.xpath('./div[@class="name-pic-box"]//text()')[0]  # 获取价格            spider_msg(href, price, title)​        index = index + 1        if index < 8:            spider(page_next)​    else:        print("请求失败:", resp.status_code)    pass​​def spider_msg(href, price, title):    time.sleep(1.5)    headers = {}    resp = requests.request("GET", href, headers=headers)    if resp.status_code == 200:        xml = etree.HTML(resp.text)        ele = xml.xpath('//div[@class="price-box font12"]/div//text()')        result = "".join(ele).strip()        result = re.sub(r'\s+', "-", result)        with open("./ce.txt", "a", encoding="utf-8") as f:            f.write(f'店铺:{title}\n价格:{price}\n时间:{result}\n网址:{href}\n\n')​if __name__ == "__main__":    url = "https://www.zbj.com/fw/?es=9011"    index = 1    spider(url)