开源 python 应用 开发（六）网络爬虫

pip install requests -i https://mirrors.aliyun.com/pypi/simple/

pip install beautifulsoup4 -i https://mirrors.aliyun.com/pypi/simple/

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoinclass WebsiteLogin:def __init__(self, base_url):self.base_url = base_urlself.session = requests.Session()self.session.headers.update({'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'})def find_hidden_inputs(self, html):"""查找表单中的隐藏字段"""soup = BeautifulSoup(html, 'html.parser')hidden_inputs = {}for input_tag in soup.find_all('input', type='hidden'):hidden_inputs[input_tag.get('name')] = input_tag.get('value', '')return hidden_inputsdef login(self, login_url, username, password, extra_data=None):"""登录网站"""# 获取登录页面try:login_page = self.session.get(urljoin(self.base_url, login_url))login_page.raise_for_status()except requests.RequestException as e:print(f"访问登录页面失败: {e}")return False# 获取隐藏字段hidden_inputs = self.find_hidden_inputs(login_page.text)# 准备登录数据login_data = {'inputName': username,'inputPwd': password,**hidden_inputs}# 添加额外的表单数据if extra_data:login_data.update(extra_data)# 提交登录表单try:response = self.session.post(urljoin(self.base_url, login_url),data=login_data,headers={'Referer': urljoin(self.base_url, login_url)})response.raise_for_status()except requests.RequestException as e:print(f"提交登录表单失败: {e}")return False# 验证登录是否成功（根据实际网站调整）#if 'logout' not in response.text.lower() and 'sign out' not in response.text.lower():#    print("登录失败，请检查凭据")#    return Falseif 'login' in response.url.lower():return Falseprint("登录成功！")return Truedef get_page(self, page_url):"""获取指定页面的内容"""try:response = self.session.get(urljoin(self.base_url, page_url))response.raise_for_status()return response.textexcept requests.RequestException as e:print(f"获取页面失败: {e}")return None# 使用示例
if __name__ == "__main__":# 配置信息config = {'base_url': 'http://xxx.com','login_url': '','username': 'xxx','password': 'xxx','target_page': '/xxx/xxx'}# 创建登录实例web_login = WebsiteLogin(config['base_url'])# 尝试登录if web_login.login(config['login_url'], config['username'], config['password']):# 登录成功后获取目标页面target_content = web_login.get_page(config['target_page'])if target_content:print(f"成功获取目标页面内容，长度: {len(target_content)}")# 保存内容with open('target_page.html', 'w', encoding='utf-8') as f:f.write(target_content)else:print("未能获取目标页面内容")else:print("登录失败，程序终止")