项目设计文档——爬虫项目（爬取天气预报）

一、项目背景以及项目意义

项目背景：

爬虫技术的核心目的是自动化地从互联网上采集，提取和存储数据。网络爬虫是一种自动化程序，用于从互联网上抓取数据并进行处理。C语言因其高效性和接近硬件的特性，常被用于开发高性能的网络爬虫，尤其是在资源受限或需要极致性能的场景中。

项目意义：

综合使用了学习到的网络的TCP协议，套接字编程和http协议，与网络服务端建立相关的联系，从网络网址当中爬取自己想要的相关数据。

二、项目实现的功能

1.通过TCP协议建立联系

2.通过http协议获得响应报文以及正文

3.解析HTML和相应网页内容

4.提取自己想要的数据

三、项目的整体框图以及各部分如何实现

四、项目中的流程图及说明

五、项目中遇到的问题及解决方法

1.在查询https://sapi.k780.com/的端口和IP地址时只找到https协议的地址造成无法爬取

解决方法：通过在网上搜集资料了解应该将网址上的s和sapi的s去掉，即可访问http网址

2.在将请求报文发送时，不能将所想要查询的城市输入到请求报文当中

解决方法：通过主函数传参以及利用了snprintf函数将请求报文保存到了一个数组当中

3.在将回复的响应报文保存到文本当中后，不知道怎么将文本当中的数据读到cjson当中

解决方法：利用了一个strstr函数对文本当中的数据进行了一个定位，将指针定位到了{\处进行解析报文

4.在解析cjson报文当中不知道如何解析嵌套的JSON数据

解决方法：通过第一次先解析到外层的JSON数据，然后再依次往里面进行解析，其相应报文中的数字，汉语，字母等都是字符串类型的要用valuestring。

5.在JSON数组当中解析JSON数据

解决方法：在一个JSON数组当中转换字符串应该先获得该数组的大小，然后进行循环，利用item进行获取数组，再获取数组当中想要的数据

#include "pachong.h"
#include "cJSON.h"int main(int argc,const char *argv[])
{if(argc != 2){printf("Usage : ./a.out <城市名字>\n");return -1;}int sockfd = create_tcp_connect();if(sockfd < 0){perror("socket error");return -1;}printf("-----------------------------\n");printf("查询实时天气或天气预报-------\n");printf("1.查询实时天气---------------\n");printf("2.查询天气预报---------------\n");printf("3.退出-----------------------\n");printf("-----------------------------\n");int choose;scanf("%d",&choose);switch (choose){case 1: send_http_request_weather_today(sockfd,argv[1]);recv_http_response(sockfd);find_today_weather();break;case 2: send_http_requesr_weather_prediction(sockfd,argv[1]);recv_http_response(sockfd);find_prediction_weather();break;case 3: exit_program(sockfd);break;default:break;}close(sockfd);return 0;
}

#include "pachong.h"
#include "cJSON.h"
#define SER_PORT 80
#define SER_IP "8.129.233.227"
int create_tcp_connect()
{int sockfd = socket(AF_INET,SOCK_STREAM,0);if(sockfd < 0){perror("socket error");return -1;}struct  sockaddr_in seraddr;seraddr.sin_family = AF_INET;seraddr.sin_port = htons(SER_PORT);seraddr.sin_addr.s_addr = inet_addr(SER_IP); int ret = connect(sockfd,(struct sockaddr *)&seraddr,sizeof(seraddr));if(ret < 0){perror("connect error");return -1;}return sockfd;}
int send_http_request_weather_today(int sockfd,const char *cityname)
{char buff[4096] = {0};snprintf(buff,sizeof(buff),"GET /?app=weather.today&cityNm=%s&appkey=77275&sign=3b1921902ede0a45608c50b0f1b919c3&format=json HTTP/1.1\r\n""Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\r\n""Accept-Encoding: gzip, deflate\r\n""Accept-Language: zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6\r\n""Cache-Control: max-age=0\r\n""Connection: close\r\n""Host: api.k780.com\r\n""Upgrade-Insecure-Requests: 1\r\n""User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0 \r\n""\r\n",cityname);ssize_t cnt = send(sockfd,buff,strlen(buff),0);if(cnt < 0){perror("send error");return -1;}return 0;}
void exit_program(int sockfd)
{if(sockfd >= 0){close(sockfd);}printf("已退出\n");exit(0);}int send_http_requesr_weather_prediction(int sockfd,const char *cityname)
{ char buff[4096] = {0};snprintf(buff,sizeof(buff),"GET /?app=weather.future&cityNm=%s&appkey=77275&sign=3b1921902ede0a45608c50b0f1b919c3&format=json HTTP/1.1\r\n""Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\r\n""Accept-Encoding: gzip, deflate\r\n""Accept-Language: zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6\r\n""Cache-Control: max-age=0\r\n""Connection: close\r\n""Host: api.k780.com\r\n""Upgrade-Insecure-Requests: 1\r\n""User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0\r\n""\r\n",cityname);// printf("buff:%s\n",buff);ssize_t cnt = send(sockfd,buff,strlen(buff),0);if(cnt < 0){perror("send error");return -1;}return 0;}
int recv_http_response(int sockfd)
{char buff[1024] = {0};int fd = open("./1.txt",O_WRONLY | O_CREAT |O_TRUNC,0664);if(fd < 0){perror("open error");return -1;}while(1){ssize_t cnt = recv(sockfd,buff,sizeof(buff),0);if(cnt < 0){perror("recv error");return -1;}else if(0 == cnt){printf("off\n");break;}write(fd,buff,cnt);// write(1,buff,cnt);}close(fd);return 0;}
int find_today_weather()
{int fd = open("1.txt",O_RDONLY);if(fd < 0){perror("open error");return -1;}char buff[40960] = {0};ssize_t cnt = read(fd,buff,sizeof(buff) - 1);if(cnt <= 0){perror("read error");return -1;}close(fd);buff[cnt] = '\0';const char *json_start = strstr(buff,"\r\n\r\n");json_start = strstr(buff,"{\"");cJSON* json = NULL;json = cJSON_Parse(json_start);if(!json){   printf("json error : %s",cJSON_GetErrorPtr());return -1;}else{cJSON* json_result = cJSON_GetObjectItem(json,"result");cJSON* json_days = cJSON_GetObjectItem(json_result,"days");if(cJSON_IsString(json_days)){printf("日期：%s\n",json_days->valuestring);}cJSON* json_citynm = cJSON_GetObjectItem(json_result,"citynm");if(cJSON_IsString(json_citynm)){printf("城市：%s\n",json_citynm->valuestring);}cJSON* json_week = cJSON_GetObjectItem(json_result,"week");if(cJSON_IsString(json_week)){printf("星期：%s\n",json_week->valuestring);}cJSON* json_temperature = cJSON_GetObjectItem(json_result,"temperature");if(cJSON_IsString(json_temperature)){printf("最高温度/最低温度：%s\n",json_temperature->valuestring);}cJSON* json_temperature_curr = cJSON_GetObjectItem(json_result,"temperature_curr");if(cJSON_IsString(json_temperature_curr)){printf("当前温度：%s\n",json_temperature_curr->valuestring);}cJSON* json_humidity = cJSON_GetObjectItem(json_result,"humidity");if(cJSON_IsString(json_humidity)){printf("湿度：%s\n",json_humidity->valuestring);}cJSON* json_weather = cJSON_GetObjectItem(json_result,"weather");if(cJSON_IsString(json_weather)){printf("今天天气：%s\n",json_weather->valuestring);}cJSON* json_weather_curr = cJSON_GetObjectItem(json_result,"weather_curr");if(cJSON_IsString(json_weather_curr)){printf("当前天气：%s\n",json_weather_curr->valuestring);}cJSON* json_wind = cJSON_GetObjectItem(json_result,"wind");if(cJSON_IsString(json_wind)){printf("风向：%s\n",json_wind->valuestring);}   cJSON* json_winp = cJSON_GetObjectItem(json_result,"winp");if(cJSON_IsString(json_winp)){printf("风力：%s\n",json_winp->valuestring);}   }cJSON_Delete(json);return 0;}
int find_prediction_weather()
{int fd = open("1.txt",O_RDONLY);if(fd < 0){perror("open error");return -1;}char buff[40960] = {0};ssize_t cnt = read(fd,buff,sizeof(buff) - 1);if(cnt <= 0){perror("read error");return -1;}close(fd);buff[cnt] = '\0';const char *json_start = strstr(buff,"\r\n\r\n");json_start = strstr(buff,"{\"");cJSON* json = NULL;json = cJSON_Parse(json_start);if(!json){   printf("json error : %s",cJSON_GetErrorPtr());return -1;}else{cJSON* json_result = cJSON_GetObjectItem(json,"result");if(!json_result){printf("json_result error : %s",cJSON_GetErrorPtr());return -1;}int size = cJSON_GetArraySize(json_result);for(int i = 0;i < size;i++){cJSON* item = cJSON_GetArrayItem(json_result,i);if(!item){printf("item error : %s",cJSON_GetErrorPtr());return -1;}if(cJSON_IsObject(item)){cJSON* json_days = cJSON_GetObjectItem(item,"days");if(cJSON_IsString(json_days)){printf("日期：%s\n",json_days->valuestring);}else{printf("error\n");}cJSON* json_citynm = cJSON_GetObjectItem(item,"citynm");if(cJSON_IsString(json_citynm)){printf("城市：%s\n",json_citynm->valuestring);}cJSON* json_week = cJSON_GetObjectItem(item,"week");if(cJSON_IsString(json_week)){printf("星期：%s\n",json_week->valuestring);}cJSON* json_temperature = cJSON_GetObjectItem(item,"temperature");if(cJSON_IsString(json_temperature)){printf("最高温度/最低温度：%s\n",json_temperature->valuestring);}cJSON* json_temperature_curr = cJSON_GetObjectItem(item,"temperature_curr");if(cJSON_IsString(json_temperature_curr)){printf("当前温度：%s\n",json_temperature_curr->valuestring);}cJSON* json_humidity = cJSON_GetObjectItem(item,"humidity");if(cJSON_IsString(json_humidity)){printf("湿度：%s\n",json_humidity->valuestring);}cJSON* json_weather = cJSON_GetObjectItem(item,"weather");if(cJSON_IsString(json_weather)){printf("今天天气：%s\n",json_weather->valuestring);}cJSON* json_weather_curr = cJSON_GetObjectItem(item,"weather_curr");if(cJSON_IsString(json_weather_curr)){printf("当前天气：%s\n",json_weather_curr->valuestring);}cJSON* json_wind = cJSON_GetObjectItem(item,"wind");if(cJSON_IsString(json_wind)){printf("风向：%s\n",json_wind->valuestring);}   cJSON* json_winp = cJSON_GetObjectItem(item,"winp");if(cJSON_IsString(json_winp)){printf("风力：%s\n",json_winp->valuestring);}printf("\n");}   }}return 0;
}

#ifndef __PACHONG_H__
#define __PACHONG_H__#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>          /* See NOTES */
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/ip.h> 
#include <arpa/inet.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
extern int find_today_weather();
extern void exit_program(int sockfd);
extern int send_http_requesr_weather_prediction(int sockfd,const char *cityname);
extern int send_http_request_weather_today(int sockfd,const char *cityname);
extern int recv_http_response(int sockfd);
extern int create_tcp_connect();
extern int find_prediction_weather();
#endif