当前位置：首页 > news >正文

Go语言高并发爬虫程序源码

news 2025/6/14 10:32:32

因为最近工作量有点大，都是反复的做那几个事情，正好之前有用GO语言写的爬虫，现在稍微修改下，然后实现高并发实现快速抓取数据，因为有些属于商业机密，我就写成一个通用的模版以供大家参考。

在这里插入图片描述

下面是一个使用Go语言编写的高并发爬虫程序模板，采用工作池模式实现高效并发处理，并包含美观的UI界面展示爬取结果。

package mainimport ("fmt""io""net/http""sync""time""fyne.io/fyne/v2""fyne.io/fyne/v2/app""fyne.io/fyne/v2/container""fyne.io/fyne/v2/data/binding""fyne.io/fyne/v2/widget"
)// 爬虫任务结构体
type CrawlTask struct {URL string
}// 爬取结果结构体
type CrawlResult struct {URL        stringStatus     intTitle      stringSize       intDuration   time.DurationError      error
}func main() {// 创建Fyne应用myApp := app.New()myWindow := myApp.NewWindow("Go高并发爬虫")myWindow.Resize(fyne.NewSize(800, 600))// 创建数据绑定urls := binding.NewStringList()results := binding.NewStringList()activeWorkers := binding.NewInt()completedTasks := binding.NewInt()failedTasks := binding.NewInt()totalSize := binding.NewInt()// 初始化数据urls.Set([]string{"https://golang.org","https://golang.org/pkg","https://golang.org/pkg/net/http/","https://golang.org/doc","https://golang.org/blog","https://golang.org/pkg/fmt/","https://golang.org/pkg/os/","https://golang.org/pkg/encoding/json/",})activeWorkers.Set(0)completedTasks.Set(0)failedTasks.Set(0)totalSize.Set(0)// 创建UI组件title := widget.NewLabelWithStyle("Go高并发爬虫", fyne.TextAlignCenter, fyne.TextStyle{Bold: true})urlList := widget.NewListWithData(urls,func() fyne.CanvasObject {return widget.NewLabel("template")},func(i binding.DataItem, o fyne.CanvasObject) {o.(*widget.Label).Bind(i.(binding.String))})addUrlEntry := widget.NewEntry()addUrlEntry.SetPlaceHolder("输入要爬取的URL...")addButton := widget.NewButton("添加URL", func() {if addUrlEntry.Text != "" {current, _ := urls.Get()urls.Set(append(current, addUrlEntry.Text))addUrlEntry.SetText("")}})statusGrid := container.NewGridWithColumns(4,widget.NewLabel("活跃工作协程:"),widget.NewLabelWithData(binding.IntToString(activeWorkers)),widget.NewLabel("已完成任务:"),widget.NewLabelWithData(binding.IntToString(completedTasks)),widget.NewLabel("失败任务:"),widget.NewLabelWithData(binding.IntToString(failedTasks)),widget.NewLabel("总下载量:"),widget.NewLabelWithData(binding.IntToStringWithFormat(totalSize, "%.0f KB")),)resultList := widget.NewListWithData(results,func() fyne.CanvasObject {return widget.NewLabel("result template")},func(i binding.DataItem, o fyne.CanvasObject) {o.(*widget.Label).Bind(i.(binding.String))})// 创建控制按钮startButton := widget.NewButton("开始爬取", nil)stopButton := widget.NewButton("停止", nil)clearButton := widget.NewButton("清空结果", func() {results.Set([]string{})completedTasks.Set(0)failedTasks.Set(0)totalSize.Set(0)})// 创建任务通道和控制通道taskQueue := make(chan CrawlTask, 100)resultChan := make(chan CrawlResult, 100)stopChan := make(chan struct{})var wg sync.WaitGroup// 更新开始按钮功能startButton.OnTapped = func() {// 重置统计completedTasks.Set(0)failedTasks.Set(0)totalSize.Set(0)results.Set([]string{})// 获取URL列表urlList, _ := urls.Get()// 启动工作池workerCount := 5for i := 0; i < workerCount; i++ {wg.Add(1)go worker(taskQueue, resultChan, stopChan, &wg, activeWorkers)}// 添加任务到队列go func() {for _, u := range urlList {taskQueue <- CrawlTask{URL: u}}close(taskQueue)}()// 处理结果go func() {for result := range resultChan {// 更新统计信息if result.Error != nil {failedTasks.Set(failedTasks.Get() + 1)} else {completedTasks.Set(completedTasks.Get() + 1)totalSize.Set(totalSize.Get() + result.Size/1024) // 转换为KB}// 添加结果到列表current, _ := results.Get()var status stringif result.Error != nil {status = fmt.Sprintf("失败: %s", result.Error)} else {status = fmt.Sprintf("成功: %d 字节 (耗时: %v)", result.Size, result.Duration)}newResult := fmt.Sprintf("[%s] %s", result.URL, status)results.Set(append(current, newResult))}}()// 等待所有工作完成go func() {wg.Wait()close(resultChan)}()}// 停止按钮功能stopButton.OnTapped = func() {close(stopChan)}// 布局界面topBar := container.NewHBox(addUrlEntry, addButton)controlBar := container.NewHBox(startButton, stopButton, clearButton)content := container.NewBorder(container.NewVBox(title, topBar, statusGrid, controlBar),nil,nil,nil,container.NewHSplit(container.NewBorder(widget.NewLabel("待爬取URL列表"), nil, nil, nil, urlList),container.NewBorder(widget.NewLabel("爬取结果"), nil, nil, nil, resultList),),)myWindow.SetContent(content)myWindow.ShowAndRun()
}// 工作协程函数
func worker(tasks <-chan CrawlTask, results chan<- CrawlResult, stopCh <-chan struct{}, wg *sync.WaitGroup, active binding.Int) {defer wg.Done()active.Set(active.Get() + 1)defer func() {active.Set(active.Get() - 1)}()for task := range tasks {select {case <-stopCh:returndefault:start := time.Now()// 执行HTTP请求resp, err := http.Get(task.URL)if err != nil {results <- CrawlResult{URL:   task.URL,Error: err,}continue}defer resp.Body.Close()// 读取响应体body, err := io.ReadAll(resp.Body)if err != nil {results <- CrawlResult{URL:   task.URL,Error: err,}continue}// 创建结果duration := time.Since(start)results <- CrawlResult{URL:      task.URL,Status:   resp.StatusCode,Size:     len(body),Duration: duration,}}}
}