当前位置：首页 > backend >正文

Go语言多线程爬虫与代理IP反爬

backend 2025/5/14 12:52:02

有个朋友想用Go语言编写一个多线程爬虫，并且使用代理IP来应对反爬措施。多线程在Go中通常是通过goroutine实现的，所以应该使用goroutine来并发处理多个网页的抓取。然后，代理IP的话，可能需要一个代理池，从中随机选择代理来发送请求，避免同一个IP被封锁。大体思路就是这样，具体看我下面实操吧。

在这里插入图片描述

以下是一个使用Go语言编写的多线程爬虫程序，结合代理IP和反爬策略的示例代码：

package mainimport ("bufio""crypto/tls""fmt""io/ioutil""math/rand""net""net/http""net/url""os""sync""time"
)var (proxyPool  []*url.URLproxyMux   sync.MutexuserAgents = []string{"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",}visitedURLs sync.Map
)func main() {// 初始化随机种子rand.Seed(time.Now().UnixNano())// 加载代理列表if err := loadProxies("proxies.txt"); err != nil {fmt.Println("Error loading proxies:", err)return}// 创建任务通道taskChan := make(chan string, 100)// 启动workervar wg sync.WaitGroupworkerCount := 10for i := 0; i < workerCount; i++ {wg.Add(1)go worker(taskChan, &wg)}// 添加初始任务go func() {taskChan <- "https://httpbin.org/ip"taskChan <- "https://example.com"}()// 等待所有worker完成（实际可能需要其他停止机制）wg.Wait()close(taskChan)
}func loadProxies(filename string) error {file, err := os.Open(filename)if err != nil {return err}defer file.Close()scanner := bufio.NewScanner(file)for scanner.Scan() {line := scanner.Text()proxyURL, err := url.Parse("http://" + line)if err != nil {continue}proxyPool = append(proxyPool, proxyURL)}return nil
}func worker(taskChan <-chan string, wg *sync.WaitGroup) {defer wg.Done()for targetURL := range taskChan {// URL去重if _, loaded := visitedURLs.LoadOrStore(targetURL, true); loaded {continue}// 随机延迟（1-3秒）time.Sleep(time.Duration(1+rand.Intn(2)) * time.Second)// 获取随机代理proxy := getRandomProxy()if proxy == nil {fmt.Println("No available proxies")continue}// 创建自定义Transporttransport := &http.Transport{Proxy: http.ProxyURL(proxy),DialContext: (&net.Dialer{Timeout:   10 * time.Second,KeepAlive: 10 * time.Second,}).DialContext,TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},MaxIdleConns:          100,IdleConnTimeout:       90 * time.Second,TLSHandshakeTimeout:   10 * time.Second,ExpectContinueTimeout: 1 * time.Second,}// 创建HTTP客户端client := &http.Client{Transport: transport,Timeout:   15 * time.Second,}// 创建请求req, err := http.NewRequest("GET", targetURL, nil)if err != nil {fmt.Printf("Error creating request: %v\n", err)continue}// 设置随机User-Agentreq.Header.Set("User-Agent", userAgents[rand.Intn(len(userAgents))])// 发送请求resp, err := client.Do(req)if err != nil {fmt.Printf("Error fetching %s: %v\n", targetURL, err)markBadProxy(proxy)continue}defer resp.Body.Close()// 处理响应if resp.StatusCode != http.StatusOK {fmt.Printf("Non-200 status code: %d for %s\n", resp.StatusCode, targetURL)continue}// 读取响应内容body, err := ioutil.ReadAll(resp.Body)if err != nil {fmt.Printf("Error reading response body: %v\n", err)continue}// 处理页面内容（示例：输出内容长度）fmt.Printf("Successfully fetched %s (%d bytes) using proxy %s\n",targetURL, len(body), proxy.Host)// 这里可以添加链接提取和任务添加逻辑// parseLinks(body, taskChan)}
}func getRandomProxy() *url.URL {proxyMux.Lock()defer proxyMux.Unlock()if len(proxyPool) == 0 {return nil}return proxyPool[rand.Intn(len(proxyPool))]
}func markBadProxy(badProxy *url.URL) {proxyMux.Lock()defer proxyMux.Unlock()for i, p := range proxyPool {if p.String() == badProxy.String() {// 移除无效代理proxyPool = append(proxyPool[:i], proxyPool[i+1:]...)fmt.Printf("Removed bad proxy: %s\n", badProxy.String())break}}
}// 示例链接解析函数（需要根据实际情况实现）
func parseLinks(body []byte, taskChan chan<- string) {// 这里可以添加HTML解析逻辑，提取链接并发送到taskChan// 注意：需要处理相对路径转换和域名限制
}