golang:封装 google search engine

简介

破解 google 搜索接口，进行代码自动化接入 google engine，而不是通过 web driver方式，不占服务器太多带宽
利用 proto 文件作为配置结构,请自行利用 protoc 编译成对于的 pb.go 文件，配置文件采用 yaml，请自行实现读取配置关联

目录结构

image.png

代码

  • conf.proto
syntax = "proto3";
package kxconfig;

option go_package = "ggsearchengine";
import "google/protobuf/struct.proto";
import "google/protobuf/duration.proto";

message Bootstrap {
  Client client = 5;
}

message Client {
  message HTTP {
    google.protobuf.Duration timeout = 1;
    optional string proxy = 2;
    int32 max_idle_conns = 3;
    int32 max_conns_per_host = 4;
    int32 max_idle_conns_per_host = 5;
    google.protobuf.Duration proxy_timeout = 6;
    optional string premium_proxy = 7;
    google.protobuf.Duration premium_proxy_timeout = 8;
  }
  HTTP http = 1;
}
  • config.yaml
client:
  http:
    timeout: 10s
    max_idle_conns: 10
    max_conns_per_host: 100
    max_idle_conns_per_host: 10
    proxy: 127.0.0.1n:8118
    premium_proxy: 127.0.0.1n:8228
    proxy_timeout: 10s
    premium_proxy_timeout: 10s
  • google_search_engine.go
package ggsearchengine

import (
    "bytes"
    "context"
    "fmt"
    "io/ioutil"
    "net/http"
    "regexp"
    "strings"
    "time"

    ggenerator "ggsearch_engine/ggsearchengine/useragent"
    "github.com/PuerkitoBio/goquery"
    "github.com/go-kratos/kratos/v2/errors"
    "github.com/go-kratos/kratos/v2/log"
)

var (
    baseUrl = "https://www.google.com/search"
    Err     = errors.New(400, "GOOGLE_HTTP_BAD_CODE", "google search http bad code")
)

// search engine 属性不能包含每次搜索变化的变量， search engine 属性是全局唯一的
type googleSearchEngine struct {
    log          *log.Helper
    nProxyClient *NoPoolProxyHttpClient
}

func NewGoogleSearchEngine(logger log.Logger, nProxyClient *NoPoolProxyHttpClient) SearchEngine {
    return &googleSearchEngine{
        log:          log.NewHelper(logger),
        nProxyClient: nProxyClient,
    }
}

func WithMonitor(metricKey string) Option {
    return func(o *options) {
        o.metricKey = &metricKey
    }
}

func WithProxy(isFreeProxy bool) Option {
    return func(o *options) {
        o.isFreeProxy = isFreeProxy
    }
}

func WithTextExtract(textExtractionSyntax string, textExtractionIndex []int) Option {
    return func(o *options) {
        o.textExtractionSyntax = &textExtractionSyntax
        o.textExtractionIndex = textExtractionIndex
    }
}

func (g *googleSearchEngine) textExtract(ctx context.Context, text string, extractSyntax string, extractIndexes []int) []string {
    regx := regexp.MustCompile(extractSyntax)
    extractContents := regx.FindStringSubmatch(text)
    // 按照提取顺序返回
    result := make([]string, len(extractIndexes))
    for i, extractIndex := range extractIndexes {
        if len(extractContents) < extractIndex+1 {
            continue
        } else {
            result[i] = extractContents[extractIndex]
        }
    }
    return result
}

func (g *googleSearchEngine) parseGoogleResponse(ctx context.Context, o *options, r []byte) (results []*SearchResult, err error) {
    doc, err := goquery.NewDocumentFromReader(bytes.NewReader(r))
    if err != nil {
        return nil, err
    }
    results = make([]*SearchResult, 0)
    doc.Find(".xpd").Has("a>h3").Each(func(i int, s *goquery.Selection) {
        var (
            title   string
            link    string
            content string
        )
        result := &SearchResult{}
        // 返回的数组是提取的多个关键字部分
        result.Content = make([]string, 0)
        // 标题
        title = s.Find("a>h3").First().Text()
        // 内容
        contents := make([]string, 0)
        // 提取解析内容
        s.Find("div").Each(func(i int, selection *goquery.Selection) {
            contents = append(contents, selection.Text())
        })
        content = strings.Join(contents, ",")
        if o.textExtractionSyntax != nil {
            extractionContent := g.textExtract(ctx, content, *o.textExtractionSyntax, o.textExtractionIndex)
            result.Content = extractionContent
        } else {
            result.Content = []string{content}
        }
        // 提取解析 link
        originLink := s.Find("a").First().AttrOr("href", "")
        regx := regexp.MustCompile(`(?U)^.*/url\?q=.*(?P<link>.*)&sa=.*`)
        extractLink := regx.FindStringSubmatch(originLink)
        if len(extractLink) >= 2 {
            link = extractLink[1]
        }
        result.Title = title
        result.Url = link
        results = append(results, result)
    })

    return results, nil
}

func (g *googleSearchEngine) sendGoogleRequest(ctx context.Context, o *options, pageNum, pageSize uint32, query string, uule string) (ret []byte, err error) {
    params := map[string]string{
        "newwindow": "1",
        "gbv":       "1",
        "num":       fmt.Sprintf("%d", pageSize),
        "start":     fmt.Sprintf("%d", pageNum-1),
        "ie":        "UTF-8",
        "oe":        "UTF-8",
        "hl":        "en-US",
        "q":         query,
    }
    //uule为空表示搜全球
    if uule != "" {
        params["uule"] = uule
    }
    req, err := newHttpGetRequest(baseUrl, params)
    if err != nil {
        return nil, err
    }
    headers := []string{ggenerator.USER_AGENT}
    headerMap := ggenerator.HeaderGenerator(headers)
    for headerKey, headerValue := range headerMap {
        req.Header.Add(headerKey, headerValue)
    }
    g.log.WithContext(ctx).Infof("get google search:%s, header: %v, request parameter:%v",
        baseUrl, headerMap, params)
    begin := time.Now()
    resp, err := getHttpClient(g.nProxyClient, o.isFreeProxy).Do(req)
    if err != nil {
        return nil, err
    }
    defer resp.Body.Close()
    respBody, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return nil, err
    }
    if resp.StatusCode != http.StatusOK {
        g.log.WithContext(ctx).Infof("get google search:%s， response:%s",
            resp.Request.URL, string(respBody))
        err = Err
    }
    // 监控
    setMonitor(o.metricKey, begin, err)
    if err != nil {
        return nil, err
    }

    return respBody, nil
}

func (g *googleSearchEngine) Search(ctx context.Context, pageNum, pageSize uint32, query, uule string, opts ...Option) (results []*SearchResult, err error) {
    o := &options{
        isFreeProxy: true,
    }
    // 执行 option
    for _, opt := range opts {
        opt(o)
    }
    body, err := g.sendGoogleRequest(ctx, o, pageNum, pageSize, query, uule)
    if err != nil {
        return nil, err
    }

    results, err = g.parseGoogleResponse(ctx, o, body)
    if err != nil {
        return nil, err
    }

    return results, nil
}
  • iface.go
package ggsearchengine

import "context"

type SearchEngine interface {
    Search(ctx context.Context, pageNum, pageSize uint32, query, uule string, options ...Option) (results []*SearchResult, err error)
}
  • native_client.go
package ggsearchengine

import (
    "net/http"
    "net/url"

    "github.com/go-kratos/kratos/v2/log"
)

type NoPoolProxyHttpClient struct {
    Client        *http.Client
    PremiumClient *http.Client
}

func NewNoPoolProxyHttpClient(bc *Bootstrap, logger log.Logger) *NoPoolProxyHttpClient {
    httpConf := bc.Client.Http
    httpTransport := http.DefaultTransport.(*http.Transport).Clone()
    premiumHttpTransport := http.DefaultTransport.(*http.Transport).Clone()
    // 代理设置
    if bc.Client.Http.Proxy != nil {
        httpTransport.Proxy = setProxy(*bc.Client.Http.Proxy)
    }
    if bc.Client.Http.PremiumProxy != nil {
        premiumHttpTransport.Proxy = setProxy(*bc.Client.Http.PremiumProxy)
    }
    // 必须关闭http keep-alive 设置
    httpTransport.DisableKeepAlives = true
    premiumHttpTransport.DisableKeepAlives = true
    noPoolClient := &NoPoolProxyHttpClient{
        Client: &http.Client{
            Transport: httpTransport,
            Timeout:   httpConf.ProxyTimeout.AsDuration(),
        },
        PremiumClient: &http.Client{
            Transport: premiumHttpTransport,
            Timeout:   httpConf.PremiumProxyTimeout.AsDuration(),
        },
    }
    return noPoolClient
}

func setProxy(p string) func(*http.Request) (*url.URL, error) {
    return func(req *http.Request) (*url.URL, error) {
        iurl := url.URL{}
        return iurl.Parse(p)
    }
}
  • types.go
package ggsearchengine

import (
    "net/http"
    "net/url"
    "strings"
    "time"
)

type SearchResult struct {
    Title   string
    Content []string
    Url     string
}

type Option func(*options)

type options struct {
    metricKey            *string
    isFreeProxy          bool
    textExtractionSyntax *string
    textExtractionIndex  []int
}

func newHttpGetRequest(baseUrl string, params map[string]string) (*http.Request, error) {
    var urlBuilder strings.Builder
    urlBuilder.WriteString(baseUrl)
    urlBuilder.WriteString("?")
    flag := true
    for k, v := range params {
        if flag {
            flag = false
        } else {
            urlBuilder.WriteString("&")
        }
        urlBuilder.WriteString(k)
        urlBuilder.WriteString("=")
        urlBuilder.WriteString(url.QueryEscape(v))
    }
    return http.NewRequest("GET", urlBuilder.String(), nil)
}

// 设置监控
// 此处为公司敏感信息，请自行根据prometheus exporter 和 metric 实现
func setMonitor(metricKey *string, begin time.Time, err error) {
    if metricKey != nil {
        // 上报耗时
        if err != nil {
            // 上报错误计数
        } else {
            // 上报成功计数
        }
    }
}

func getHttpClient(nProxyClient *NoPoolProxyHttpClient, isFreeProxy bool) *http.Client {
    var httpClient *http.Client
    if isFreeProxy {
        httpClient = nProxyClient.Client
    } else {
        httpClient = nProxyClient.PremiumClient
    }

    return httpClient
}
  • useragent/data.go
package ggenerator

const (
    USER_AGENT = "User-Agent"
)

var userAgentList = []string{
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36 Edg/100.0.1185.29",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.109 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 OPR/85.0.4341.39",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
    "Mozilla/5.0 (Linux; U; Android 9; itel W6004 Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/79.0.3945.116 Mobile Safari/537.36 OPR/55.0.2254.56695",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36",
    "Mozilla/5.0 (X11; CrOS x86_64 14268.67.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.111 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36 Edg/98.0.1108.43",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36",
    "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Mobile Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.39",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
    "Mozilla/5.0 (Linux; U; Android 6.0.1; SM-N910C Build/MMB29K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/96.0.4664.45 Mobile Safari/537.36 OPR/62.1.2254.60552",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.62",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 15_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.30",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36",
    "Mozilla/5.0 (Linux; U; Android 10; TECNO KD6a Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/80.0.3987.99 Mobile Safari/537.36 OPR/62.3.2254.60988",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.109 Safari/537.36",
}
  • useragent/useragent_generator.go
package ggenerator

import (
    "math/rand"
    "time"
)

func RandomRange(min, max int) int {
    rand.Seed(time.Now().UnixNano())
    return rand.Intn(max+1-min) + min
}

func randomSelectElements(selectableSet []string) string {
    randomIndex := RandomRange(0, len(selectableSet)-1)
    return selectableSet[randomIndex]
}

func userAgentHandler() string {
    userAgentValue := randomSelectElements(userAgentList)

    return userAgentValue
}

func HeaderGenerator(headers []string) map[string]string {
    headerMap := make(map[string]string)

    for _, header := range headers {
        switch header {
        case USER_AGENT:
            userAgentValue := userAgentHandler()
            headerMap[USER_AGENT] = userAgentValue
        default:
            break
        }
    }

    return headerMap
}

关于 uule

https://site-analyzer.pro/services-seo/uule/

  • uule.proto
syntax = "proto3";
package kratos.api;

option go_package = "uule";

message Uule {
  int32 role = 1;
  int32 producer = 2;
  string canonical_name =4;
}
最后编辑于 ：
©著作权归作者所有,转载或内容合作请联系作者
  • 人面猴
    序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
    沈念sama阅读 206,968评论 6赞 482
  • 死咒
    序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
    沈念sama阅读 88,601评论 2赞 382
  • 救了他两次的神仙让他今天三更去死
    文/潘晓璐 我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
    开封第一讲书人阅读 153,220评论 0赞 344
  • 道士缉凶录：失踪的卖姜人
    文/不坏的土叔 我叫张陵，是天一观的道长。 经常有香客问我，道长，这世上最难降的妖魔是什么？ 我笑而不...
    开封第一讲书人阅读 55,416评论 1赞 279
  • 港岛之恋（遗憾婚礼）
    正文 为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
    茶点故事阅读 64,425评论 5赞 374
  • 恶毒庶女顶嫁案：这布局不是一般人想出来的
    文/花漫 我一把揭开白布。 她就那样静静地躺着，像睡着了一般。 火红的嫁衣衬着肌肤如雪。 梳的纹丝不乱的头发上，一...
    开封第一讲书人阅读 49,144评论 1赞 285
  • 城市分裂传说
    那天，我揣着相机与录音，去河边找鬼。 笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。 我是一名探鬼主播，决...
    沈念sama阅读 38,432评论 3赞 401
  • 双鸳鸯连环套：你想象不到人心有多黑
    文/苍兰香墨 我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
    开封第一讲书人阅读 37,088评论 0赞 261
  • 万荣杀人案实录
    序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
    沈念sama阅读 43,586评论 1赞 300
  • 护林员之死
    正文 独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋 以下内容为张勋视角 年9月15日...
    茶点故事阅读 36,028评论 2赞 325
  • 白月光启示录
    正文 我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。 大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
    茶点故事阅读 38,137评论 1赞 334
  • 活死人
    序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
    沈念sama阅读 33,783评论 4赞 324
  • 日本核电站爆炸内幕
    正文 年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
    茶点故事阅读 39,343评论 3赞 307
  • 男人毒药：我在死后第九天来索命
    文/蒙蒙 一、第九天 我趴在偏房一处隐蔽的房顶上张望。 院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
    开封第一讲书人阅读 30,333评论 0赞 19
  • 一桩弑父案，背后竟有这般阴谋
    文/苍兰香墨 我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。 一阵脚步声响...
    开封第一讲书人阅读 31,559评论 1赞 262
  • 情欲美人皮
    我被黑心中介骗来泰国打工， 没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。 一个月前我还...
    沈念sama阅读 45,595评论 2赞 355
  • 代替公主和亲
    正文 我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。 传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
    茶点故事阅读 42,901评论 2赞 345