agents

package
v1.3.8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 1, 2024 License: BSD-3-Clause Imports: 22 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var AllAgents = []*Agent{
	NewAgent("baidu", []string{".baidu.com."}, nil, []string{"Baidu"}),
	NewAgent("google", []string{".googlebot.com."}, nil, []string{"Google"}),
	NewAgent("bing", []string{".search.msn.com."}, nil, []string{"bingbot"}),
	NewAgent("sogou", []string{".sogou.com."}, nil, []string{"Sogou"}),
	NewAgent("youdao", []string{".163.com."}, nil, []string{"Youdao"}),
	NewAgent("yahoo", []string{".yahoo.com."}, nil, []string{"Yahoo"}),
	NewAgent("bytedance", []string{".bytedance.com."}, nil, []string{"Bytespider"}),
	NewAgent("sm", []string{".sm.cn."}, nil, []string{"YisouSpider"}),
	NewAgent("yandex", []string{".yandex.com.", ".yndx.net."}, nil, []string{"Yandex"}),
	NewAgent("semrush", []string{".semrush.com."}, nil, []string{"SEMrush"}),
}
View Source 
var SharedManager = NewManager()
View Source 
var SharedQueue = NewQueue()

Functions

func IsAgentFromUserAgent 

func IsAgentFromUserAgent(userAgent string) bool

Types

type Agent 

type Agent struct {
	Code     string
	Keywords []string // user agent keywords
	// contains filtered or unexported fields
}

func NewAgent 

func NewAgent(code string, suffixes []string, reg *regexp.Regexp, keywords []string) *Agent

func (*Agent) Match 

func (this *Agent) Match(ptr string) bool

type AgentIP 

type AgentIP struct {
	Id        int64  `json:"id"`
	IP        string `json:"ip"`
	AgentCode string `json:"agentCode"`
}

type AgentIPEncoder added in v1.3.5 

type AgentIPEncoder[T interface{ *AgentIP }] struct {
}

func (*AgentIPEncoder[T]) Decode added in v1.3.5 

func (this *AgentIPEncoder[T]) Decode(valueBytes []byte) (value T, err error)

func (*AgentIPEncoder[T]) Encode added in v1.3.5 

func (this *AgentIPEncoder[T]) Encode(value T) ([]byte, error)

func (*AgentIPEncoder[T]) EncodeField added in v1.3.5 

func (this *AgentIPEncoder[T]) EncodeField(value T, fieldName string) ([]byte, error)

func (*AgentIPEncoder[T]) EncodeKey added in v1.3.5 

func (this *AgentIPEncoder[T]) EncodeKey(item *AgentIP) string

EncodeKey generate key for ip item

type DB 

type DB interface {
	Init() error
	InsertAgentIP(ipId int64, ip string, agentCode string) error
	ListAgentIPs(offset int64, size int64) (agentIPs []*AgentIP, err error)
}

type IPCacheMap 

type IPCacheMap struct {
	// contains filtered or unexported fields
}

func NewIPCacheMap 

func NewIPCacheMap(maxLen int) *IPCacheMap

func (*IPCacheMap) Add 

func (this *IPCacheMap) Add(ip string)

func (*IPCacheMap) Contains 

func (this *IPCacheMap) Contains(ip string) bool

type KVDB added in v1.3.5 

type KVDB struct {
	// contains filtered or unexported fields
}

func NewKVDB added in v1.3.5 

func NewKVDB() *KVDB

func (*KVDB) Close added in v1.3.5 

func (this *KVDB) Close() error

func (*KVDB) Flush added in v1.3.5 

func (this *KVDB) Flush() error

func (*KVDB) Init added in v1.3.5 

func (this *KVDB) Init() error

func (*KVDB) InsertAgentIP added in v1.3.5 

func (this *KVDB) InsertAgentIP(ipId int64, ip string, agentCode string) error

func (*KVDB) ListAgentIPs added in v1.3.5 

func (this *KVDB) ListAgentIPs(offset int64, size int64) (agentIPs []*AgentIP, err error)

type Manager 

type Manager struct {
	// contains filtered or unexported fields
}

Manager Agent管理器

func NewManager 

func NewManager() *Manager

func (*Manager) AddIP 

func (this *Manager) AddIP(ip string, agentCode string)

AddIP 添加记录

func (*Manager) ContainsIP 

func (this *Manager) ContainsIP(ip string) bool

ContainsIP 检查是否有IP相关数据

func (*Manager) Load 

func (this *Manager) Load() error

func (*Manager) LookupIP 

func (this *Manager) LookupIP(ip string) (agentCode string)

LookupIP 查询IP所属Agent

func (*Manager) Loop 

func (this *Manager) Loop() (hasNext bool, err error)

Loop 单次循环获取数据

func (*Manager) LoopAll 

func (this *Manager) LoopAll() error

func (*Manager) SetDB 

func (this *Manager) SetDB(db DB)

func (*Manager) Start 

func (this *Manager) Start()

type Queue 

type Queue struct {
	// contains filtered or unexported fields
}

func NewQueue 

func NewQueue() *Queue

func (*Queue) ParseIP 

func (this *Queue) ParseIP(ip string) (ptr string, err error)

ParseIP 分析IP的PTR值

func (*Queue) ParsePtr 

func (this *Queue) ParsePtr(ptr string) (agentCode string)

ParsePtr 分析PTR对应的Agent

func (*Queue) Process 

func (this *Queue) Process(ip string) error

Process 处理IP

func (*Queue) Push 

func (this *Queue) Push(ip string)

Push 将IP加入到处理队列

func (*Queue) Start 

func (this *Queue) Start()

type SQLiteDB added in v1.3.5 

type SQLiteDB struct {
	// contains filtered or unexported fields
}

func NewSQLiteDB added in v1.3.5 

func NewSQLiteDB(path string) *SQLiteDB

func (*SQLiteDB) Close added in v1.3.5 

func (this *SQLiteDB) Close() error

func (*SQLiteDB) Init added in v1.3.5 

func (this *SQLiteDB) Init() error

func (*SQLiteDB) InsertAgentIP added in v1.3.5 

func (this *SQLiteDB) InsertAgentIP(ipId int64, ip string, agentCode string) error

func (*SQLiteDB) ListAgentIPs added in v1.3.5 

func (this *SQLiteDB) ListAgentIPs(offset int64, size int64) (agentIPs []*AgentIP, err error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.