proxypool

package module

v0.0.0-...-7ebde7b Latest Latest Go to latest Published: Jul 2, 2021 License: Apache-2.0 Imports: 18 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/crawlerclub/proxypool

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func AcquireProxy(addr string) error
func CrawlProxy(exitCh chan bool, wg *sync.WaitGroup)
func GetMySQLHandler() *gorm.DB
func InsertProxy(proxy *Proxy) error
func InsertProxyStr(p string) error
func InvalidProxy(addr string) error
func Run(exitCh chan bool, wg *sync.WaitGroup)
func Validate(addr string) bool
type Crawler
- func NewCrawler(conf string) *Crawler
- func (c *Crawler) Crawl() []string
type Proxy
- func ReadProxy() (records []*Proxy, err error)
- func (Proxy) TableName() string
type ProxyServer

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	DiggerUrl        = "http://www.site-digger.com/html/articles/20110516/proxieslist.html"
	DiggerPattern    = `<td><script>document.write\(decrypt\("(.*)"\)\);</script></td>`
	ReDigger         = regexp.MustCompile(DiggerPattern)
	DiggerKeyPattern = `var baidu_union_id = "(.+)";`
	ReDiggerKey      = regexp.MustCompile(DiggerKeyPattern)
)

View Source

var NameFuncs = make(map[string]func() []string)

Functions ¶

func AcquireProxy ¶

func AcquireProxy(addr string) error

func CrawlProxy ¶

func CrawlProxy(exitCh chan bool, wg *sync.WaitGroup)

func GetMySQLHandler ¶

func GetMySQLHandler() *gorm.DB

func InsertProxy ¶

func InsertProxy(proxy *Proxy) error

func InsertProxyStr ¶

func InsertProxyStr(p string) error

func InvalidProxy ¶

func InvalidProxy(addr string) error

func Run ¶

func Run(exitCh chan bool, wg *sync.WaitGroup)

func Validate ¶

func Validate(addr string) bool

Types ¶

type Crawler ¶

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler ¶

func NewCrawler(conf string) *Crawler

func (*Crawler) Crawl ¶

func (c *Crawler) Crawl() []string

type Proxy ¶

type Proxy struct {
	Id           uint64    `json:"id" gorm:"id"`
	IpPort       string    `json:"ip_port" gorm:"ip_port"`
	UpdateTime   time.Time `json:"update_time" gorm:"update_time"`
	LastFailTime time.Time `json:"last_fail_time" gorm:"last_fail_time"`
	TotalCrawl   uint64    `json:"total_crawl" gorm:"total_crawl"`
	TotalFail    uint64    `json:"total_fail" gorm:"total_fail"`
}

func ReadProxy ¶

func ReadProxy() (records []*Proxy, err error)

func (Proxy) TableName ¶

func (Proxy) TableName() string

type ProxyServer ¶

type ProxyServer struct {
	sync.RWMutex
	ProxyList []*Proxy
	N         int
}

func (*ProxyServer) Get ¶

func (s *ProxyServer) Get() string

func (*ProxyServer) ReadProxy ¶

func (s *ProxyServer) ReadProxy()

func (*ProxyServer) Run ¶

func (s *ProxyServer) Run(exitCh chan bool, wg *sync.WaitGroup)

func (*ProxyServer) Web ¶

func (s *ProxyServer) Web()

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL