predator

package module

Go to main page

Versions in this module

v0

v0.2.4

Feb 28, 2022

Changes in this version

+ const QueryParam

+ const RequestBodyParam

+ var ErrInvalidCacheTypeCode = errors.New("invalid cache type code")

+ var ErrNotAllowedCacheFieldType = errors.New("only query parameters are allowed as cached fields in `GET` requests")

+ type CacheField struct

+ Field string

+ func NewQueryParamField(field string) CacheField

+ func NewRequestBodyParamField(field string) CacheField

+ func (cf CacheField) String() string

type Crawler

+ func (c *Crawler) ParseJSON(strict bool, f HandleJSON)

+ func (c *Crawler) SetCache(cc Cache, compressed bool, cacheCondition CacheCondition, ...)

+ type HandleJSON func(j json.JSONResult)

+ type JSONParser struct

+ Handle HandleJSON

type Response

+ func (r *Response) BodyGunzip() ([]byte, error)

+ func (r *Response) Invalidate()

+ func (r *Response) Unmarshal(cachedBody []byte) error

v0.2.3

Dec 3, 2021

Changes in this version

+ type Cache interface

+ Cache func(key string, val []byte) error

+ Clear func() error

+ Compressed func(yes bool)

+ Init func() error

+ IsCached func(key string) ([]byte, bool)

+ type CacheModel struct

+ Key string

+ Value []byte

+ func (CacheModel) TableName() string

v0.2.2

Nov 22, 2021

Changes in this version

+ var ErrRequestFailed = errors.New("request failed")

+ var ErrTimeout = errors.New(...)

+ var ErrUnkownType = errors.New("recover only allows error type, but an unknown type is received")

type Crawler

+ func (c *Crawler) SetConcurrency(count uint64, blockPanic bool)

+ func (c *Crawler) SetRetry(count uint32, cond RetryConditions)

+ func (c Crawler) ProxyInUse() string

type Response

+ func (r Response) IsTimeout() bool

v0.2.1

Nov 9, 2021

v0.2.0

Nov 9, 2021

Changes in this version

+ type CacheCondition func(r Response) bool

+ type ComplementProxyPool func() []string

type Crawler

+ func (c *Crawler) AddProxy(newProxy string)

+ func (c *Crawler) Debug(msg string, args ...map[string]interface{})

+ func (c *Crawler) Fatal(err error, args ...map[string]interface{})

+ func (c *Crawler) FatalOrPanic(err error)

+ func (c *Crawler) GetWithCtx(URL string, ctx pctx.Context) error

+ func (c *Crawler) Info(msg string, args ...map[string]interface{})

+ func (c *Crawler) ProxyDialerWithTimeout(proxyAddr string, timeout time.Duration) fasthttp.DialFunc

+ func (c *Crawler) SetProxyInvalidCondition(condition ProxyInvalidCondition)

+ func (c *Crawler) Warning(msg string, args ...map[string]interface{})

type CrawlerOption

+ func EnableIPv6() CrawlerOption

+ func WithComplementProxyPool(f ComplementProxyPool) CrawlerOption

+ func WithDefaultCache() CrawlerOption

+ func WithDefaultLogger() CrawlerOption

+ type ProxyInvalidCondition func(r Response) error

type Request

+ func (r Request) GetWithCache(URL string, cacheFields ...string) error

+ func (r Request) PostJSON(URL string, requestData map[string]interface{}) error

+ func (r Request) PostJSONWithCache(URL string, requestData map[string]interface{}, cacheFields ...string) error

+ func (r Request) PostMultipart(URL string, form *MultipartForm) error

+ func (r Request) PostMultipartWithCache(URL string, form *MultipartForm, cacheFields ...string) error

+ func (r Request) PostWithCache(URL string, requestData map[string]string, cacheFields ...string) error

+ func (r Request) Request(method, URL string, cachedMap map[string]string, body []byte) error

type Response

+ func (r Response) ClientIP() net.Addr

+ func (r Response) LocalIP() net.Addr

v0.1.9

Oct 21, 2021

Changes in this version

type Request

+ func (r *Request) SetTimeout(t time.Duration)

+ func (r Request) Visit(method, URL string, cachedMap map[string]string, body []byte) error

v0.1.8

Oct 12, 2021

Changes in this version

+ const RUNNING

+ const STOPED

+ var ErrEmptyProxyPool = errors.New("after deleting the invalid proxy, the current proxy ip pool is empty")

+ var ErrIncorrectResponse = errors.New("the response status code is not 200 or 201")

+ var ErrInvalidPoolCap = errors.New("invalid pool cap")

+ var ErrInvalidProxy = errors.New("the proxy ip should contain the protocol")

+ var ErrNoCacheSet = errors.New("no cache set")

+ var ErrOnlyOneProxyIP = errors.New("unable to delete the only proxy ip")

+ var ErrPoolAlreadyClosed = errors.New("pool already closed")

+ var ErrProxyExpired = errors.New("the proxy ip has expired")

+ var ErrUnknownProtocol = errors.New("only support http and socks5 protocol")

+ var ErrUnkownProxyIP = errors.New("proxy is unkown")

+ func ReleaseRequest(req *Request)

+ func ReleaseResponse(resp *Response)

+ type AcquireProxies func(n int) []string

+ type Crawler struct

+ Context context.Context

+ UserAgent string

+ func NewCrawler(opts ...CrawlerOption) *Crawler

+ func (c *Crawler) AfterResponse(f HandleResponse)

+ func (c *Crawler) BeforeRequest(f HandleRequest)

+ func (c *Crawler) ClearCache() error

+ func (c *Crawler) Clone() *Crawler

+ func (c *Crawler) DialWithProxy() fasthttp.DialFunc

+ func (c *Crawler) DialWithProxyAndTimeout(timeout time.Duration) fasthttp.DialFunc

+ func (c *Crawler) Error(err error)

+ func (c *Crawler) Get(URL string) error

+ func (c *Crawler) ParseHTML(selector string, f HandleHTML)

+ func (c *Crawler) Post(URL string, requestData map[string]string, ctx pctx.Context) error

+ func (c *Crawler) PostJSON(URL string, requestData map[string]interface{}, ctx pctx.Context) error

+ func (c *Crawler) PostMultipart(URL string, form *MultipartForm, ctx pctx.Context) error

+ func (c *Crawler) PostRaw(URL string, body []byte, ctx pctx.Context) error

+ func (c *Crawler) Wait()

+ func (c Crawler) ProxyPoolAmount() int

+ type CrawlerOption func(*Crawler)

+ func SkipVerification() CrawlerOption

+ func WithCache(cc cache.Cache, compressed bool, cacheFileds ...string) CrawlerOption

+ func WithConcurrency(count uint64) CrawlerOption

+ func WithCookies(cookies map[string]string) CrawlerOption

+ func WithLogger(lop *LogOp) CrawlerOption

+ func WithProxy(proxyURL string) CrawlerOption

+ func WithProxyPool(proxyURLs []string) CrawlerOption

+ func WithRawCookie(cookie string) CrawlerOption

+ func WithRetry(count uint32, cond RetryConditions) CrawlerOption

+ func WithUserAgent(ua string) CrawlerOption

+ type CustomRandomBoundary func() string

+ type HTMLParser struct

+ Handle HandleHTML

+ Selector string

+ type HandleHTML func(he *html.HTMLElement, r *Response)

+ type HandleRequest func(r *Request)

+ type HandleResponse func(r *Response)

+ type LogOp struct

+ Out io.Writer

+ func (l *LogOp) SetLevel(level zerolog.Level)

+ func (l *LogOp) ToConsole()

+ func (l *LogOp) ToConsoleAndFile(filepath string) error

+ func (l *LogOp) ToFile(filepath string) error

+ type MultipartForm struct

+ func NewMultipartForm(dash string, f CustomRandomBoundary) *MultipartForm

+ func (mf *MultipartForm) AppendFile(name, filePath string) error

+ func (mf *MultipartForm) AppendString(name, value string)

+ func (mf *MultipartForm) Boundary() string

+ func (mf *MultipartForm) Bytes() []byte

+ func (mf *MultipartForm) FormDataContentType() string

+ type Pool struct

+ func NewPool(capacity uint64) (*Pool, error)

+ func (p *Pool) Close()

+ func (p *Pool) GetCap() uint64

+ func (p *Pool) GetRunningWorkers() uint64

+ func (p *Pool) Put(task *Task) error

+ type Request struct

+ Body []byte

+ Ctx pctx.Context

+ Headers *fasthttp.RequestHeader

+ ID uint32

+ Method string

+ URL string

+ func AcquireRequest() *Request

+ func (r *Request) Abort()

+ func (r *Request) AllowRedirect(maxRedirectsCount uint)

+ func (r *Request) New(method, URL string, body []byte) *Request

+ func (r *Request) Reset()

+ func (r *Request) SetContentType(contentType string)

+ func (r *Request) SetHeaders(headers map[string]string)

+ func (r Request) AbsoluteURL(src string) string

+ func (r Request) Get(u string) error

+ func (r Request) Hash() (string, error)

+ func (r Request) NumberOfRetries() uint32

+ func (r Request) Post(URL string, requestData map[string]string, ctx pctx.Context) error

+ type Response struct

+ Body []byte

+ Ctx ctx.Context

+ FromCache bool

+ Headers fasthttp.ResponseHeader

+ Request *Request

+ StatusCode int

+ func AcquireResponse() *Response

+ func (r *Response) ContentType() string

+ func (r *Response) GetSetCookie() string

+ func (r *Response) Reset()

+ func (r *Response) Save(fileName string) error

+ func (r *Response) String() string

+ func (r Response) Marshal() ([]byte, error)

+ type RetryConditions func(r Response) bool

+ type Task struct

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL