fastcolly

package module

Go to main page

Versions in this module

v1

v1.3.7

Mar 16, 2020

v1.3.6

Mar 16, 2020

v1.3.5

Feb 10, 2020

v1.3.4

Feb 9, 2020

v1.3.3

Feb 3, 2020

v1.3.2

Feb 3, 2020

v1.3.1

Jan 22, 2020

v1.3.0

Dec 24, 2019

Changes in this version

+ var ErrEmptyProxyURL = errors.New("Proxy URL list is empty")

+ func B2S(b []byte) string

+ func S2B(s string) (b []byte)

type Collector

+ LimitRules []*LimitRule

+ func (c *Collector) GetMatchingRule(domain string) *LimitRule

+ type CollectorOption func(*Collector)

type HTMLElement

+ func (h *HTMLElement) UnmarshalWithMap(v interface{}, structMap map[string]string) error

v1.2.0

Feb 13, 2019

Changes in this version

+ const ProxyURLKey

type Collector

+ CheckHead bool

+ func (c *Collector) Head(URL string) error

type Request

+ ProxyURL string

v1.1.0

Aug 28, 2018

Changes in this version

type HTMLElement

+ Index int

+ func (h *HTMLElement) ForEachWithBreak(goquerySelector string, callback func(int, *HTMLElement) bool)

type XMLElement

+ func (h *XMLElement) ChildTexts(xpathQuery string) []string

v1.0.0

May 12, 2018

Changes in this version

+ var ErrAlreadyVisited = errors.New("URL already visited")

+ var ErrForbiddenDomain = errors.New("Forbidden domain")

+ var ErrForbiddenURL = errors.New("ForbiddenURL")

+ var ErrMaxDepth = errors.New("Max depth limit reached")

+ var ErrMissingURL = errors.New("Missing URL")

+ var ErrNoCookieJar = errors.New("Cookie jar is not available")

+ var ErrNoPattern = errors.New("No pattern defined in LimitRule")

+ var ErrNoURLFiltersMatch = errors.New("No URLFilters match")

+ var ErrRobotsTxtBlocked = errors.New("URL blocked by robots.txt")

+ func AllowURLRevisit() func(*Collector)

+ func AllowedDomains(domains ...string) func(*Collector)

+ func Async(a bool) func(*Collector)

+ func CacheDir(path string) func(*Collector)

+ func Debugger(d debug.Debugger) func(*Collector)

+ func DetectCharset() func(*Collector)

+ func DisallowedDomains(domains ...string) func(*Collector)

+ func DisallowedURLFilters(filters ...*regexp.Regexp) func(*Collector)

+ func ID(id uint32) func(*Collector)

+ func IgnoreRobotsTxt() func(*Collector)

+ func MaxBodySize(sizeInBytes int) func(*Collector)

+ func MaxDepth(depth int) func(*Collector)

+ func ParseHTTPErrorResponse() func(*Collector)

+ func SanitizeFileName(fileName string) string

+ func URLFilters(filters ...*regexp.Regexp) func(*Collector)

+ func UnmarshalHTML(v interface{}, s *goquery.Selection) error

+ func UserAgent(ua string) func(*Collector)

+ type Collector struct

+ AllowURLRevisit bool

+ AllowedDomains []string

+ Async bool

+ CacheDir string

+ DetectCharset bool

+ DisallowedDomains []string

+ DisallowedURLFilters []*regexp.Regexp

+ ID uint32

+ IgnoreRobotsTxt bool

+ MaxBodySize int

+ MaxDepth int

+ ParseHTTPErrorResponse bool

+ RedirectHandler func(req *http.Request, via []*http.Request) error

+ URLFilters []*regexp.Regexp

+ UserAgent string

+ func NewCollector(options ...func(*Collector)) *Collector

+ func (c *Collector) Appengine(req *http.Request)

+ func (c *Collector) Clone() *Collector

+ func (c *Collector) Cookies(URL string) []*http.Cookie

+ func (c *Collector) DisableCookies()

+ func (c *Collector) Init()

+ func (c *Collector) Limit(rule *LimitRule) error

+ func (c *Collector) Limits(rules []*LimitRule) error

+ func (c *Collector) OnError(f ErrorCallback)

+ func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback)

+ func (c *Collector) OnHTMLDetach(goquerySelector string)

+ func (c *Collector) OnRequest(f RequestCallback)

+ func (c *Collector) OnResponse(f ResponseCallback)

+ func (c *Collector) OnScraped(f ScrapedCallback)

+ func (c *Collector) OnXML(xpathQuery string, f XMLCallback)

+ func (c *Collector) OnXMLDetach(xpathQuery string)

+ func (c *Collector) Post(URL string, requestData map[string]string) error

+ func (c *Collector) PostMultipart(URL string, requestData map[string][]byte) error

+ func (c *Collector) PostRaw(URL string, requestData []byte) error

+ func (c *Collector) Request(method, URL string, requestData io.Reader, ctx *Context, hdr http.Header) error

+ func (c *Collector) SetCookieJar(j *cookiejar.Jar)

+ func (c *Collector) SetCookies(URL string, cookies []*http.Cookie) error

+ func (c *Collector) SetDebugger(d debug.Debugger)

+ func (c *Collector) SetProxy(proxyURL string) error

+ func (c *Collector) SetProxyFunc(p ProxyFunc)

+ func (c *Collector) SetRequestTimeout(timeout time.Duration)

+ func (c *Collector) SetStorage(s storage.Storage) error

+ func (c *Collector) String() string

+ func (c *Collector) UnmarshalRequest(r []byte) (*Request, error)

+ func (c *Collector) Visit(URL string) error

+ func (c *Collector) Wait()

+ func (c *Collector) WithTransport(transport http.RoundTripper)

+ type Context struct

+ func NewContext() *Context

+ func (c *Context) ForEach(fn func(k string, v interface{}) interface{}) []interface{}

+ func (c *Context) Get(key string) string

+ func (c *Context) GetAny(key string) interface{}

+ func (c *Context) MarshalBinary() (_ []byte, _ error)

+ func (c *Context) Put(key string, value interface{})

+ func (c *Context) UnmarshalBinary(_ []byte) error

+ type ErrorCallback func(*Response, error)

+ type HTMLCallback func(*HTMLElement)

+ type HTMLElement struct

+ DOM *goquery.Selection

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node) *HTMLElement

+ func (h *HTMLElement) Attr(k string) string

+ func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string

+ func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string

+ func (h *HTMLElement) ChildText(goquerySelector string) string

+ func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement))

+ func (h *HTMLElement) Unmarshal(v interface{}) error

+ type LimitRule struct

+ Delay time.Duration

+ DomainGlob string

+ DomainRegexp string

+ Parallelism int

+ RandomDelay time.Duration

+ func (r *LimitRule) Init() error

+ func (r *LimitRule) Match(domain string) bool

+ type ProxyFunc func(*http.Request) (*url.URL, error)

+ type Request struct

+ Body io.Reader

+ Ctx *Context

+ Depth int

+ Headers *http.Header

+ ID uint32

+ Method string

+ ResponseCharacterEncoding string

+ URL *url.URL

+ func (r *Request) Abort()

+ func (r *Request) AbsoluteURL(u string) string

+ func (r *Request) Do() error

+ func (r *Request) Marshal() ([]byte, error)

+ func (r *Request) New(method, URL string, body io.Reader) (*Request, error)

+ func (r *Request) Post(URL string, requestData map[string]string) error

+ func (r *Request) PostMultipart(URL string, requestData map[string][]byte) error

+ func (r *Request) PostRaw(URL string, requestData []byte) error

+ func (r *Request) Retry() error

+ func (r *Request) Visit(URL string) error

+ type RequestCallback func(*Request)

+ type Response struct

+ Body []byte

+ Ctx *Context

+ Headers *http.Header

+ Request *Request

+ StatusCode int

+ func (r *Response) FileName() string

+ func (r *Response) Save(fileName string) error

+ type ResponseCallback func(*Response)

+ type ScrapedCallback func(*Response)

+ type XMLCallback func(*XMLElement)

+ type XMLElement struct

+ DOM interface{}

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewXMLElementFromHTMLNode(resp *Response, s *html.Node) *XMLElement

+ func NewXMLElementFromXMLNode(resp *Response, s *xmlquery.Node) *XMLElement

+ func (h *XMLElement) Attr(k string) string

+ func (h *XMLElement) ChildAttr(xpathQuery, attrName string) string

+ func (h *XMLElement) ChildAttrs(xpathQuery, attrName string) []string

+ func (h *XMLElement) ChildText(xpathQuery string) string

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL