colly

package module

Go to main page

Versions in this module

v1

v1.2.0

Feb 13, 2019

v1.1.7

Oct 9, 2023

Changes in this version

+ func SetHTTPDriver(backend HTTPDriver) func(*Collector)

+ type CheckHeadersFunc func(statusCode int, header http.Header) bool

+ type HTTPDriver interface

+ Cache func(request *http.Request, bodySize int, checkHeadersFunc CheckHeadersFunc, ...) (*Response, error)

+ CheckRedirect func(func(req *http.Request, via []*http.Request) error)

+ Cookies func(url *url.URL) []*http.Cookie

+ Do func(request *http.Request, bodySize int, checkHeadersFunc CheckHeadersFunc) (*Response, error)

+ GetJar func() http.CookieJar

+ GetMatchingRule func(domain string) *LimitRule

+ GetTimeout func() time.Duration

+ Init func(jar http.CookieJar)

+ Jar func(j http.CookieJar)

+ Limit func(rule *LimitRule) error

+ Limits func(rules []*LimitRule) error

+ Proxy func(pf ProxyFunc)

+ SetClient func(client *http.Client)

+ SetCookies func(url *url.URL, cookies []*http.Cookie) error

+ Timeout func(t time.Duration)

+ Transport func(t http.RoundTripper)

type LimitRule

+ WaitChan chan bool

v1.1.4

Oct 9, 2023

Changes in this version

+ var ErrAbortedAfterHeaders = errors.New("Aborted after receiving response headers")

+ var ErrEmptyProxyURL = errors.New("Proxy URL list is empty")

+ var ErrMaxRequests = errors.New("Max Requests limit reached")

+ var ErrQueueFull = errors.New("Queue MaxSize reached")

+ type AlreadyVisitedError struct

+ Destination *url.URL

+ func (e *AlreadyVisitedError) Error() string

type Collector

+ CheckHead bool

+ Context context.Context

+ Headers *http.Header

+ MaxRequests uint32

+ TraceHTTP bool

+ func (c *Collector) HasPosted(URL string, requestData map[string]string) (bool, error)

+ func (c *Collector) HasVisited(URL string) (bool, error)

+ func (c *Collector) Head(URL string) error

+ func (c *Collector) OnResponseHeaders(f ResponseHeadersCallback)

+ func (c *Collector) SetClient(client *http.Client)

+ func (c *Collector) SetRedirectHandler(f func(req *http.Request, via []*http.Request) error)

+ func (c *Collector) UnmarshalRequestWithContext(ctx *context.Context, r []byte) (*Request, error)

+ type CollectorOption func(*Collector)

+ func CheckHead() CollectorOption

+ func Headers(headers map[string]string) CollectorOption

+ func MaxRequests(max uint32) CollectorOption

+ func StdlibContext(ctx context.Context) CollectorOption

+ func TraceHTTP() CollectorOption

type HTMLElement

+ func (h *HTMLElement) ChildTexts(goquerySelector string) []string

+ func (h *HTMLElement) UnmarshalWithMap(v interface{}, structMap map[string]string) error

+ type HTTPTrace struct

+ ConnectDuration time.Duration

+ FirstByteDuration time.Duration

+ func (ht *HTTPTrace) WithTrace(req *http.Request) *http.Request

type Request

+ Host string

+ func (r *Request) HasVisited(URL string) (bool, error)

type Response

+ Trace *HTTPTrace

+ type ResponseHeadersCallback func(*Response)

v1.1.2

Oct 9, 2023

Changes in this version

+ const ProxyURLKey

+ func WithTimingsContext(ctx context.Context) context.Context

type Collector

+ func (c *Collector) VisitWithContext(ctx context.Context, URL string) error

type Context

+ func ContextDataContext(ctx context.Context) *Context

+ func WithDataContext(ctx context.Context) (context.Context, *Context)

type Request

+ ProxyURL string

+ type Timings struct

+ CharsetFixEnd time.Time

+ CharsetFixStart time.Time

+ DownloadEnd time.Time

+ DownloadStart time.Time

+ ProcessEnd time.Time

+ ProcessStart time.Time

+ RequestStart time.Time

+ func ContextTimings(ctx context.Context) *Timings

v1.1.0

Aug 28, 2018

Changes in this version

type HTMLElement

+ Index int

+ func (h *HTMLElement) ForEachWithBreak(goquerySelector string, callback func(int, *HTMLElement) bool)

type XMLElement

+ func (h *XMLElement) ChildTexts(xpathQuery string) []string

v1.0.0

May 12, 2018

Changes in this version

+ var ErrAlreadyVisited = errors.New("URL already visited")

+ var ErrForbiddenDomain = errors.New("Forbidden domain")

+ var ErrForbiddenURL = errors.New("ForbiddenURL")

+ var ErrMaxDepth = errors.New("Max depth limit reached")

+ var ErrMissingURL = errors.New("Missing URL")

+ var ErrNoCookieJar = errors.New("Cookie jar is not available")

+ var ErrNoPattern = errors.New("No pattern defined in LimitRule")

+ var ErrNoURLFiltersMatch = errors.New("No URLFilters match")

+ var ErrRobotsTxtBlocked = errors.New("URL blocked by robots.txt")

+ func AllowURLRevisit() func(*Collector)

+ func AllowedDomains(domains ...string) func(*Collector)

+ func Async(a bool) func(*Collector)

+ func CacheDir(path string) func(*Collector)

+ func Debugger(d debug.Debugger) func(*Collector)

+ func DetectCharset() func(*Collector)

+ func DisallowedDomains(domains ...string) func(*Collector)

+ func DisallowedURLFilters(filters ...*regexp.Regexp) func(*Collector)

+ func ID(id uint32) func(*Collector)

+ func IgnoreRobotsTxt() func(*Collector)

+ func MaxBodySize(sizeInBytes int) func(*Collector)

+ func MaxDepth(depth int) func(*Collector)

+ func ParseHTTPErrorResponse() func(*Collector)

+ func SanitizeFileName(fileName string) string

+ func URLFilters(filters ...*regexp.Regexp) func(*Collector)

+ func UnmarshalHTML(v interface{}, s *goquery.Selection) error

+ func UserAgent(ua string) func(*Collector)

+ type Collector struct

+ AllowURLRevisit bool

+ AllowedDomains []string

+ Async bool

+ CacheDir string

+ DetectCharset bool

+ DisallowedDomains []string

+ DisallowedURLFilters []*regexp.Regexp

+ ID uint32

+ IgnoreRobotsTxt bool

+ MaxBodySize int

+ MaxDepth int

+ ParseHTTPErrorResponse bool

+ RedirectHandler func(req *http.Request, via []*http.Request) error

+ URLFilters []*regexp.Regexp

+ UserAgent string

+ func NewCollector(options ...func(*Collector)) *Collector

+ func (c *Collector) Appengine(req *http.Request)

+ func (c *Collector) Clone() *Collector

+ func (c *Collector) Cookies(URL string) []*http.Cookie

+ func (c *Collector) DisableCookies()

+ func (c *Collector) Init()

+ func (c *Collector) Limit(rule *LimitRule) error

+ func (c *Collector) Limits(rules []*LimitRule) error

+ func (c *Collector) OnError(f ErrorCallback)

+ func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback)

+ func (c *Collector) OnHTMLDetach(goquerySelector string)

+ func (c *Collector) OnRequest(f RequestCallback)

+ func (c *Collector) OnResponse(f ResponseCallback)

+ func (c *Collector) OnScraped(f ScrapedCallback)

+ func (c *Collector) OnXML(xpathQuery string, f XMLCallback)

+ func (c *Collector) OnXMLDetach(xpathQuery string)

+ func (c *Collector) Post(URL string, requestData map[string]string) error

+ func (c *Collector) PostMultipart(URL string, requestData map[string][]byte) error

+ func (c *Collector) PostRaw(URL string, requestData []byte) error

+ func (c *Collector) Request(method, URL string, requestData io.Reader, ctx *Context, hdr http.Header) error

+ func (c *Collector) SetCookieJar(j *cookiejar.Jar)

+ func (c *Collector) SetCookies(URL string, cookies []*http.Cookie) error

+ func (c *Collector) SetDebugger(d debug.Debugger)

+ func (c *Collector) SetProxy(proxyURL string) error

+ func (c *Collector) SetProxyFunc(p ProxyFunc)

+ func (c *Collector) SetRequestTimeout(timeout time.Duration)

+ func (c *Collector) SetStorage(s storage.Storage) error

+ func (c *Collector) String() string

+ func (c *Collector) UnmarshalRequest(r []byte) (*Request, error)

+ func (c *Collector) Visit(URL string) error

+ func (c *Collector) Wait()

+ func (c *Collector) WithTransport(transport http.RoundTripper)

+ type Context struct

+ func NewContext() *Context

+ func (c *Context) ForEach(fn func(k string, v interface{}) interface{}) []interface{}

+ func (c *Context) Get(key string) string

+ func (c *Context) GetAny(key string) interface{}

+ func (c *Context) MarshalBinary() (_ []byte, _ error)

+ func (c *Context) Put(key string, value interface{})

+ func (c *Context) UnmarshalBinary(_ []byte) error

+ type ErrorCallback func(*Response, error)

+ type HTMLCallback func(*HTMLElement)

+ type HTMLElement struct

+ DOM *goquery.Selection

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node) *HTMLElement

+ func (h *HTMLElement) Attr(k string) string

+ func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string

+ func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string

+ func (h *HTMLElement) ChildText(goquerySelector string) string

+ func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement))

+ func (h *HTMLElement) Unmarshal(v interface{}) error

+ type LimitRule struct

+ Delay time.Duration

+ DomainGlob string

+ DomainRegexp string

+ Parallelism int

+ RandomDelay time.Duration

+ func (r *LimitRule) Init() error

+ func (r *LimitRule) Match(domain string) bool

+ type ProxyFunc func(*http.Request) (*url.URL, error)

+ type Request struct

+ Body io.Reader

+ Ctx *Context

+ Depth int

+ Headers *http.Header

+ ID uint32

+ Method string

+ ResponseCharacterEncoding string

+ URL *url.URL

+ func (r *Request) Abort()

+ func (r *Request) AbsoluteURL(u string) string

+ func (r *Request) Do() error

+ func (r *Request) Marshal() ([]byte, error)

+ func (r *Request) New(method, URL string, body io.Reader) (*Request, error)

+ func (r *Request) Post(URL string, requestData map[string]string) error

+ func (r *Request) PostMultipart(URL string, requestData map[string][]byte) error

+ func (r *Request) PostRaw(URL string, requestData []byte) error

+ func (r *Request) Retry() error

+ func (r *Request) Visit(URL string) error

+ type RequestCallback func(*Request)

+ type Response struct

+ Body []byte

+ Ctx *Context

+ Headers *http.Header

+ Request *Request

+ StatusCode int

+ func (r *Response) FileName() string

+ func (r *Response) Save(fileName string) error

+ type ResponseCallback func(*Response)

+ type ScrapedCallback func(*Response)

+ type XMLCallback func(*XMLElement)

+ type XMLElement struct

+ DOM interface{}

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewXMLElementFromHTMLNode(resp *Response, s *html.Node) *XMLElement

+ func NewXMLElementFromXMLNode(resp *Response, s *xmlquery.Node) *XMLElement

+ func (h *XMLElement) Attr(k string) string

+ func (h *XMLElement) ChildAttr(xpathQuery, attrName string) string

+ func (h *XMLElement) ChildAttrs(xpathQuery, attrName string) []string

+ func (h *XMLElement) ChildText(xpathQuery string) string

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL