support

func (*DefaultCluster) AddNode ¶

func (this *DefaultCluster) AddNode(nodeInfo *node.NodeInfo)

add a node to cluster node list

func (*DefaultCluster) AddRequest ¶

func (this *DefaultCluster) AddRequest(request *http.Request)

add a request to quene

func (*DefaultCluster) AddToCrawlingQuene ¶

func (this *DefaultCluster) AddToCrawlingQuene(request *http.Request)

record distribute request job

func (*DefaultCluster) CanWeStopSpider ¶

func (this *DefaultCluster) CanWeStopSpider(spiderName string) bool

func (*DefaultCluster) CrawlStatus ¶

func (this *DefaultCluster) CrawlStatus() *crawler.CrawlerStatus

get crawl status

func (*DefaultCluster) Crawled ¶

func (this *DefaultCluster) Crawled(scrapyResult *crawler.ScrapeResult)

a request job is done delete it from crawling quene add crawled num

func (*DefaultCluster) DeleteDeadNode ¶

func (this *DefaultCluster) DeleteDeadNode(nodeName string)

func (*DefaultCluster) ElectMaster ¶

func (this *DefaultCluster) ElectMaster() *node.NodeInfo

choose a new master node

func (*DefaultCluster) GetAllNode ¶

func (this *DefaultCluster) GetAllNode() []*node.NodeInfo

func (*DefaultCluster) GetClusterInfo ¶

func (this *DefaultCluster) GetClusterInfo() *ClusterInfo

func (*DefaultCluster) GetMasterName ¶

func (this *DefaultCluster) GetMasterName() string

get master node name

func (*DefaultCluster) GetMasterNode ¶

func (this *DefaultCluster) GetMasterNode() *node.NodeInfo

get master node

func (*DefaultCluster) GetRequestStatus ¶

func (this *DefaultCluster) GetRequestStatus() *RequestStatus

func (*DefaultCluster) HasNode ¶

func (this *DefaultCluster) HasNode(nodeName string) bool

func (*DefaultCluster) Init ¶

func (this *DefaultCluster) Init(node *node.NodeInfo)

func (*DefaultCluster) IsMasterNode ¶

func (this *DefaultCluster) IsMasterNode() bool

is local node master node

func (*DefaultCluster) IsReady ¶

func (this *DefaultCluster) IsReady() bool

is cluster ready for crawl

func (*DefaultCluster) IsSpiderRunning ¶

func (this *DefaultCluster) IsSpiderRunning(spiderName string) bool

is the spider running

func (*DefaultCluster) IsStop ¶

func (this *DefaultCluster) IsStop() bool

is all loop stop

func (*DefaultCluster) Join ¶

func (this *DefaultCluster) Join()

func (*DefaultCluster) MakeMasterNode ¶

func (this *DefaultCluster) MakeMasterNode(nodeName string)

make master node by node name

func (*DefaultCluster) PopRequest ¶

func (this *DefaultCluster) PopRequest() *http.Request

pop a request from waiting quene add to crawling quenu

func (*DefaultCluster) Ready ¶

func (this *DefaultCluster) Ready()

func (*DefaultCluster) StartSpider ¶

func (this *DefaultCluster) StartSpider(spiderName string)

when start a spider ,cluster should record it

func (*DefaultCluster) StopSpider ¶

func (this *DefaultCluster) StopSpider(spiderName string)