Documentation ¶
Overview ¶
hitcounter command uses the search pattern (passed in via STDIN) to count the number of hits of each term inside the passed-in dictionary. The result is an ordered list of pairs term:count (descending on the number of hits), which is written to STDOUT.
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var RootCmd = &cobra.Command{ Use: "counthits", Short: "Counts hits passed-in queries.", Long: "Counts hits passed-in queries.", RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { return fmt.Errorf("please set the url argument.") } url := args[0] buff, err := ioutil.ReadAll(os.Stdin) if err != nil { return err } query := string(buff) if strings.Contains(string(buff), rdictVar) && dict == "" { return fmt.Errorf("query defintion uses $RDICT, please set --dictionary_file.") } clients := make(chan *http.Client, numClients) for i := 0; i < numClients; i++ { clients <- &http.Client{ Transport: &http.Transport{ Dial: (&net.Dialer{ LocalAddr: &net.TCPAddr{IP: defaultLocalAddr.IP, Zone: defaultLocalAddr.Zone}, KeepAlive: 3 * timeout, Timeout: timeout, }).Dial, ResponseHeaderTimeout: timeout, MaxIdleConnsPerHost: defaultConnections, }, } } errChan := make(chan error) var hits HitsByCount dictF, err := os.Open(dict) if err != nil { return err } defer dictF.Close() count := 0 scanner := bufio.NewScanner(dictF) wg := sync.WaitGroup{} for ; scanner.Scan(); count++ { wg.Add(1) go func(term string, count int) { defer wg.Done() client := <-clients defer func() { clients <- client }() query := strings.Replace(query, rdictVar, term, 1) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() req, err := http.NewRequest("GET", url, strings.NewReader(query)) if err != nil { errChan <- err return } req.WithContext(ctx) if debug { dReq, _ := httputil.DumpRequest(req, true) fmt.Printf("Processing term: %s\n", term) fmt.Println(string(dReq)) } resp, err := client.Do(req) if err != nil { errChan <- err return } defer resp.Body.Close() if debug { dResp, _ := httputil.DumpResponse(resp, true) fmt.Println(string(dResp)) } code := resp.StatusCode if resp.StatusCode != http.StatusOK { dReq, _ := httputil.DumpRequest(req, true) dResp, _ := httputil.DumpResponse(resp, true) fmt.Fprintf(os.Stderr, "invalid status code. want:200 got:%d. term:%s lineno:%d req:%s, resp:%s\n", code, term, count+1, string(dReq), string(dResp)) return } searchResp := struct { Hits struct { Total int64 `json:"total"` } `json:"hits"` }{} if err := json.NewDecoder(resp.Body).Decode(&searchResp); err != nil { dReq, _ := httputil.DumpRequest(req, true) dResp, _ := httputil.DumpResponse(resp, true) errChan <- fmt.Errorf("error parsing response %q. term:%s lineno:%d req:%s, resp:%s\n", err, term, count+1, string(dReq), string(dResp)) return } hits = append(hits, Hit{Term: term, Count: searchResp.Hits.Total}) }(scanner.Text(), count) } if err := scanner.Err(); err != nil { return err } go func() { wg.Wait() close(errChan) }() for err := range errChan { return err } sort.Sort(hits) writer := bufio.NewWriter(os.Stdout) defer writer.Flush() out, err := json.MarshalIndent(hits, "", " ") if err != nil { return err } fmt.Fprint(writer, string(out)) fmt.Fprintf(os.Stderr, "sent %d requests.", count) return nil }, }
Functions ¶
This section is empty.
Types ¶
type HitsByCount ¶
type HitsByCount []Hit
func (HitsByCount) Len ¶
func (a HitsByCount) Len() int
func (HitsByCount) Less ¶
func (a HitsByCount) Less(i, j int) bool
func (HitsByCount) Swap ¶
func (a HitsByCount) Swap(i, j int)
Click to show internal directories.
Click to hide internal directories.