Documentation ΒΆ
Index ΒΆ
- Constants
- Variables
- func AppendAccountLikesIndex(id uint32, likes []Like)
- func AppendHttpResponse(b []byte, status, headers string, body []byte) []byte
- func B2s(b []byte) string
- func Bitset128Any(t0 *uint64, t1 *uint64, p2 *Bitset128) bool
- func Bitset128Contains(t0 *uint64, t1 *uint64, p2 *Bitset128) bool
- func Bitset128Get(bit byte, t0 *uint64, t1 *uint64) bool
- func Bitset128Set(bit byte, t0 *uint64, t1 *uint64)
- func CalculateGroups()
- func CalculateGroupsParallel(wg *sync.WaitGroup)
- func EvioServer(port int, ...)
- func GetFilterCache(token string) []byte
- func GetGroupCache(token string) []byte
- func GetRecommendCache(token string) []byte
- func GetSuggestCache(token string) []byte
- func LoadOptions(dir string) (int32, int32)
- func MakeIndexes(capacity int)
- func Parsereq(data []byte, req *Request) (leftover []byte, err error)
- func PrintMemUsage(info string)
- func PutFilterCache(token string, value []byte)
- func PutGroupCache(token string, value []byte)
- func PutRecommendCache(token string, value []byte)
- func PutSuggestCache(token string, value []byte)
- func RebuildAggregates(accounts *[]Account, maxId uint32)
- func RebuildIndexes(accounts *[]Account, maxId uint32)
- func RebuildRecommendIndexes(accounts *[]Account, maxId uint32)
- func Recommend(myId uint32, limit int, params map[string]string) []uint32
- func ResetCaches()
- func ResetIndexes()
- func S2b(s string) []byte
- func SearchBytes(a []byte, x byte) int
- func SearchLikes(a []Like, id uint32) int
- func SearchUint32(a []uint32, x uint32) int
- func Suggest(myId uint32, limit int, params map[string]string) []uint32
- func Timenow() string
- func UnmarshalNew(account *AccountJson, bytes []byte) error
- func UnmarshalUpdate(account *AccountJson, bytes []byte) error
- func ValidateLikes(likes []LikeUpdate) error
- func ValidateNew(accountJson *AccountJson) error
- func ValidateUpdate(accountJson *AccountJson, id uint32) error
- func WaitRebuildComplete()
- func WriteAccountOut(out []byte, separate bool, fields map[string]bool, account *Account, ...) []byte
- func WriteAccountOutRecommend(out []byte, separate bool, account *Account, accountId uint32) []byte
- func WriteAccountOutSuggest(out []byte, separate bool, account *Account, accountId uint32) []byte
- type Account
- type Account2
- type AccountJson
- type Aggregate
- type Bitset128
- func (p *Bitset128) Any(p2 *Bitset128) bool
- func (p *Bitset128) Contains(p2 *Bitset128) bool
- func (p *Bitset128) Count() (count int)
- func (p *Bitset128) Get(bit byte) bool
- func (p *Bitset128) IsEmpty() bool
- func (p *Bitset128) IsNotEmpty() bool
- func (p *Bitset128) Reset()
- func (p *Bitset128) Set(bit byte)
- type Bitset256
- type ByteSlice
- type CacheKey
- type Context
- type Dict16
- type Dict8
- type Filter
- type Group
- func (p *Group) CacheKey() GroupCacheKey
- func (g *Group) Classify(accounts *[]Account, updateCache bool) []GroupItem
- func (g *Group) ClassifyAccount(id uint32, collect func(uint64))
- func (g *Group) DebugWriteJsonGroupItem(w io.Writer, groupKey uint64, count int32)
- func (p *Group) FromArgs(args *fasthttp.Args) error
- func (g *Group) FromParams(params map[string]string) error
- func (g *Group) Sort(counters map[uint64]int32, limit int) []GroupItem
- func (g *Group) WriteJsonGroupItemOut(out []byte, groupKey uint64, count int32) []byte
- type GroupCacheKey
- type GroupItem
- type IdArray
- type IndexAndIterator
- type IndexIterator
- type IndexOrIterator
- type Like
- type LikeSlice
- type LikeUpdate
- type NullOp
- type Other
- type Premium
- type Reply
- type Request
- type Storage
- type Uint32Slice
- type VectorUint32
Constants ΒΆ
const ( BIRTH_YEARS = 56 JOINED_YEARS = 8 STATUSES = 3 )
const ( HUGE_BLOCK_SIZE = 32 * 1024 // Π΄Π»Ρ ΠΎΡΠ΅Π½Ρ Π½Π΅ΡΠ΅Π»Π΅ΠΊΡΠΈΠ²Π½ΡΡ ΠΈΠ½Π΄Π΅ΠΊΡΠΎΠ² (ΠΏΠΎΠ») LARGE_BLOCK_SIZE = 4 * 1024 // Π΄Π»Ρ ΡΡΠ΅Π΄Π½Π΅-ΡΠ΅Π»Π΅ΠΊΡΠΈΠ²Π½ΡΡ ΠΈΠ½Π΄Π΅ΠΊΡΠΎΠ² (Π΄Π΅ΡΡΡΠΊΠΈ-ΡΠΎΡΠ½ΠΈ Π±Π°ΠΊΠ΅ΡΠΎΠ²) MEDIUM_BLOCK_SIZE = 1024 // Π΄Π»Ρ Π²ΡΡΠΎΠΊΠΎ-ΡΡΠ΅Π΄Π½Π΅-ΡΠ΅Π»Π΅ΠΊΡΠΈΠ²Π½ΡΡ SMALL_BLOCK_SIZE = 32 // Π΄Π»Ρ Π²ΡΡΠΎΠΊΠΎ-ΡΠ΅Π»Π΅ΠΊΡΠΈΠ²Π½ΡΡ LIKES_INDEX_VECTOR_CAPACITY = 24 )
const BIRTH_YEARS_LEN = 56
const DEFAULT_ITER_CAPACITY = 16
const LIKE_MOD_FLAG = 2000000
const TEST_BITS_32 = 4 * 8
const TEST_BITS_64 = 8 * 8
Variables ΒΆ
var ( LikesIndexCompact [][]uint32 InterestsIndexCompact [][]uint32 InterestsSexIndexCompact [][]uint32 Interests2xIndexCompact [][]uint32 BirthYearIndexCompact [][]uint32 CityIndexCompact [][]uint32 CountryIndexCompact [][]uint32 FnameIndexCompact [][]uint32 SexIndexCompact [][]uint32 StatusIndexCompact [][]uint32 //sex, premium, status, interest [,country|city] RecommendIndexCompact map[int][]uint32 RecommendIndexCountryCompact map[int][]uint32 RecommendIndexCityCompact map[int][]uint32 // ΠΈΠΌΠ΅Π½Π° ΠΊΠΎΡΠΎΡΡΠ΅ ΠΌΡ Π²ΠΈΠ΄Π΅Π»ΠΈ Π΄Π»Ρ ΠΏΠΎΠ»ΠΎΠ² [ΠΏΠΎΠ»][ΠΈΠΌΡ] - 1:0 SexNames [2][]byte BirthYearCityIndexCompact [][]uint32 EmailPrefixIndexCompact [][]uint32 CountryPhoneCodeSexIndex map[int][]uint32 CityPhoneCodeSexIndex map[int][]uint32 )
var ( Store *Storage FnameDict = makeDict8() SnameDict = makeDict16() DomainDict = makeDict8() InterestDict = makeDict8() CityDict = makeDict16() CountryDict = makeDict8() EmailMap = make(map[string]uint32, 1340000) PhoneMap = make(map[string]uint32, 600000) MinJoined = timeStampOf("2011-01-01T00:00:00Z") MaxJoinedMinPremium = timeStampOf("2018-01-01T00:00:00Z") BirthMin = timeStampOf("1950-01-01T00:00:00Z") BirthMax = timeStampOf("2005-01-01T00:00:00Z") )
var ( STATUS_OFFSET = uint(1) STATUS_MASK = mask(STATUS_OFFSET, 2) DOMAIN_OFFSET = uint(3) DOMAIN_MASK = mask(DOMAIN_OFFSET, 4) COUNTRY_OFFSET = uint(7) COUNTRY_MASK = mask(COUNTRY_OFFSET, 7) CITY_OFFSET = uint(14) CITY_MASK = mask(CITY_OFFSET, 10) FNAME_OFFSET = uint(24) FNAME_MASK = mask(FNAME_OFFSET, 7) SNAME_OFFSET = uint(31) SNAME_MASK = mask(SNAME_OFFSET, 11) BIRTH_Y_OFFSET = uint(42) BIRTH_Y_MASK = mask(BIRTH_Y_OFFSET, 6) PHONECODE_OFFSET = uint(48) PHONECODE_MASK = mask(PHONECODE_OFFSET, 10) JOINED_Y_OFFSET = uint(58) JOINED_Y_MASK = mask(JOINED_Y_OFFSET, 3) IS_PREMIUM_OFFSET = uint(61) IS_PREMIUM_MASK = mask(IS_PREMIUM_OFFSET, 1) )
var CityCount int
var CountryCount int
var InterestsCount int
var TOKEN_MASK = uint64(mask(0, 24))
Functions ΒΆ
func AppendHttpResponse ΒΆ
func Bitset128Set ΒΆ
func CalculateGroups ΒΆ
func CalculateGroups()
func EvioServer ΒΆ
func GetFilterCache ΒΆ
func GetGroupCache ΒΆ
func GetRecommendCache ΒΆ
func GetSuggestCache ΒΆ
func LoadOptions ΒΆ
func MakeIndexes ΒΆ
func MakeIndexes(capacity int)
func PrintMemUsage ΒΆ
func PrintMemUsage(info string)
func ioutil.ReadFile(filename string) ([]byte, error)
----
file, err := os.Open("binary.dat") if err != nil { fmt.Println(err) return } defer file.Close() info, err := file.Stat() if err != nil { return nil, err } // calculate the bytes size var size int64 = info.Size() bytes := make([]byte, size) // read into buffer buffer := bufio.NewReader(file) _,err = buffer.Read(bytes)
func LoadDataArray2(dir string, arr *[]Account) (maxId uint32, runType int32, err error) {
if Store.Now, runType = LoadOptions(dir); Store.Now == 0 { return 0, 0, fmt.Errorf("error load %soptions.txt", dir) } r, err := zip.OpenReader(dir + "data.zip") if err != nil { return 0, runType, err } defer r.Close() //bytes := make([]byte, 20*1024*1024) // Iterate through the files in the archive, fmt.Printf("%v\tloading\n", Timenow()) for _, f := range r.File { fmt.Print(".") rc, err := f.Open() if err != nil { return 0, runType, err } bytes, err := ioutil.ReadAll(rc) var p fastjson.Parser acc, err := p.ParseBytes(bytes) if err != nil { log.Fatal(err) } for _, v := range acc.GetArray("accounts") { id := uint32(v.GetInt("id")) if maxId < id { maxId = id } if err = CompressJsonToAccount(id, &(*arr)[id-1], v); err != nil { return 0, runType, err } } _ = rc.Close() } Rebuild(arr, maxId) return maxId, runType, nil }
PrintMemUsage outputs the current, total and OS memory being used. As well as the number of garage collection cycles completed.
func PutFilterCache ΒΆ
func PutGroupCache ΒΆ
func PutRecommendCache ΒΆ
func PutSuggestCache ΒΆ
func RebuildAggregates ΒΆ
func RebuildIndexes ΒΆ
func ResetCaches ΒΆ
func ResetCaches()
func ResetIndexes ΒΆ
func ResetIndexes()
func SearchBytes ΒΆ
func SearchLikes ΒΆ
func SearchUint32 ΒΆ
func UnmarshalNew ΒΆ
func UnmarshalNew(account *AccountJson, bytes []byte) error
func UnmarshalUpdate ΒΆ
func UnmarshalUpdate(account *AccountJson, bytes []byte) error
func ValidateLikes ΒΆ
func ValidateLikes(likes []LikeUpdate) error
func ValidateNew ΒΆ
func ValidateNew(accountJson *AccountJson) error
func ValidateUpdate ΒΆ
func ValidateUpdate(accountJson *AccountJson, id uint32) error
func WaitRebuildComplete ΒΆ
func WaitRebuildComplete()
func WriteAccountOut ΒΆ
func WriteAccountOut(out []byte, separate bool, fields map[string]bool, account *Account, accountId uint32) []byte
* ΠΎΡΠ΄Π°Π²Π°ΡΡ Π½ΡΠΆΠ½ΠΎ id, email ΠΈ ΠΏΠΎΠ»Ρ ΠΈΠ· Π·Π°ΠΏΡΠΎΡΠ° (ΠΊΡΠΎΠΌΠ΅ interests ΠΈ likes)
func WriteAccountOutRecommend ΒΆ
var recommendFields = map[string]bool{"status": true, "fname": true, "sname": true, "birth": true, "premium": true}
Types ΒΆ
type AccountJson ΒΆ
type AccountJson struct { Id uint32 `json:"id"` Fname string `json:"fname"` Sname string `json:"sname"` Email string `json:"email"` Interests []string `json:"interests"` Status string `json:"status"` Premium Premium `json:"premium"` Sex string `json:"sex"` Phone string `json:"phone"` Likes []Like `json:"likes"` Birth int32 `json:"birth"` City string `json:"city"` Country string `json:"country"` Joined int32 `json:"joined"` }
func (AccountJson) MarshalEasyJSON ΒΆ
func (v AccountJson) MarshalEasyJSON(w *jwriter.Writer)
MarshalEasyJSON supports easyjson.Marshaler interface
func (AccountJson) MarshalJSON ΒΆ
func (v AccountJson) MarshalJSON() ([]byte, error)
MarshalJSON supports json.Marshaler interface
func (*AccountJson) UnmarshalEasyJSON ΒΆ
func (v *AccountJson) UnmarshalEasyJSON(l *jlexer.Lexer)
UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (*AccountJson) UnmarshalJSON ΒΆ
func (v *AccountJson) UnmarshalJSON(data []byte) error
UnmarshalJSON supports json.Unmarshaler interface
type Context ΒΆ
type Context struct { Is evio.InputStream Out [8192]byte }
type Filter ΒΆ
type Filter struct { Limit int ShortCircuit bool // ΡΠΏΠΈΡΠΎΠΊ ΠΏΠΎΠ»Π΅ΠΉ ΡΠΈΠ»ΡΡΡΠ° Fields map[string]bool Len int // contains filtered or unexported fields }
func MakeFilterArgs ΒΆ
todo: eliminate Atoi?
func (*Filter) Index ΒΆ
func (f *Filter) Index(sorted bool) ([]uint32, IndexIterator)
todo: Π΄Π»Ρ Π³ΡΡΠΏΠΏ ΡΠΎΡΡΠΈΡΠΎΠ²Π°ΡΡ Π½Π΅ Π½ΡΠΆΠ½ΠΎ ?
type Group ΒΆ
type Group struct { Limit int GroupBy []string FilterBy map[string]bool Filter Filter Order int8 // contains filtered or unexported fields }
ΠΠΎΠ»Π΅ΠΉ Π΄Π»Ρ Π³ΡΡΠΏΠΏΠΈΡΠΎΠ²ΠΊΠΈ Π²ΡΠ΅Π³ΠΎ ΠΏΡΡΡ - sex, status, interests, country, city.
func (*Group) CacheKey ΒΆ
func (p *Group) CacheKey() GroupCacheKey
func (*Group) ClassifyAccount ΒΆ
* ΠΏΠ΅ΡΠ΅Π΄Π°Π΅ΡΡΡ ΡΡΠ½ΠΊΡΠΈΡ Π΄Π»Ρ ΡΡΠ΅ΡΠ° ΠΊΠ»ΡΡΠ° ΠΊΠ»Π°ΡΡΠΈΡΠΈΠΊΠ°ΡΠΈΠΈ
func (*Group) DebugWriteJsonGroupItem ΒΆ
type GroupCacheKey ΒΆ
type GroupCacheKey struct {
// contains filtered or unexported fields
}
type IndexAndIterator ΒΆ
type IndexAndIterator struct {
// contains filtered or unexported fields
}
func (*IndexAndIterator) Len ΒΆ
func (p *IndexAndIterator) Len() (l int)
func (*IndexAndIterator) Next ΒΆ
func (p *IndexAndIterator) Next() uint32
todo: Π±ΡΠ°ΡΡ ΡΠ°ΠΌΡΠΉ ΠΊΠΎΡΠΎΡΠΊΠΈΠΉ ΠΈΠ½Π΄Π΅ΠΊΡ ΠΈ ΠΏΡΡΠΈΡΡΡ ΠΏΠΎ Π½Π΅ΠΌΡ Π½Π°Π·Π°Π΄
func (*IndexAndIterator) Prepare ΒΆ
func (p *IndexAndIterator) Prepare() *IndexAndIterator
func (*IndexAndIterator) ToSingle ΒΆ
func (p *IndexAndIterator) ToSingle() (index []uint32)
type IndexIterator ΒΆ
type IndexOrIterator ΒΆ
type IndexOrIterator struct {
// contains filtered or unexported fields
}
func (*IndexOrIterator) Len ΒΆ
func (p *IndexOrIterator) Len() (l int)
func (*IndexOrIterator) Next ΒΆ
func (p *IndexOrIterator) Next() uint32
func (*IndexOrIterator) ToSingle ΒΆ
func (p *IndexOrIterator) ToSingle() (index []uint32)
type Like ΒΆ
func (Like) MarshalEasyJSON ΒΆ
MarshalEasyJSON supports easyjson.Marshaler interface
func (Like) MarshalJSON ΒΆ
MarshalJSON supports json.Marshaler interface
func (*Like) UnmarshalEasyJSON ΒΆ
UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (*Like) UnmarshalJSON ΒΆ
UnmarshalJSON supports json.Unmarshaler interface
type LikeUpdate ΒΆ
func ParseLikesUpdate ΒΆ
func ParseLikesUpdate(bytes []byte) ([]LikeUpdate, error)
type Premium ΒΆ
func (Premium) MarshalEasyJSON ΒΆ
MarshalEasyJSON supports easyjson.Marshaler interface
func (Premium) MarshalJSON ΒΆ
MarshalJSON supports json.Marshaler interface
func (*Premium) UnmarshalEasyJSON ΒΆ
UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (*Premium) UnmarshalJSON ΒΆ
UnmarshalJSON supports json.Unmarshaler interface
type Storage ΒΆ
type Storage struct { Accounts []Account Accounts2 []Account2 MaxId uint32 Now int32 RunType int32 Likes [][]Like // contains filtered or unexported fields }
func (*Storage) CompressToAccount ΒΆ
func (p *Storage) CompressToAccount(accJson *AccountJson) error
func (*Storage) CompressUpdateToAccount ΒΆ
func (p *Storage) CompressUpdateToAccount(accJson *AccountJson) error
func (*Storage) LikesUpdate ΒΆ
func (p *Storage) LikesUpdate(likes []LikeUpdate)
type Uint32Slice ΒΆ
type Uint32Slice []uint32
func (Uint32Slice) Len ΒΆ
func (p Uint32Slice) Len() int
func (Uint32Slice) Less ΒΆ
func (p Uint32Slice) Less(i, j int) bool
func (Uint32Slice) Sort ΒΆ
func (p Uint32Slice) Sort()
func (Uint32Slice) Swap ΒΆ
func (p Uint32Slice) Swap(i, j int)
type VectorUint32 ΒΆ
type VectorUint32 struct {
// contains filtered or unexported fields
}
func (*VectorUint32) CopyTo ΒΆ
func (p *VectorUint32) CopyTo(out []uint32)
func (*VectorUint32) Get ΒΆ
func (p *VectorUint32) Get(index int) uint32
func (*VectorUint32) Iterate ΒΆ
func (p *VectorUint32) Iterate(iter func(i int, v uint32) bool) int
* Iterate() returns last index iterated
func (*VectorUint32) Len ΒΆ
func (p *VectorUint32) Len() int
Π²ΠΎΠ·Π²ΡΠ°ΡΠ°Π΅ΠΌ ΠΏΡΠ΅Π΄ΡΠ΄ΡΡΠ΅Π΅ Π·Π½Π°Π΅Π½ΠΈΠ΅
func (*VectorUint32) Push ΒΆ
func (p *VectorUint32) Push(value uint32) int