From c8cab4def37c2b497a29658817e6da43e173a051 Mon Sep 17 00:00:00 2001 From: surdeus Date: Wed, 5 Jun 2024 02:08:54 +0500 Subject: [PATCH] feat: using generics for getters. --- amocrm.go | 1 - cmd/amocli/common.go | 4 +- cmd/amocli/getcom.go | 65 ++++++----------- cmd/amocli/getlead.go | 149 +++++---------------------------------- cmd/amocli/getter.go | 150 ++++++++++++++++++++++++++++++++++++++++ cmd/amocli/updatecom.go | 2 +- companies.go | 52 ++++++++------ errors.go | 7 ++ 8 files changed, 232 insertions(+), 198 deletions(-) create mode 100644 cmd/amocli/getter.go create mode 100644 errors.go diff --git a/amocrm.go b/amocrm.go index efeb83f..378446c 100644 --- a/amocrm.go +++ b/amocrm.go @@ -32,4 +32,3 @@ func NewClient(secretPath string) (*Client, error) { }, nil } - diff --git a/cmd/amocli/common.go b/cmd/amocli/common.go index f435560..7573433 100644 --- a/cmd/amocli/common.go +++ b/cmd/amocli/common.go @@ -63,7 +63,7 @@ func MakeGetterFlags( &opts.All, "all", false, - "get all leads", + "get all entities", ) flags.IntVar( &opts.StartPage, @@ -160,7 +160,7 @@ func RunForSliceInThreads[V any]( return ret } -func ReadIDs(idStrs []string, flags *mtool.Flags) []int { +func ReadIDs(idStrs []string) []int { var ids []int if len(idStrs) > 0 { ids = make([]int, 0, len(idStrs)) diff --git a/cmd/amocli/getcom.go b/cmd/amocli/getcom.go index 70444b3..c3fd832 100644 --- a/cmd/amocli/getcom.go +++ b/cmd/amocli/getcom.go @@ -3,53 +3,32 @@ package main import "surdeus.su/core/amo" import "surdeus.su/core/ss/urlenc" import "surdeus.su/core/cli/mtool" -import "encoding/json" -import "strconv" -import "log" -import "fmt" +//import "encoding/json" +//import "strconv" +//import "log" +//import "fmt" //import "os" -var getComs = mtool.T("get-coms").Func(func(flags *mtool.Flags){ - var ( - secretPath string - ) - flags.StringVar( - &secretPath, - "secret", - "", - "path to JSON file with AMO CRM secrets", - "AMO_SECRET", - ) - idStrs := flags.Parse() - ids := make([]int, len(idStrs)) - for i, idStr := range idStrs { - var err error - ids[i], err = strconv.Atoi(idStr) - if err != nil { - log.Printf("Error: Atoi(%q): %s\n", err) - return - } - } +type CompanyGetter struct{} - c, err := amo.NewClient(secretPath) - if err != nil { - log.Fatalf("NewAmoClient(...): %s\n", err) - } +func (l CompanyGetter) GetValues( + c *amo.Client, + opts ...urlenc.Builder, +) ([]amo.Company, amo.NextFunc[[]amo.Company], error) { + return c.GetCompanies(opts...) +} - coms, err := c.GetCompanies( - urlenc.Array[int]{ - "id", - ids, - }, - ) - if err != nil { - log.Fatalf("GetCompanies(...): %s\n", err) - } +func (l CompanyGetter) GetNameMul() string { + return "companies" +} - bts, err := json.MarshalIndent(coms, "", " ") - if err != nil { - log.Fatalf("json.Marshal(...): %s\n", err) - } - fmt.Printf("%s\n", bts) +func (l CompanyGetter) GetFuncName() string { + return "amo.GetCompanies" +} + +var getComs = mtool.T("get-companies").Func(func( + flags *mtool.Flags, +){ + RunGetter(CompanyGetter{}, flags) }) diff --git a/cmd/amocli/getlead.go b/cmd/amocli/getlead.go index ad72950..63f3472 100644 --- a/cmd/amocli/getlead.go +++ b/cmd/amocli/getlead.go @@ -1,140 +1,29 @@ package main -import "surdeus.su/core/amo" -//import "surdeus.su/core/amo/api" -import "surdeus.su/core/ss/urlenc" import "surdeus.su/core/cli/mtool" -import "encoding/json" -import "log" -import "fmt" -import "time" -import "os" -//import "sync" +import "surdeus.su/core/amo" +import "surdeus.su/core/ss/urlenc" -var getLead = mtool.T("get-leads").Func(func(flags *mtool.Flags){ - var ( - opts DefaultFlags - ) +type LeadGetter struct {} - now := time.Now() - MakeDefaultFlags(&opts, flags) - MakeGetterFlags(&opts, flags) +func (l LeadGetter) GetValues( + c *amo.Client, + opts ...urlenc.Builder, +) ([]amo.Lead, amo.NextFunc[[]amo.Lead], error) { + return c.GetLeads(opts...) +} - idStrs := flags.Parse() - c, err := amo.NewClient(opts.SecretPath) - if err != nil { - log.Fatalf("NewAmoClient(...): %s\n", err) - } - c.API.SetMRPS(opts.MRPS) - finalLeads := []amo.Lead{} +func (l LeadGetter) GetNameMul() string { + return "leads" +} - if opts.All { - page := opts.StartPage - leads, next, err := c.GetLeads( - urlenc.Value[int]{"page", page}, - urlenc.Value[int]{"limit", opts.MEPR}, - ) - if err != nil { - log.Fatalf("amo.GetLeads(...): %s\n", err) - } - finalLeads = append(finalLeads, leads...) - if opts.Verbose { - log.Printf("Got %d leads (%d, page %d)\n", - len(leads), len(finalLeads), page) - } - page++ - for page <= opts.EndPage && next != nil { - leads, next, err = next() - if err != nil { - log.Fatalf("amo.GetLeads(...): %s\n", err) - } - finalLeads = append(finalLeads, leads...) - if opts.Verbose { - log.Printf("Got %d leads (%d, page %d)\n", - len(leads), len(finalLeads), page) - } - page++ - } - - bts, err := json.MarshalIndent(finalLeads, "", " ") - if err != nil { - log.Fatalf("json.MarshalIndent(...) %s\n", err) - } - os.Stdout.Write(bts) - return - } - - ids := ReadIDs(idStrs, flags) - if len(ids) == 0 { - log.Fatalf("Got no IDs to read leads") - return - } - - leadChan := make(chan []amo.Lead) - finish := RunForSliceInThreads[int]( - opts.Threads, opts.MEPR, - ids, func(thread int, s []int){ - leads, _, err := c.GetLeads( - urlenc.Array[int]{ - "id", - s, - }, - urlenc.Value[string]{ - "with", - "contacts", - }, - urlenc.Value[int]{ - "limit", - opts.MEPR, - }, - ) - if err != nil { - log.Printf("GetLeadsByIDs(...): %s\n", err) - } - leadChan <- leads - if opts.Verbose { - log.Printf( - "%d: Got %d leads\n", - thread, len(leads), - ) - } - }, - ) - - //var wg sync.WaitGroup - go func(){ - // Waiting for appending so we do not lose data. - <-finish - for len(leadChan) > 0 {} - close(leadChan) - }() - - for leads := range leadChan { - finalLeads = append(finalLeads, leads...) - } - - if opts.Verbose { - rm := c.API.RequestsMade() - log.Printf( - "Summarized got %d leads\n", - len(finalLeads), - ) - log.Printf( - "Made %d requests in process\n", - rm, - ) - took := time.Since(now).Seconds() - log.Printf( - "Took %f seconds\n", - took, - ) - log.Printf("RPS = %f\n", float64(rm)/took) - } - bts, err := json.MarshalIndent(finalLeads, "", " ") - if err != nil { - log.Fatalf("json.Marshal(...): %s\n", err) - } - fmt.Printf("%s\n", bts) +func (l LeadGetter) GetFuncName() string { + return "amo.GetLeads" +} +var getLead = mtool.T("get-leads").Func(func( + flags *mtool.Flags, +){ + RunGetter(LeadGetter{}, flags) }).Usage( "[id1 id2 ... idN]", ) diff --git a/cmd/amocli/getter.go b/cmd/amocli/getter.go new file mode 100644 index 0000000..1477d9f --- /dev/null +++ b/cmd/amocli/getter.go @@ -0,0 +1,150 @@ +package main + +import "surdeus.su/core/amo" +import "surdeus.su/core/ss/urlenc" + +import "surdeus.su/core/cli/mtool" + +import "fmt" +import "log" +import "time" +import "os" +import "encoding/json" + +type Getter[V any] interface { + GetValues(*amo.Client, ...urlenc.Builder) ([]V, amo.NextFunc[[]V], error) + GetNameMul() string + GetFuncName() string +} + +func RunGetter[V any, G Getter[V]](g G, flags *mtool.Flags) { + var ( + opts DefaultFlags + ) + + now := time.Now() + MakeDefaultFlags(&opts, flags) + MakeGetterFlags(&opts, flags) + + idStrs := flags.Parse() + c, err := amo.NewClient(opts.SecretPath) + if err != nil { + log.Fatalf("NewAmoClient(...): %s\n", err) + } + c.API.SetMRPS(opts.MRPS) + finalValues := []V{} + + if opts.All { + page := opts.StartPage + values, next, err := g.GetValues( + c, + urlenc.Value[int]{"page", page}, + urlenc.Value[int]{"limit", opts.MEPR}, + ) + if err != nil { + log.Fatalf( + "%s(...): %s\n", g.GetFuncName(), err, + ) + } + finalValues = append(finalValues, values...) + if opts.Verbose { + log.Printf("Got %d %s (%d, page %d)\n", + len(values), g.GetNameMul(), len(finalValues), page) + } + page++ + for page <= opts.EndPage && next != nil { + values, next, err = next() + if err != nil { + log.Fatalf("amo.GetLeads(...): %s\n", err) + } + finalValues = append(finalValues, values...) + if opts.Verbose { + log.Printf("Got %d %s (%d, page %d)\n", + len(values), g.GetNameMul(), + len(finalValues), page) + } + page++ + } + + bts, err := json.MarshalIndent(finalValues, "", " ") + if err != nil { + log.Fatalf("json.MarshalIndent(...) %s\n", err) + } + os.Stdout.Write(bts) + return + } + + ids := ReadIDs(idStrs) + if len(ids) == 0 { + log.Fatalf("Got no IDs to read %s", g.GetNameMul()) + return + } + + valueChan := make(chan []V) + finish := RunForSliceInThreads[int]( + opts.Threads, opts.MEPR, + ids, func(thread int, s []int){ + values, _, err := g.GetValues( + c, + urlenc.Array[int]{ + "id", + s, + }, + urlenc.Value[string]{ + "with", + "contacts", + }, + urlenc.Value[int]{ + "limit", + opts.MEPR, + }, + ) + if err != nil { + log.Printf("GetLeadsByIDs(...): %s\n", err) + } + valueChan <- values + if opts.Verbose { + log.Printf( + "%d: Got %d %s\n", + thread, len(values), + g.GetNameMul(), + ) + } + }, + ) + + //var wg sync.WaitGroup + go func(){ + // Waiting for appending so we do not lose data. + <-finish + for len(valueChan) > 0 {} + close(valueChan) + }() + + for values := range valueChan { + finalValues = append(finalValues, values...) + } + + if opts.Verbose { + rm := c.API.RequestsMade() + log.Printf( + "Summarized got %d %s\n", + len(finalValues), g.GetNameMul(), + ) + log.Printf( + "Made %d requests in process\n", + rm, + ) + took := time.Since(now).Seconds() + log.Printf( + "Took %f seconds\n", + took, + ) + log.Printf("RPS = %f\n", float64(rm)/took) + } + bts, err := json.MarshalIndent(finalValues, "", " ") + if err != nil { + log.Fatalf("json.Marshal(...): %s\n", err) + } + fmt.Printf("%s\n", bts) +} diff --git a/cmd/amocli/updatecom.go b/cmd/amocli/updatecom.go index 3c63cf7..11235b6 100644 --- a/cmd/amocli/updatecom.go +++ b/cmd/amocli/updatecom.go @@ -11,7 +11,7 @@ import "os" const MEPR = api.MaxEntitiesPerRequest var updateComs = -mtool.T("update-coms").Func(func(flags *mtool.Flags){ +mtool.T("update-companies").Func(func(flags *mtool.Flags){ var ( opts DefaultFlags ) diff --git a/companies.go b/companies.go index 7c86205..080068e 100644 --- a/companies.go +++ b/companies.go @@ -9,7 +9,7 @@ import "fmt" func (client *Client) GetCompany( companyID int, opts ...urlenc.Builder, -) (*companies.Company, error) { +) (*Company, error) { deal := new(companies.Company) resource := fmt.Sprintf( "/api/v4/companies/%d?%s", @@ -26,32 +26,39 @@ func (client *Client) GetCompany( // Get list of leads. func (client *Client) GetCompanies( opts ...urlenc.Builder, -) ([]companies.Company, error) { +) ([]Company, NextFunc[[]Company], error) { res := fmt.Sprintf( "/api/v4/companies?%s", urlenc.Join(opts...).Encode(), ) - ret := []companies.Company{} - for { - var coms companies.Companies - err := client.API.Get(res, &coms) - if err != nil { - if errors.Is(err, api.ErrNoContent) { - break - } - return nil, err + return client.GetCompaniesByURL(res) +} + +func (client *Client) GetCompaniesByURL( + u string, +) ([]Company, NextFunc[[]Company], error) { + var fn NextFunc[[]Company] + + coms := companies.Companies{} + err := client.API.Get(u, &coms) + if err != nil { + if errors.Is(err, api.ErrNoContent) { + return nil, nil, nil } - - ret = append(ret, coms.Embedded.Companies...) - - if coms.Links.Next.Href == "" { - break - } - res = coms.Links.Next.Href - + return nil, nil, err } - return ret, nil + + //ret = append(ret, coms.Embedded.Companies...) + + nextHref := coms.Links.Next.Href + if nextHref != "" { + fn = MakeNextFunc( + nextHref, + client.GetCompaniesByURL, + ) + } + return coms.Embedded.Companies, fn, nil } func (client *Client) UpdateCompany( @@ -67,7 +74,10 @@ func (client *Client) UpdateCompany( func (client *Client) UpdateCompanies( cs []companies.Company, ) error { - //ret := []companies.Company{} + if len(cs) > MEPR { + return ErrTooManyEntities + } + //ret := []Company{} err := client.API.Patch( "/api/v4/companies", cs, diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..917dd2a --- /dev/null +++ b/errors.go @@ -0,0 +1,7 @@ +package amo + +import "errors" + +var ( + ErrTooManyEntities = errors.New("too many entities") +)