feat: using generics for getters.
This commit is contained in:
parent
baf3d06c3e
commit
c8cab4def3
8 changed files with 232 additions and 198 deletions
|
|
@ -32,4 +32,3 @@ func NewClient(secretPath string) (*Client, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ func MakeGetterFlags(
|
|||
&opts.All,
|
||||
"all",
|
||||
false,
|
||||
"get all leads",
|
||||
"get all entities",
|
||||
)
|
||||
flags.IntVar(
|
||||
&opts.StartPage,
|
||||
|
|
@ -160,7 +160,7 @@ func RunForSliceInThreads[V any](
|
|||
return ret
|
||||
}
|
||||
|
||||
func ReadIDs(idStrs []string, flags *mtool.Flags) []int {
|
||||
func ReadIDs(idStrs []string) []int {
|
||||
var ids []int
|
||||
if len(idStrs) > 0 {
|
||||
ids = make([]int, 0, len(idStrs))
|
||||
|
|
|
|||
|
|
@ -3,53 +3,32 @@ package main
|
|||
import "surdeus.su/core/amo"
|
||||
import "surdeus.su/core/ss/urlenc"
|
||||
import "surdeus.su/core/cli/mtool"
|
||||
import "encoding/json"
|
||||
import "strconv"
|
||||
import "log"
|
||||
import "fmt"
|
||||
//import "encoding/json"
|
||||
//import "strconv"
|
||||
//import "log"
|
||||
//import "fmt"
|
||||
//import "os"
|
||||
|
||||
var getComs = mtool.T("get-coms").Func(func(flags *mtool.Flags){
|
||||
var (
|
||||
secretPath string
|
||||
)
|
||||
|
||||
flags.StringVar(
|
||||
&secretPath,
|
||||
"secret",
|
||||
"",
|
||||
"path to JSON file with AMO CRM secrets",
|
||||
"AMO_SECRET",
|
||||
)
|
||||
idStrs := flags.Parse()
|
||||
ids := make([]int, len(idStrs))
|
||||
for i, idStr := range idStrs {
|
||||
var err error
|
||||
ids[i], err = strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
log.Printf("Error: Atoi(%q): %s\n", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
type CompanyGetter struct{}
|
||||
|
||||
c, err := amo.NewClient(secretPath)
|
||||
if err != nil {
|
||||
log.Fatalf("NewAmoClient(...): %s\n", err)
|
||||
}
|
||||
func (l CompanyGetter) GetValues(
|
||||
c *amo.Client,
|
||||
opts ...urlenc.Builder,
|
||||
) ([]amo.Company, amo.NextFunc[[]amo.Company], error) {
|
||||
return c.GetCompanies(opts...)
|
||||
}
|
||||
|
||||
coms, err := c.GetCompanies(
|
||||
urlenc.Array[int]{
|
||||
"id",
|
||||
ids,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("GetCompanies(...): %s\n", err)
|
||||
}
|
||||
func (l CompanyGetter) GetNameMul() string {
|
||||
return "companies"
|
||||
}
|
||||
|
||||
bts, err := json.MarshalIndent(coms, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("json.Marshal(...): %s\n", err)
|
||||
}
|
||||
fmt.Printf("%s\n", bts)
|
||||
func (l CompanyGetter) GetFuncName() string {
|
||||
return "amo.GetCompanies"
|
||||
}
|
||||
|
||||
var getComs = mtool.T("get-companies").Func(func(
|
||||
flags *mtool.Flags,
|
||||
){
|
||||
RunGetter(CompanyGetter{}, flags)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,140 +1,29 @@
|
|||
package main
|
||||
|
||||
import "surdeus.su/core/amo"
|
||||
//import "surdeus.su/core/amo/api"
|
||||
import "surdeus.su/core/ss/urlenc"
|
||||
import "surdeus.su/core/cli/mtool"
|
||||
import "encoding/json"
|
||||
import "log"
|
||||
import "fmt"
|
||||
import "time"
|
||||
import "os"
|
||||
//import "sync"
|
||||
import "surdeus.su/core/amo"
|
||||
import "surdeus.su/core/ss/urlenc"
|
||||
|
||||
var getLead = mtool.T("get-leads").Func(func(flags *mtool.Flags){
|
||||
var (
|
||||
opts DefaultFlags
|
||||
)
|
||||
type LeadGetter struct {}
|
||||
|
||||
now := time.Now()
|
||||
MakeDefaultFlags(&opts, flags)
|
||||
MakeGetterFlags(&opts, flags)
|
||||
func (l LeadGetter) GetValues(
|
||||
c *amo.Client,
|
||||
opts ...urlenc.Builder,
|
||||
) ([]amo.Lead, amo.NextFunc[[]amo.Lead], error) {
|
||||
return c.GetLeads(opts...)
|
||||
}
|
||||
|
||||
idStrs := flags.Parse()
|
||||
c, err := amo.NewClient(opts.SecretPath)
|
||||
if err != nil {
|
||||
log.Fatalf("NewAmoClient(...): %s\n", err)
|
||||
}
|
||||
c.API.SetMRPS(opts.MRPS)
|
||||
finalLeads := []amo.Lead{}
|
||||
func (l LeadGetter) GetNameMul() string {
|
||||
return "leads"
|
||||
}
|
||||
|
||||
if opts.All {
|
||||
page := opts.StartPage
|
||||
leads, next, err := c.GetLeads(
|
||||
urlenc.Value[int]{"page", page},
|
||||
urlenc.Value[int]{"limit", opts.MEPR},
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("amo.GetLeads(...): %s\n", err)
|
||||
}
|
||||
finalLeads = append(finalLeads, leads...)
|
||||
if opts.Verbose {
|
||||
log.Printf("Got %d leads (%d, page %d)\n",
|
||||
len(leads), len(finalLeads), page)
|
||||
}
|
||||
page++
|
||||
for page <= opts.EndPage && next != nil {
|
||||
leads, next, err = next()
|
||||
if err != nil {
|
||||
log.Fatalf("amo.GetLeads(...): %s\n", err)
|
||||
}
|
||||
finalLeads = append(finalLeads, leads...)
|
||||
if opts.Verbose {
|
||||
log.Printf("Got %d leads (%d, page %d)\n",
|
||||
len(leads), len(finalLeads), page)
|
||||
}
|
||||
page++
|
||||
}
|
||||
|
||||
bts, err := json.MarshalIndent(finalLeads, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("json.MarshalIndent(...) %s\n", err)
|
||||
}
|
||||
os.Stdout.Write(bts)
|
||||
return
|
||||
}
|
||||
|
||||
ids := ReadIDs(idStrs, flags)
|
||||
if len(ids) == 0 {
|
||||
log.Fatalf("Got no IDs to read leads")
|
||||
return
|
||||
}
|
||||
|
||||
leadChan := make(chan []amo.Lead)
|
||||
finish := RunForSliceInThreads[int](
|
||||
opts.Threads, opts.MEPR,
|
||||
ids, func(thread int, s []int){
|
||||
leads, _, err := c.GetLeads(
|
||||
urlenc.Array[int]{
|
||||
"id",
|
||||
s,
|
||||
},
|
||||
urlenc.Value[string]{
|
||||
"with",
|
||||
"contacts",
|
||||
},
|
||||
urlenc.Value[int]{
|
||||
"limit",
|
||||
opts.MEPR,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("GetLeadsByIDs(...): %s\n", err)
|
||||
}
|
||||
leadChan <- leads
|
||||
if opts.Verbose {
|
||||
log.Printf(
|
||||
"%d: Got %d leads\n",
|
||||
thread, len(leads),
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
//var wg sync.WaitGroup
|
||||
go func(){
|
||||
// Waiting for appending so we do not lose data.
|
||||
<-finish
|
||||
for len(leadChan) > 0 {}
|
||||
close(leadChan)
|
||||
}()
|
||||
|
||||
for leads := range leadChan {
|
||||
finalLeads = append(finalLeads, leads...)
|
||||
}
|
||||
|
||||
if opts.Verbose {
|
||||
rm := c.API.RequestsMade()
|
||||
log.Printf(
|
||||
"Summarized got %d leads\n",
|
||||
len(finalLeads),
|
||||
)
|
||||
log.Printf(
|
||||
"Made %d requests in process\n",
|
||||
rm,
|
||||
)
|
||||
took := time.Since(now).Seconds()
|
||||
log.Printf(
|
||||
"Took %f seconds\n",
|
||||
took,
|
||||
)
|
||||
log.Printf("RPS = %f\n", float64(rm)/took)
|
||||
}
|
||||
bts, err := json.MarshalIndent(finalLeads, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("json.Marshal(...): %s\n", err)
|
||||
}
|
||||
fmt.Printf("%s\n", bts)
|
||||
func (l LeadGetter) GetFuncName() string {
|
||||
return "amo.GetLeads"
|
||||
}
|
||||
var getLead = mtool.T("get-leads").Func(func(
|
||||
flags *mtool.Flags,
|
||||
){
|
||||
RunGetter(LeadGetter{}, flags)
|
||||
}).Usage(
|
||||
"[id1 id2 ... idN]",
|
||||
)
|
||||
|
|
|
|||
150
cmd/amocli/getter.go
Normal file
150
cmd/amocli/getter.go
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
package main
|
||||
|
||||
import "surdeus.su/core/amo"
|
||||
import "surdeus.su/core/ss/urlenc"
|
||||
|
||||
import "surdeus.su/core/cli/mtool"
|
||||
|
||||
import "fmt"
|
||||
import "log"
|
||||
import "time"
|
||||
import "os"
|
||||
import "encoding/json"
|
||||
|
||||
type Getter[V any] interface {
|
||||
GetValues(*amo.Client, ...urlenc.Builder) ([]V, amo.NextFunc[[]V], error)
|
||||
GetNameMul() string
|
||||
GetFuncName() string
|
||||
}
|
||||
|
||||
func RunGetter[V any, G Getter[V]](g G, flags *mtool.Flags) {
|
||||
var (
|
||||
opts DefaultFlags
|
||||
)
|
||||
|
||||
now := time.Now()
|
||||
MakeDefaultFlags(&opts, flags)
|
||||
MakeGetterFlags(&opts, flags)
|
||||
|
||||
idStrs := flags.Parse()
|
||||
c, err := amo.NewClient(opts.SecretPath)
|
||||
if err != nil {
|
||||
log.Fatalf("NewAmoClient(...): %s\n", err)
|
||||
}
|
||||
c.API.SetMRPS(opts.MRPS)
|
||||
finalValues := []V{}
|
||||
|
||||
if opts.All {
|
||||
page := opts.StartPage
|
||||
values, next, err := g.GetValues(
|
||||
c,
|
||||
urlenc.Value[int]{"page", page},
|
||||
urlenc.Value[int]{"limit", opts.MEPR},
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf(
|
||||
"%s(...): %s\n", g.GetFuncName(), err,
|
||||
)
|
||||
}
|
||||
finalValues = append(finalValues, values...)
|
||||
if opts.Verbose {
|
||||
log.Printf("Got %d %s (%d, page %d)\n",
|
||||
len(values), g.GetNameMul(), len(finalValues), page)
|
||||
}
|
||||
page++
|
||||
for page <= opts.EndPage && next != nil {
|
||||
values, next, err = next()
|
||||
if err != nil {
|
||||
log.Fatalf("amo.GetLeads(...): %s\n", err)
|
||||
}
|
||||
finalValues = append(finalValues, values...)
|
||||
if opts.Verbose {
|
||||
log.Printf("Got %d %s (%d, page %d)\n",
|
||||
len(values), g.GetNameMul(),
|
||||
len(finalValues), page)
|
||||
}
|
||||
page++
|
||||
}
|
||||
|
||||
bts, err := json.MarshalIndent(finalValues, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("json.MarshalIndent(...) %s\n", err)
|
||||
}
|
||||
os.Stdout.Write(bts)
|
||||
return
|
||||
}
|
||||
|
||||
ids := ReadIDs(idStrs)
|
||||
if len(ids) == 0 {
|
||||
log.Fatalf("Got no IDs to read %s", g.GetNameMul())
|
||||
return
|
||||
}
|
||||
|
||||
valueChan := make(chan []V)
|
||||
finish := RunForSliceInThreads[int](
|
||||
opts.Threads, opts.MEPR,
|
||||
ids, func(thread int, s []int){
|
||||
values, _, err := g.GetValues(
|
||||
c,
|
||||
urlenc.Array[int]{
|
||||
"id",
|
||||
s,
|
||||
},
|
||||
urlenc.Value[string]{
|
||||
"with",
|
||||
"contacts",
|
||||
},
|
||||
urlenc.Value[int]{
|
||||
"limit",
|
||||
opts.MEPR,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("GetLeadsByIDs(...): %s\n", err)
|
||||
}
|
||||
valueChan <- values
|
||||
if opts.Verbose {
|
||||
log.Printf(
|
||||
"%d: Got %d %s\n",
|
||||
thread, len(values),
|
||||
g.GetNameMul(),
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
//var wg sync.WaitGroup
|
||||
go func(){
|
||||
// Waiting for appending so we do not lose data.
|
||||
<-finish
|
||||
for len(valueChan) > 0 {}
|
||||
close(valueChan)
|
||||
}()
|
||||
|
||||
for values := range valueChan {
|
||||
finalValues = append(finalValues, values...)
|
||||
}
|
||||
|
||||
if opts.Verbose {
|
||||
rm := c.API.RequestsMade()
|
||||
log.Printf(
|
||||
"Summarized got %d %s\n",
|
||||
len(finalValues), g.GetNameMul(),
|
||||
)
|
||||
log.Printf(
|
||||
"Made %d requests in process\n",
|
||||
rm,
|
||||
)
|
||||
took := time.Since(now).Seconds()
|
||||
log.Printf(
|
||||
"Took %f seconds\n",
|
||||
took,
|
||||
)
|
||||
log.Printf("RPS = %f\n", float64(rm)/took)
|
||||
}
|
||||
bts, err := json.MarshalIndent(finalValues, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("json.Marshal(...): %s\n", err)
|
||||
}
|
||||
fmt.Printf("%s\n", bts)
|
||||
}
|
||||
|
|
@ -11,7 +11,7 @@ import "os"
|
|||
const MEPR = api.MaxEntitiesPerRequest
|
||||
|
||||
var updateComs =
|
||||
mtool.T("update-coms").Func(func(flags *mtool.Flags){
|
||||
mtool.T("update-companies").Func(func(flags *mtool.Flags){
|
||||
var (
|
||||
opts DefaultFlags
|
||||
)
|
||||
|
|
|
|||
52
companies.go
52
companies.go
|
|
@ -9,7 +9,7 @@ import "fmt"
|
|||
func (client *Client) GetCompany(
|
||||
companyID int,
|
||||
opts ...urlenc.Builder,
|
||||
) (*companies.Company, error) {
|
||||
) (*Company, error) {
|
||||
deal := new(companies.Company)
|
||||
resource := fmt.Sprintf(
|
||||
"/api/v4/companies/%d?%s",
|
||||
|
|
@ -26,32 +26,39 @@ func (client *Client) GetCompany(
|
|||
// Get list of leads.
|
||||
func (client *Client) GetCompanies(
|
||||
opts ...urlenc.Builder,
|
||||
) ([]companies.Company, error) {
|
||||
) ([]Company, NextFunc[[]Company], error) {
|
||||
res := fmt.Sprintf(
|
||||
"/api/v4/companies?%s",
|
||||
urlenc.Join(opts...).Encode(),
|
||||
)
|
||||
|
||||
ret := []companies.Company{}
|
||||
for {
|
||||
var coms companies.Companies
|
||||
err := client.API.Get(res, &coms)
|
||||
if err != nil {
|
||||
if errors.Is(err, api.ErrNoContent) {
|
||||
break
|
||||
}
|
||||
return nil, err
|
||||
return client.GetCompaniesByURL(res)
|
||||
}
|
||||
|
||||
func (client *Client) GetCompaniesByURL(
|
||||
u string,
|
||||
) ([]Company, NextFunc[[]Company], error) {
|
||||
var fn NextFunc[[]Company]
|
||||
|
||||
coms := companies.Companies{}
|
||||
err := client.API.Get(u, &coms)
|
||||
if err != nil {
|
||||
if errors.Is(err, api.ErrNoContent) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
ret = append(ret, coms.Embedded.Companies...)
|
||||
|
||||
if coms.Links.Next.Href == "" {
|
||||
break
|
||||
}
|
||||
res = coms.Links.Next.Href
|
||||
|
||||
return nil, nil, err
|
||||
}
|
||||
return ret, nil
|
||||
|
||||
//ret = append(ret, coms.Embedded.Companies...)
|
||||
|
||||
nextHref := coms.Links.Next.Href
|
||||
if nextHref != "" {
|
||||
fn = MakeNextFunc(
|
||||
nextHref,
|
||||
client.GetCompaniesByURL,
|
||||
)
|
||||
}
|
||||
return coms.Embedded.Companies, fn, nil
|
||||
}
|
||||
|
||||
func (client *Client) UpdateCompany(
|
||||
|
|
@ -67,7 +74,10 @@ func (client *Client) UpdateCompany(
|
|||
func (client *Client) UpdateCompanies(
|
||||
cs []companies.Company,
|
||||
) error {
|
||||
//ret := []companies.Company{}
|
||||
if len(cs) > MEPR {
|
||||
return ErrTooManyEntities
|
||||
}
|
||||
//ret := []Company{}
|
||||
err := client.API.Patch(
|
||||
"/api/v4/companies",
|
||||
cs,
|
||||
|
|
|
|||
7
errors.go
Normal file
7
errors.go
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
package amo
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrTooManyEntities = errors.New("too many entities")
|
||||
)
|
||||
Loading…
Reference in a new issue