core/amo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: use stream JSON instead of array.

Browse source
This commit is contained in:
Andrey Parhomenko 2024-06-06 00:46:14 +05:00
parent e981111774
commit 5177e03bef
1 changed files with 37 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
cmd/amocli/getter.go
View file

@ -33,23 +33,20 @@ func RunGetter[V any, G Getter[V]](g G, flags *mtool.Flags) {
log.Fatalf("NewAmoClient(...): %s\n", err)
}
c.API.SetMRPS(opts.MRPS)
finalValues := []V{}
enc := json.NewEncoder(os.Stdout)
if opts.Indent {
enc.SetIndent("", " ")
}
inter := make(chan os.Signal, 1)
signal.Notify(inter, os.Interrupt)
go func() {
<-inter
enc := json.NewEncoder(os.Stdout)
if opts.Indent {
enc.SetIndent("", " ")
}
err = enc.Encode(finalValues)
if err != nil {
log.Fatalf("json.Encode(...): %s\n", err)
}
os.Stdout.Sync()
os.Exit(0)
}()
finalNum := 0
if opts.All {
page := opts.StartPage
values, next, err := g.GetValues(
@ -62,11 +59,20 @@ func RunGetter[V any, G Getter[V]](g G, flags *mtool.Flags) {
"%s(...): %s\n", g.GetFuncName(), err,
)
}
finalValues = append(finalValues, values...)
finalNum += len(values)
if opts.Verbose {
log.Printf("Got %d %s (%d, page %d)\n",
len(values), g.GetNameMul(), len(finalValues), page)
len(values), g.GetNameMul(), finalNum, page)
}
for _, value := range values {
err := enc.Encode(value)
if err != nil {
log.Fatalf("json.Encode(...): %s\n", err)
}
}
page++
for page <= opts.EndPage && next != nil {
values, next, err = next()
@ -76,21 +82,23 @@ func RunGetter[V any, G Getter[V]](g G, flags *mtool.Flags) {
g.GetFuncName(), err,
)
}
finalValues = append(finalValues, values...)
finalNum += len(values)
if opts.Verbose {
log.Printf("Got %d %s (%d, page %d)\n",
len(values), g.GetNameMul(),
len(finalValues), page)
finalNum, page)
}
for _, value := range values {
err = enc.Encode(value)
if err != nil {
log.Fatalf("json.Encode(...): %s\n", err)
}
}
page++
}
bts, err := json.MarshalIndent(finalValues, "", " ")
if err != nil {
log.Fatalf("json.MarshalIndent(...) %s\n", err)
}
os.Stdout.Write(bts)
return
}
ids := ReadIDs(idStrs)
@ -143,12 +151,18 @@ func RunGetter[V any, G Getter[V]](g G, flags *mtool.Flags) {
}()
for values := range valueChan {
finalValues = append(finalValues, values...)
finalNum += len(values)
for _, value := range values {
err := enc.Encode(value)
if err != nil {
log.Fatalf("json.Encode(...): %s\n", err)
}
}
if opts.Verbose {
log.Printf(
"thtread(main) %.2f%%, sument=%d",
float32(len(finalValues))/float32(len(ids))*100.,
len(finalValues),
float32(finalNum)/float32(len(ids))*100.,
finalNum,
)
}
}
@ -157,7 +171,7 @@ func RunGetter[V any, G Getter[V]](g G, flags *mtool.Flags) {
rm := c.API.RequestsMade()
log.Printf(
"Summarized got %d %s\n",
len(finalValues), g.GetNameMul(),
finalNum, g.GetNameMul(),
)
log.Printf(
"Made %d requests in process\n",
@ -171,12 +185,4 @@ func RunGetter[V any, G Getter[V]](g G, flags *mtool.Flags) {
log.Printf("RPS = %f\n", float64(rm)/took)
}
enc := json.NewEncoder(os.Stdout)
if opts.Indent {
enc.SetIndent("", " ")
}
err = enc.Encode(finalValues)
if err != nil {
log.Fatalf("json.Encode(...): %s\n", err)
}
}