Skip to content

Commit e01b174

Browse files
tlimoncellitlimoncelli
authored
FEATURE: Output memory statistics (--full) (#4031)
# Issue How memory-efficient is dnscontrol? I'm interested in how much memory per record we are using. # Resolution Add memory stats when --full is output. A large config: ``` Inaccurate statistics: {"memory_in_use":39930120,"memory_in_use_str":"40 MB","num_records":2806,"num_zones":176,"rc_size":520,"benchmark1":14230,"benchmark1str":"14 KiB bytes"} ``` A small config: ``` Inaccurate statistics: {"memory_in_use":24447240,"memory_in_use_str":"24 MB","num_records":146,"num_zones":20,"rc_size":520,"benchmark1":167446,"benchmark1str":"164 KiB bytes"} ``` RecordConfig is 520 bytes yet the overhead per record is 14K to 164K.
1 parent 566df5e commit e01b174

File tree

3 files changed

+58
-0
lines changed

3 files changed

+58
-0
lines changed

‎commands/ppreviewPush.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@ import (
88
"fmt"
99
"os"
1010
"regexp"
11+
"runtime"
1112
"strconv"
1213
"strings"
1314
"sync"
1415
"sync/atomic"
1516
"time"
17+
"unsafe"
1618

1719
"github.com/StackExchange/dnscontrol/v4/models"
1820
"github.com/StackExchange/dnscontrol/v4/pkg/bindserial"
@@ -25,6 +27,7 @@ import (
2527
"github.com/StackExchange/dnscontrol/v4/pkg/providers"
2628
"github.com/StackExchange/dnscontrol/v4/pkg/rfc4183"
2729
"github.com/StackExchange/dnscontrol/v4/pkg/zonerecs"
30+
"github.com/dustin/go-humanize"
2831
"github.com/nozzle/throttler"
2932
"github.com/urfave/cli/v3"
3033
"golang.org/x/exp/slices"
@@ -409,8 +412,10 @@ func prun(args PPreviewArgs, push bool, interactive bool, out printer.CLI, repor
409412
fmt.Fprintf(os.Stderr, "##teamcity[buildStatus status='SUCCESS' text='%d corrections']", totalCorrections)
410413
}
411414
rfc4183.PrintWarning()
415+
out.PrintfIf(fullMode, "Inaccurate statistics: %s\n", stats(cfg))
412416
notifier.Done()
413417
out.Printf("Done. %d corrections.\n", totalCorrections)
418+
414419
err = writeReport(report, reportItems)
415420
if err != nil {
416421
return errors.New("could not write report")
@@ -424,6 +429,56 @@ func prun(args PPreviewArgs, push bool, interactive bool, out printer.CLI, repor
424429
return nil
425430
}
426431

432+
// stats returns a JSON string with memory usage statistics.
433+
// These stats are unofficial and subject to change without notice.
434+
// "average_mem_per_record" is misleading because it includes all memory overhead.
435+
func stats(cfg *models.DNSConfig) string {
436+
437+
// https://www.datadoghq.com/blog/go-memory-metrics/
438+
// [T]he following expression accurately reflects the value the runtime attempts to maintain as the limit:
439+
// runtime.MemStats.Sys − runtime.MemStats.HeapReleased
440+
runtime.GC()
441+
var m runtime.MemStats
442+
runtime.ReadMemStats(&m)
443+
memoryInUse := m.Sys - m.HeapReleased
444+
445+
numRecords := countRecords(cfg)
446+
memPerRecord := int64(float64(memoryInUse) / float64(max(1, numRecords)))
447+
memPerRecordStr := humanize.IBytes(uint64(memPerRecord)) + " bytes"
448+
449+
statsInfo := struct {
450+
MemoryInUse uint64 `json:"memory_in_use"`
451+
MemoryInUseStr string `json:"memory_in_use_str"`
452+
NumRecords int `json:"num_records"`
453+
NumZones int `json:"num_zones"`
454+
RCSize int `json:"rc_size"`
455+
Benchmark1 int64 `json:"benchmark1"`
456+
Benchmark1Str string `json:"benchmark1str"`
457+
}{
458+
MemoryInUse: memoryInUse,
459+
MemoryInUseStr: humanize.Bytes(memoryInUse),
460+
NumRecords: numRecords,
461+
NumZones: len(cfg.Domains),
462+
RCSize: int(unsafe.Sizeof((models.RecordConfig{}))),
463+
Benchmark1: memPerRecord,
464+
Benchmark1Str: memPerRecordStr,
465+
}
466+
467+
jsonBytes, err := json.Marshal(statsInfo)
468+
if err != nil {
469+
return fmt.Sprintf("error marshaling stats: %v", err)
470+
}
471+
return string(jsonBytes)
472+
}
473+
474+
func countRecords(cfg *models.DNSConfig) int {
475+
total := 0
476+
for _, domain := range cfg.Domains {
477+
total += len(domain.Records)
478+
}
479+
return total
480+
}
481+
427482
// whichZonesToProcess takes a list of DomainConfigs and a filter string and
428483
// returns a list of DomainConfigs whose Domain.UniqueName matched the
429484
// filter. The filter string is a comma-separated list of domain names. If the

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ require (
6262
github.com/aliyun/alibaba-cloud-sdk-go v1.63.107
6363
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6
6464
github.com/centralnicgroup-opensource/rtldev-middleware-go-sdk/v5 v5.0.18
65+
github.com/dustin/go-humanize v1.0.1
6566
github.com/failsafe-go/failsafe-go v0.9.5
6667
github.com/fatih/color v1.18.0
6768
github.com/fbiville/markdown-table-formatter v0.3.0

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,8 @@ github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c h1:+Zo5Ca9
126126
github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c/go.mod h1:HJGU9ULdREjOcVGZVPB5s6zYmHi1RxzT71l2wQyLmnE=
127127
github.com/dnsimple/dnsimple-go v1.7.0 h1:JKu9xJtZ3SqOC+BuYgAWeab7+EEx0sz422vu8j611ZY=
128128
github.com/dnsimple/dnsimple-go v1.7.0/go.mod h1:EKpuihlWizqYafSnQHGCd/gyvy3HkEQJ7ODB4KdV8T8=
129+
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
130+
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
129131
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
130132
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
131133
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=

0 commit comments

Comments
 (0)