Merge pull request #32 from Shopify/statsd-gometrics

Add go runtime metrics to statsd reporting

Author: grodowski

go/cmd/gh-ost/main.go

@@ -13,6 +13,7 @@ import (
 	"os/signal"
 	"regexp"
 	"syscall"
+	"time"
 
 	"github.com/github/gh-ost/go/base"
 	"github.com/github/gh-ost/go/logic"
@@ -174,6 +175,7 @@ func main() {
 	statsdAddr := flag.String("statsd-addr", "", "StatsD endpoint (host:port or unix socket); empty disables StatsD")
 	var statsdTags statsdTagList
 	flag.Var(&statsdTags, "statsd-tags", "global StatsD tags applied to every metric (repeatable), format key:value. Example: --statsd-tags 'env:prod,service:my-service'")
+	runtimeMetricsInterval := flag.Int("runtime-metrics-interval", 10, "Seconds between Go runtime memory/GC gauge samples (requires --statsd-addr); 0 disables")
 	quiet := flag.Bool("quiet", false, "quiet")
 	verbose := flag.Bool("verbose", false, "verbose")
 	debug := flag.Bool("debug", false, "debug mode (very verbose)")
@@ -400,6 +402,9 @@ func main() {
 	defer func() { _ = metricsClient.Close() }()
 	migrationContext.Metrics = metricsClient
 	metricsClient.Count("startup", 1)
+	if *runtimeMetricsInterval > 0 {
+		metrics.StartGoRuntimeReporter(migrationContext.GetContext(), metricsClient, time.Duration(*runtimeMetricsInterval)*time.Second)
+	}
 
 	migrator := logic.NewMigrator(migrationContext, AppVersion)
 	var err error

go/metrics/go_runtime.go

@@ -0,0 +1,61 @@
+/*
+   Copyright 2022 GitHub Inc.
+	 See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package metrics
+
+import (
+	"context"
+	"runtime"
+	"time"
+)
+
+// MemStatsGaugeEmitter is implemented by *Client; used for tests without UDP.
+type MemStatsGaugeEmitter interface {
+	Gauge(name string, value float64, tags ...string)
+}
+
+// EmitGoRuntimeGauges emits gh_ost.go_runtime.* gauges (namespace is applied by the client).
+// m and numGoroutine are typically from runtime.ReadMemStats and runtime.NumGoroutine.
+func EmitGoRuntimeGauges(emit MemStatsGaugeEmitter, m *runtime.MemStats, numGoroutine int) {
+	if emit == nil || m == nil {
+		return
+	}
+	emit.Gauge("go_runtime.alloc_bytes", float64(m.Alloc))
+	emit.Gauge("go_runtime.sys_bytes", float64(m.Sys))
+	emit.Gauge("go_runtime.heap_inuse_bytes", float64(m.HeapInuse))
+	emit.Gauge("go_runtime.num_gc", float64(m.NumGC))
+	emit.Gauge("go_runtime.gc_pause_total_ns", float64(m.PauseTotalNs))
+	emit.Gauge("go_runtime.goroutines", float64(numGoroutine))
+}
+
+// StartGoRuntimeReporter periodically samples runtime memory and goroutines and emits gauges
+// until ctx is cancelled. It is a no-op when interval <= 0, client is nil, or StatsD is disabled
+// (noop client).
+func StartGoRuntimeReporter(ctx context.Context, client *Client, interval time.Duration) {
+	if ctx == nil || client == nil || interval <= 0 || client.sd == nil {
+		return
+	}
+
+	emit := func() {
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+		EmitGoRuntimeGauges(client, &m, runtime.NumGoroutine())
+	}
+
+	go func() {
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+
+		emit()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				emit()
+			}
+		}
+	}()
+}

go/metrics/go_runtime_test.go

@@ -0,0 +1,67 @@
+/*
+   Copyright 2022 GitHub Inc.
+	 See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package metrics
+
+import (
+	"context"
+	"runtime"
+	"testing"
+	"time"
+)
+
+type gaugeSpy struct {
+	names  []string
+	values []float64
+}
+
+func (g *gaugeSpy) Gauge(name string, value float64, _ ...string) {
+	g.names = append(g.names, name)
+	g.values = append(g.values, value)
+}
+
+func TestEmitGoRuntimeGauges(t *testing.T) {
+	spy := &gaugeSpy{}
+	m := &runtime.MemStats{
+		Alloc:        100,
+		Sys:          200,
+		HeapInuse:    300,
+		NumGC:        7,
+		PauseTotalNs: 42,
+	}
+	EmitGoRuntimeGauges(spy, m, 123)
+
+	wantNames := []string{
+		"go_runtime.alloc_bytes",
+		"go_runtime.sys_bytes",
+		"go_runtime.heap_inuse_bytes",
+		"go_runtime.num_gc",
+		"go_runtime.gc_pause_total_ns",
+		"go_runtime.goroutines",
+	}
+	wantVals := []float64{100, 200, 300, 7, 42, 123}
+
+	if len(spy.names) != len(wantNames) {
+		t.Fatalf("got %d gauges, want %d", len(spy.names), len(wantNames))
+	}
+	for i := range wantNames {
+		if spy.names[i] != wantNames[i] || spy.values[i] != wantVals[i] {
+			t.Fatalf("[%d] got %s=%v want %s=%v", i, spy.names[i], spy.values[i], wantNames[i], wantVals[i])
+		}
+	}
+}
+
+func TestEmitGoRuntimeGauges_nilSafe(t *testing.T) {
+	EmitGoRuntimeGauges(nil, &runtime.MemStats{}, 1)
+	EmitGoRuntimeGauges(&gaugeSpy{}, nil, 1)
+}
+
+func TestStartGoRuntimeReporter_stopsOnCancel(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	c := &Client{} // sd nil — should not start
+	StartGoRuntimeReporter(ctx, c, time.Millisecond)
+	cancel()
+	time.Sleep(20 * time.Millisecond)
+}