mm: get uncached internal mappings in the read-locked I/O fast path - #16565

// crio measures syscall copy throughput into a large anonymous mapping across
// checkpoint/restore.
//
// Hot threads repeatedly call getcpu(2) with output pointers in a fixed page
// of the mapping (one page per thread, spread across the mapping), so each call
// does two small CopyOuts into the mapping. Optional walker threads, started
// after restore, call getcpu(2) once per 2 MiB across the whole mapping, so
// they keep touching pages that may not have been loaded yet.
//
// A reporter samples counters every -interval and detects restore as a
// wall-clock gap. It writes the timeline to -out after -after seconds past
// restore and exits.
package main

import (
	"flag"
	"fmt"
	"math/bits"
	"os"
	"runtime"
	"sync/atomic"
	"syscall"
	"time"
	"unsafe"
)

var (
	heapGiB  = flag.Int("heap", 16, "mapping size in GiB")
	hot      = flag.Int("hot", 8, "hot threads")
	walkers  = flag.Int("walk", 0, "walker threads started after restore")
	after    = flag.Duration("after", 20*time.Second, "run time after restore")
	interval = flag.Duration("interval", 50*time.Millisecond, "sample interval")
	outDir   = flag.String("out", "/out", "output directory")
	latWin   = flag.Duration("latwin", 20*time.Second, "latency window after restore")
)

const (
	sysGetcpu = 309 // amd64

	pageSize = 4096
	walkStep = 2 << 20
)

type counter struct {
	n atomic.Uint64
	_ [56]byte
}

// hist is a per-thread histogram of getcpu(2) latencies in log2(ns) buckets,
// for the steady state before checkpoint and for the window after restore.
type hist [2][64]uint64

// phase is 0 before restore, 1 within -latwin after restore, and 2 after.
var phase atomic.Int32

func getcpu(p uintptr) {
	syscall.Syscall(sysGetcpu, p, p+4, 0)
}

func main() {
	flag.Parse()
	size := *heapGiB << 30
	heap, err := syscall.Mmap(-1, 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_PRIVATE|syscall.MAP_ANON|syscall.MAP_POPULATE)
	if err != nil {
		panic(err)
	}
	base := uintptr(unsafe.Pointer(&heap[0]))

	// Make every page non-zero, so that checkpoint saves it.
	fillStart := time.Now()
	nfill := runtime.NumCPU()
	done := make(chan struct{})
	for i := 0; i < nfill; i++ {
		go func(i int) {
			chunk := size / nfill
			for off := i * chunk; off < (i+1)*chunk; off += pageSize {
				heap[off] = byte(off>>12) | 1
			}
			done <- struct{}{}
		}(i)
	}
	for i := 0; i < nfill; i++ {
		<-done
	}
	fmt.Fprintf(os.Stderr, "filled %d GiB in %v\n", *heapGiB, time.Since(fillStart))

	hotCtrs := make([]counter, *hot)
	hists := make([]hist, *hot)
	stride := uintptr(size / *hot)
	for i := 0; i < *hot; i++ {
		go func(i int) {
			runtime.LockOSThread()
			p := base + uintptr(i)*stride + stride/2
			for {
				t := time.Now()
				getcpu(p)
				d := time.Since(t)
				if ph := phase.Load(); ph < 2 {
					hists[i][ph][bits.Len64(uint64(d))]++
				}
				hotCtrs[i].n.Add(1)
			}
		}(i)
	}

	var restored atomic.Bool
	walkCtrs := make([]counter, *walkers)
	for i := 0; i < *walkers; i++ {
		go func(i int) {
			runtime.LockOSThread()
			for !restored.Load() {
				time.Sleep(time.Millisecond)
			}
			// Walkers start at different offsets and wrap around.
			start := uintptr(i) * uintptr(size / *walkers)
			for off := uintptr(0); off < uintptr(size); off += walkStep {
				getcpu(base + (start+off)%uintptr(size) + 2*pageSize)
				walkCtrs[i].n.Add(1)
			}
		}(i)
	}

	type sample struct {
		t           int64
		hot, walked uint64
	}
	samples := make([]sample, 0, 1<<16)
	var restoreAt int64
	readyAt := time.Now().Add(3 * time.Second)
	readyWritten := false
	last := time.Now().UnixNano()
	for {
		time.Sleep(*interval)
		now := time.Now().UnixNano()
		var s sample
		s.t = now
		for i := range hotCtrs {
			s.hot += hotCtrs[i].n.Load()
		}
		for i := range walkCtrs {
			s.walked += walkCtrs[i].n.Load()
		}
		samples = append(samples, s)
		if restoreAt == 0 && now-last > int64(time.Second) {
			restoreAt = now
			restored.Store(true)
			phase.Store(1)
		}
		if restoreAt != 0 && now-restoreAt > int64(*latWin) {
			phase.Store(2)
		}
		last = now
		if !readyWritten && time.Now().After(readyAt) {
			os.WriteFile(*outDir+"/ready", nil, 0644)
			readyWritten = true
		}
		if restoreAt != 0 && now-restoreAt > int64(*after) {
			break
		}
	}

	f, err := os.Create(*outDir + "/timeline.txt")
	if err != nil {
		panic(err)
	}
	fmt.Fprintf(f, "# restore_at_ns %d hot %d walkers %d heap_gib %d\n", restoreAt, *hot, *walkers, *heapGiB)
	for _, s := range samples {
		fmt.Fprintf(f, "%d %d %d\n", s.t, s.hot, s.walked)
	}
	f.Close()

	f, err = os.Create(*outDir + "/lat.txt")
	if err != nil {
		panic(err)
	}
	for ph := 0; ph < 2; ph++ {
		var sum [64]uint64
		for i := range hists {
			for b := range sum {
				sum[b] += hists[i][ph][b]
			}
		}
		fmt.Fprintf(f, "%d", ph)
		for _, n := range sum {
			fmt.Fprintf(f, " %d", n)
		}
		fmt.Fprintln(f)
	}
	f.Close()
}