Go Project Layout

and Practice
Bo-Yi Wu
2019.08.29
ModernWeb
About me
• Software Engineer in Mediatek

• Member of Drone CI/CD Platform

• Member of Gitea Platform

• Member of Gin Golang Framework

• Teacher of Udemy Platform: Golang + Drone

https://blog.wu-boy.com
Agenda
• Go in Mediatek

• Go Project Layout

• Go Practices

• RESTful api and GraphQL

• Model testing (Postgres, SQLite, MySQL)

• Software Quality

• Data Metrics

• Go Testing
Tech Stack

• Initial Project using Go in 2018/01

• Golang

• Easy to Learn

• Performance

• Deployment
 ├── api

Repository folder ├──

│
assets
└── dist
├── cmd
│ └── ggz
• api ├──
├──
configs
docker
│ ├── server
└── pkg
• assets ├── config
├── errors
├── fixtures
• cmd ├── helper
├── middleware
│ ├── auth
│ └── header
• configs ├── model
├── module
│ ├── mailer
• docker │
│
├── metrics
└── storage
├── router
│ └── routes
• pkg ├── schema
└── version
 Root folder
• .drone.yml (deploy config)

• .revive.toml (golint config)

• docker-compose.yml (DB, Redis and UI)

• Makefile

• go module config (go.mod and go.sum)

• .env.example
 Go Module
https://blog.golang.org/using-go-modules
 Improve Deployment
Using Go Module Proxy
https://github.com/gomods/athens
save time
with proxy
97s -> 6s
Makefile
Build, Testing, Deploy
GOFMT ?= gofmt "-s"
GO ?= go
TARGETS ?= linux darwin windows
ARCHS ?= amd64 386
BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
GOFILES := $(shell find . -name "*.go" -type f)
TAGS ?= sqlite sqlite_unlock_notify

ifneq ($(shell uname), Darwin)

EXTLDFLAGS = -extldflags "-static" $(null)
else
EXTLDFLAGS =
endif

ifneq ($(DRONE_TAG),)
VERSION ?= $(subst v,,$(DRONE_TAG))
else
VERSION ?= $(shell git describe --tags --always')
endif
.env
GGZ_DB_DRIVER=mysql
GGZ_DB_USERNAME=root
GGZ_DB_PASSWORD=123456
GGZ_DB_NAME=ggz
GGZ_DB_HOST=127.0.0.1:3307
GGZ_SERVER_ADDR=:8080
GGZ_DEBUG=true
GGZ_SERVER_HOST=http://localhost:8080
GGZ_STORAGE_DRIVER=disk
GGZ_MINIO_ACCESS_ID=xxxxxxxx
GGZ_MINIO_SECRET_KEY=xxxxxxxx
GGZ_MINIO_ENDPOINT=s3.example.com
GGZ_MINIO_BUCKET=example
GGZ_MINIO_SSL=true
GGZ_AUTH0_DEBUG=true
docker-compose.yml
db:
image: mysql
restart: always
Development
volumes:
- mysql-data:/var/lib/mysql
environment:
MYSQL_USER: example
MYSQL_PASSWORD: example
MYSQL_DATABASE: example
MYSQL_ROOT_PASSWORD: example

minio:
image: minio/minio
restart: always
ports:
volumes:
- minio-data:/data
environment:
MINIO_ACCESS_KEY: minio123456
MINIO_SECRET_KEY: minio123456
command: server /data
api: Production
image: foo/bar
restart: always
ports:
- 8080:8080
environment:
- GGZ_METRICS_TOKEN=test-prometheus-token
- GGZ_METRICS_ENABLED=true
labels:
- "traefik.enable=true"
- "traefik.basic.frontend.rule=Host:${WEB_HOST}"
- "traefik.basic.protocol=http"
Version
Compile version info into Go binary
Version

• -X github.com/go-ggz/ggz/pkg/
version.Version=$(VERSION)

• -X github.com/go-ggz/ggz/pkg/
version.BuildDate=$(BUILD_DATE)

go build -o bin/api -ldflags

var (
// Version number for git tag.
Version string
// BuildDate is the ISO 8601 day drone was built.
BuildDate string
)

// PrintCLIVersion print server info

func PrintCLIVersion() string {
return fmt.Sprintf(
"version %s, built on %s, %s",
Version,
BuildDate,
runtime.Version(),
)
}
BUILD_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
ifneq ($(DRONE_TAG),)
VERSION ?= $(subst v,,$(DRONE_TAG))
else
VERSION ?= $(shell git describe --tags --always | sed 's/-/+/' | sed
's/^v//')
endif
Assets
Embed files in Go

https://github.com/UnnoTed/fileb0x
func ReadSource(origPath string) (content []byte, err error) {
content, err = ReadFile(origPath)

if err != nil {
log.Warn().Err(err).Msgf("Failed to read builtin %s file.", origPath)
}
Debug Setting
if config.Server.Assets != "" && file.IsDir(config.Server.Assets) {
origPath = path.Join(config.Server.Assets, origPath)

if file.IsFile(origPath) {
content, err = ioutil.ReadFile(origPath)
if err != nil {
log.Warn().Err(err).Msgf("Failed to read custom %s file", origPath)
}
}
}

return content, err

}
// ViewHandler support dist handler from UI
func ViewHandler() gin.HandlerFunc {
fileServer := http.FileServer(dist.HTTP)
data := []byte(time.Now().String())
etag := fmt.Sprintf("%x", md5.Sum(data))

return func(c *gin.Context) {

c.Header("Cache-Control", "public, max-age=31536000")
c.Header("ETag", etag)

if match := c.GetHeader("If-None-Match"); match != "" {

if strings.Contains(match, etag) {
c.Status(http.StatusNotModified)
return
}
}
File Server Handler

fileServer.ServeHTTP(c.Writer, c.Request)
}
}
// Favicon represents the favicon.
func Favicon(c *gin.Context) {
file, _ := dist.ReadFile("favicon.ico")
etag := fmt.Sprintf("%x", md5.Sum(file))
c.Header("ETag", etag)
c.Header("Cache-Control", "max-age=0")NO Cache

if match := c.GetHeader("If-None-Match"); match != "" {

if strings.Contains(match, etag) {
c.Status(http.StatusNotModified)
return
}
}

c.Data(
http.StatusOK,
"image/x-icon",
file,
)
}
API
/healthz
• health check for load balancer

func Heartbeat(c *gin.Context) {

c.AbortWithStatus(http.StatusOK)
c.String(http.StatusOK, "ok")
}
CMD
Command line
Command line package

• Golang package: flag

• urfave/cli

• spf13/cobra
├── agent
│ ├── config
│ │ └── config.go
│ └── main.go
├── notify
│ └── main.go
└── tcp-server
├── config
│ └── config.go
└── main.go
 Config
Management
github.com/spf13/viper
Config management

• Load config from File

• .json

• .ini

• Load config from Environment Variables

• .env
 var envfile string
flag.StringVar(&envfile, "env-file", ".env", "Read in a file of environment
variables")
flag.Parse()

godotenv.Load(envfile)

_ "github.com/joho/godotenv/autoload"
Logging struct {
Debug bool `envconfig:"GGZ_LOGS_DEBUG"`
Level string `envconfig:"GGZ_LOGS_LEVEL" default:"info"`
Color bool `envconfig:"GGZ_LOGS_COLOR"`
Pretty bool `envconfig:"GGZ_LOGS_PRETTY"`
Text bool `envconfig:"GGZ_LOGS_TEXT"`
}

// Server provides the server configuration.

Server struct {
Addr string `envconfig:"GGZ_SERVER_ADDR"`
Port string `envconfig:"GGZ_SERVER_PORT" default:"12000"`
Path string `envconfig:”GGZ_SERVER_PATH" default:"data"`
}
github.com/kelseyhightower/envconfig
config, err := config.Environ()
if err != nil {
log.Fatal().
Err(err).
Msg("invalid configuration")
}

initLogging(config)

// check folder exist Load env from structure

if !file.IsDir(config.Server.Path) {
log.Fatal().
Str("path", config.Server.Path).
Msg("log folder not found")
}
/configs
Configuration file templates or default config
global:
scrape_interval: 5s
external_labels:
monitor: 'my-monitor'
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'ggz-server'
static_configs:
- targets: ['ggz-server:8080']
bearer_token: 'test-prometheus-token'
/docker
Docker file template
├── ggz-redirect
│ ├── Dockerfile.linux.amd64
│ ├── Dockerfile.linux.arm
│ ├── Dockerfile.linux.arm64
│ ├── Dockerfile.windows.amd64
│ └── manifest.tmpl
└── ggz-server
├── Dockerfile.linux.amd64
├── Dockerfile.linux.arm
├── Dockerfile.linux.arm64
├── Dockerfile.windows.amd64
└── manifest.tmpl
/integrations
ctx := context.Background()
req := testcontainers.ContainerRequest{
Image: "goggz/ggz-server",
ExposedPorts: []string{"8080/tcp"},
WaitingFor: wait.ForLog("Starting shorten server on :8080")
}
ggzServer, err := testcontainers.GenericContainer(
ctx,
testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})

if err != nil {
t.Fatal(err)
}
github.com/testcontainers/testcontainers-go
/pkg
├── config
├── errors
├── fixtures
├── helper
├── middleware
│ ├── auth
│ └── header
├── model
├── module
│ ├── metrics
│ └── storage
│ ├── disk
│ └── minio
├── router
│ └── routes
├── schema
└── version
/pkg/errors
// Type defines the type of an error
type Type string

const (
// Internal error
Internal Type = "internal"
// NotFound error means that a specific item does not exis
NotFound Type = "not_found"
// BadRequest error
BadRequest Type = "bad_request"
// Validation error
Validation Type = "validation"
// AlreadyExists error
AlreadyExists Type = "already_exists"
// Unauthorized error
Unauthorized Type = "unauthorized"
)
// ENotExists creates an error of type NotExist
func ENotExists(msg string, err error, arg ...interface{}) error {
return New(NotFound, fmt.Sprintf(msg, arg...), err)
}

// EBadRequest creates an error of type BadRequest

func EBadRequest(msg string, err error, arg ...interface{}) error {
return New(BadRequest, fmt.Sprintf(msg, arg...), err)
}

// EAlreadyExists creates an error of type AlreadyExists

func EAlreadyExists(msg string, err error, arg ...interface{}) error {
return New(AlreadyExists, fmt.Sprintf(msg, arg...), err)
}
/pkg/fixtures
Rails-like test fixtures
Write tests against a real database
github.com/go-testfixtures/testfixtures
fixtures/
posts.yml
comments.yml
tags.yml
posts_tags.yml
users.yml
-
id: 1
email: [email protected]
full_name: test
avatar: http://example.com
avatar_email: [email protected]

-
id: 2
email: [email protected]
full_name: test1234
avatar: http://example.com
avatar_email: [email protected]
Unit Testing with Database
func TestMain(m *testing.M) {
// test program to do extra
setup or teardown before or after
testing.
os.Exit(m.Run())
}

https://golang.org/pkg/testing/#hdr-Main
func MainTest(m *testing.M, pathToRoot string) {
var err error
fixturesDir := filepath.Join(pathToRoot, "pkg", "fixtures")
if err = createTestEngine(fixturesDir); err != nil {
fatalTestError("Error creating test engine: %v\n", err)
}
os.Exit(m.Run())
}

func createTestEngine(fixturesDir string) error {

var err error
x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared")
if err != nil {
return err
Testing with SQLite
}

x.ShowSQL(config.Server.Debug)

return InitFixtures(&testfixtures.SQLite{}, fixturesDir)

}
func TestIsUserExist(t *testing.T) {
assert.NoError(t, PrepareTestDatabase())
exists, err := IsUserExist(0, "[email protected]")
assert.NoError(t, err)
assert.True(t, exists)

exists, err = IsUserExist(0, "[email protected]")

assert.NoError(t, err)
assert.False(t, exists)

exists, err = IsUserExist(1, "[email protected]")

assert.NoError(t, err)
assert.True(t, exists)

exists, err = IsUserExist(1, "[email protected]")

assert.NoError(t, err)
assert.False(t, exists)
} go test -v -run=TestIsUserExist ./pkg/models/
/pkg/helper
Helper func

• Encrypt and Decrypt

• Regexp func

• IsEmail, IsUsername

• Zipfile
/pkg/middleware
func Secure(c *gin.Context) {
c.Header("Access-Control-Allow-Origin", "*")
c.Header("X-Frame-Options", "DENY")
c.Header("X-Content-Type-Options", "nosniff")
c.Header("X-XSS-Protection", "1; mode=block")
if c.Request.TLS != nil {
c.Header("Strict-Transport-Security", "max-age=31536000")
}
}
/pkg/model
Use gorm or xorm
Build in
SQLite3
// +build sqlite
go build -v -tags 'sqlite sqlite_unlock_notify'

package model

import (
_ "github.com/mattn/go-sqlite3"
)

func init() {
EnableSQLite3 = true
}
/pkg/module
├── cron
├── download
├── jwt
├── ldap
├── mailer
│ ├── ses
│ └── smtp
├── queue
├── metrics
├── redis
└── storage
├── disk
└── minio
 Integration with
Prometheus + Grafana
func NewCollector() Collector {
return Collector{
Users: prometheus.NewDesc(
namespace+"users",
"Number of Users",
nil, nil,
),
}

// Collect returns the metrics with values

func (c Collector) Collect(ch chan<- prometheus.Metric) {
stats := model.GetStatistic()

ch <- prometheus.MustNewConstMetric(
c.Users,
prometheus.GaugeValue,
float64(stats.Counter.User),
)
}
Prometheus Handler
func Metrics(token string) gin.HandlerFunc {
h := promhttp.Handler()
return func(c *gin.Context) {
if token == "" {
h.ServeHTTP(c.Writer, c.Request)
return
}
header := c.Request.Header.Get("Authorization")
if header == "" {
c.String(http.StatusUnauthorized, errInvalidToken.Error())
return
}
bearer := fmt.Sprintf("Bearer %s", token)
if header != bearer {
c.String(http.StatusUnauthorized, errInvalidToken.Error())
return
}
h.ServeHTTP(c.Writer, c.Request)
}
}
c := metrics.NewCollector()
prometheus.MustRegister(c)
if config.Metrics.Enabled {
root.GET("/metrics", router.Metrics(config.Metrics.Token))
}
Your prometheus token
/pkg/schema
RESTful vs GraphQL
See the Slide: GraphQL in Go
var rootQuery = graphql.NewObject(
graphql.ObjectConfig{
Name: "RootQuery",
Description: "Root Query",
Fields: graphql.Fields{
"queryShortenURL": &queryShortenURL,
"queryMe": &queryMe,
},
})

var rootMutation = graphql.NewObject(

graphql.ObjectConfig{
Name: "RootMutation",
Description: "Root Mutation",
Fields: graphql.Fields{
"createUser": &createUser,
},
})

// Schema is the GraphQL schema served by the server.

var Schema, _ = graphql.NewSchema(
graphql.SchemaConfig{
Query: rootQuery,
Mutation: rootMutation,
})
Write the GraphQL Testing
 assert.NoError(t, model.PrepareTestDatabase())
t.Run("user not login", func(t *testing.T) {
test := T{
Query: `{
queryMe {
email
}
}`,
Schema: Schema,
Expected: &graphql.Result{
Data: map[string]interface{}{
"queryMe": nil,
},
Errors: []gqlerrors.FormattedError{
{
Message: errorYouAreNotLogin,
},
},
},
}
})
}
 Best Practice
Testing your Go code
Testable Code

• Code Quality

• Readability

• Maintainability

• Testability
 #1. Testing in Go

go test package_name

func TestFooBar(t *testing.T) {}

func ExampleFooBar(t *testing.T) {}
func BenchmarkFooBar(b *testing.B) {}
#2. Benchmark Testing
Profiling: CPU, Memory, Goroutine Block
func BenchmarkPlaylyfeGraphQLMaster(b *testing.B) {
for i := 0; i < b.N; i++ {
context := map[string]interface{}{}
variables := map[string]interface{}{}
playlyfeExecutor.Execute(context, "{hello}", variables, "")
}
}

func BenchmarkGophersGraphQLMaster(b *testing.B) {

for i := 0; i < b.N; i++ {
ctx := context.Background()
variables := map[string]interface{}{}
gopherSchema.Exec(ctx, "{hello}", "", variables)
}
} http://bit.ly/2L0CG3Q
#3. Example Testing
Examples on how to use your code
func ExampleFooBar() {
fmt.Println(strings.Compare("a", "b"))
fmt.Println(strings.Compare("a", "a"))
fmt.Println(strings.Compare("b", "a"))
// Output:
// -1
// 0
// 1
}
$ go test -v -tags=sqlite -run=ExampleFooBar ./pkg/model/...
=== RUN ExampleFooBar
--- PASS: ExampleFooBar (0.00s)
PASS
ok github.com/go-ggz/ggz/pkg/model 0.022s
#4. Subtests in Testing Package

func (t *T) Run(name string, f func(t *T)) bool {}

func (b *B) Run(name string, f func(b *B)) bool {}
 tests := []struct {
name string
fields fields
args args
}{}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := Collector{
Shortens: tt.fields.Shortens,
Users: tt.fields.Users,
}
c.Describe(tt.args.ch)
})
}
}
#5. Skipping Testing
t.Skip()
package metrics

import (
"os"
"testing"
)

func TestSkip(t *testing.T) {

if os.Getenv("DEBUG_MODE") == "true" {
t.Skipf("test skipped")
}
}
#6. Running Tests in Parallel
Speedup your CI/CD Flow
t.Parallel()
func TestFooBar01(t *testing.T) {
t.Parallel()
time.Sleep(time.Second)
}
func TestFooBar02(t *testing.T) {
t.Parallel()
time.Sleep(time.Second * 2)
}
func TestFooBar03(t *testing.T) {
t.Parallel()
time.Sleep(time.Second * 3)
}
Just only use
one package
github.com/stretchr/testify
https://www.udemy.com/course/golang-fight/?couponCode=GOLANG2019
https://www.udemy.com/course/devops-oneday/?couponCode=DRONE2019
END