0x00 前言

Distributed tracing, also called distributed request tracing, is a method used to profile and monitor applications. Distributed tracing helps pinpoint where failures occur and what causes poor performance.Distributed tracing is particularly well-suited to debugging and monitoring modern distributed software architectures, such as microservices.

分布式追踪,也被成为分布式请求跟踪,是一种分析和监控应用程序的手段。分布式追踪有助于查明故障发生的位置以及导致性能低下的原因。此外,分布式跟踪十分合适调试和监控现代分布式架构的软件,尤其是那些使用微服务体系结构构建的应用程序

如上述引用所属,分布式追踪在现代分布式架构的软件中起着举足轻重的作用,尤其是在微服务这类调用链冗长且复杂的软件架构中,分布式追踪的出现使得上述分析和监控应用程序等工作变得更加简单且方便。

0x01 OpenTracing

OpenTracing API提供了一个标准的、与供应商无关的框架,这意味着如果开发者想要尝试一种不同的分布式追踪系统,开发者只需要简单地修改Tracer配置即可,而不需要替换整个分布式追踪系统。

OpenTracing 由 API 规范(描述了语言无关的数据模型和 Opentracing API 指南)、实现该规范的框架和库以及项目文档组成,OpenTracing 不是一个标准,OpenTracing API 项目正致力于为分布式跟踪创建更加标准化的 API 和工具。

0x02 Jaeger

Jaeger 是由 Uber 开发的且兼容 OpenTracing API 的分布式追踪系统,用于监控和调试基于微服务体系结构构建的应用程序,主要功能包括:

  • 分布式上下文传播
  • 分布式事务监控
  • 根因分析
  • 服务依赖分析
  • 性能和延迟优化

0x03 测试环境

测试环境使用 Docker 进行测试,测试镜像为 Uber 官方提供的 All-In-One 镜像:

version: '3'
services:
  jaeger:
    container_name: local-jaeger
    image: jaegertracing/all-in-one:1.18
    ports:
      - "5778:5778"
      - "6831-6832:6831-6832/udp"
      - "14250:14250"
      - "14268:14268"
      - "16686:16686"
    networks:
      - local
networks:
  local:

0x04 初始化 Jaeger 并注册为全局 Tracer

func NewGlobalTestTracer() (opentracing.Tracer, io.Closer, error) {
  cfg := jaegercfg.Configuration{
      Sampler: &jaegercfg.SamplerConfig{
        Type:  jaeger.SamplerTypeConst,
        Param: 1,
      },
      Reporter: &jaegercfg.ReporterConfig{
        LogSpans: true,
      },
    }

    // 示例 Logger 和 Metric 分别使用 github.com/uber/jaeger-client-go/log 和 github.com/uber/jaeger-lib/metrics
    jLogger := jaegerlog.StdLogger
    jMetricsFactory := metrics.NullFactory

    // 初始化 Tracer 实例
    tracer, closer, err := cfg.NewTracer(
      "serviceName",
      jaegercfg.Logger(jLogger),
      jaegercfg.Metrics(jMetricsFactory),
      // 设置最大 Tag 长度,根据情况设置
      jaegercfg.MaxTagValueLength(65535),
    )
    if err != nil {
      log.Printf("Could not initialize jaeger tracer: %s", err.Error())
      panic(err)
    }
    return tracer, closer, err
}

0x05 编写 Gin 中间件

func OpenTracing() gin.HandlerFunc {
    return func(c *gin.Context) {
    // 使用 opentracing.GlobalTracer() 获取全局 Tracer
        wireCtx, _ := opentracing.GlobalTracer().Extract(
            opentracing.HTTPHeaders,
            opentracing.HTTPHeadersCarrier(c.Request.Header),
        )

    // OpenTracing Span 概念,详情参见  https://opentracing.io/docs/overview/spans/
        serverSpan := opentracing.StartSpan(
            c.Request.URL.Path,
            ext.RPCServerOption(wireCtx),
        )
        defer serverSpan.Finish()

    // 记录请求 Url
        ext.HTTPUrl.Set(serverSpan, c.Request.URL.Path)
    // Http Method
        ext.HTTPMethod.Set(serverSpan, c.Request.Method)
    // 记录组件名称
        ext.Component.Set(serverSpan, "Gin-Http")
    // 自定义 Tag X-Forwarded-For
        opentracing.Tag{Key: "http.headers.x-forwarded-for", Value: c.Request.Header.Get("X-Forwarded-For")}.Set(serverSpan)
    // 自定义 Tag User-Agent
        opentracing.Tag{Key: "http.headers.user-agent", Value: c.Request.Header.Get("User-Agent")}.Set(serverSpan)
    // 自定义 Tag Request-Time
        opentracing.Tag{Key: "request.time", Value: time.Now().Format(time.RFC3339)}.Set(serverSpan)
    // 自定义 Tag Server-Mode
        opentracing.Tag{Key: "http.server.mode", Value: gin.Mode()}.Set(serverSpan)

        body, err := ioutil.ReadAll(c.Request.Body)
        if err == nil {
      // 自定义 Tag Request-Body
            opentracing.Tag{Key: "http.request_body", Value: string(body)}.Set(serverSpan)
        }

        c.Request = c.Request.WithContext(opentracing.ContextWithSpan(c.Request.Context(), serverSpan))

        c.Next()
        if gin.Mode() == gin.DebugMode {
      // 自定义 Tag StackTrace
            opentracing.Tag{Key: "debug.trace", Value: string(debug.Stack())}.Set(serverSpan)
        }

        ext.HTTPStatusCode.Set(serverSpan, uint16(c.Writer.Status()))
        opentracing.Tag{Key: "request.errors", Value: c.Errors.String()}.Set(serverSpan)
    }
}

0x06 启动 Gin 服务

package main

import (
    "io/ioutil"
    "runtime/debug"
    "time"

    "github.com/gin-gonic/gin"
    "github.com/opentracing/opentracing-go"
    "github.com/opentracing/opentracing-go/ext"
    "github.com/gin-gonic/gin"
)

func main() {
    t, closer, err := NewGlobalTestTracer()
    if err != nil {
        panic(err)
    }
    defer closer.Close()
    opentracing.SetGlobalTracer(t) 
    r := gin.Default()
    // 注入先前编写的中间件
    r.Use(OpenTracing())
    r.GET("/ping", func(c *gin.Context) {
        c.JSON(200, gin.H{
            "message": "pong",
        })
    })
    r.Run()
}

0x07 结果验证

打开 Jaeger UI 进行验证,按照对应的测试环境,Jaeger Web UI 地址为 http://127.0.0.1:16686/search

Tags:
component=Gin-Http
debug.trace=goroutine 60 [running]: runtime/debug.Stack(0xc00017c0f0, 0xc00062c2d0, 0x54a2380) /usr/local/Cellar/go/1.15/libexec/src/runtime/debug/stack.go:24 +0x9f fusion-pc.com/eam/internal/pkg/plugins/jaeger.OpenTracing.func1(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/internal/pkg/plugins/jaeger/jaeger_gin.go:43 +0x81c github.com/gin-gonic/gin.(*Context).Next(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/context.go:161 +0x3b github.com/gin-contrib/zap.RecoveryWithZap.func1(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-contrib/zap/zap.go:109 +0x68 github.com/gin-gonic/gin.(*Context).Next(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/context.go:161 +0x3b github.com/gin-contrib/zap.Ginzap.func1(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-contrib/zap/zap.go:32 +0xce github.com/gin-gonic/gin.(*Context).Next(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/context.go:161 +0x3b github.com/gin-gonic/gin.RecoveryWithWriter.func1(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/recovery.go:83 +0x65 github.com/gin-gonic/gin.(*Context).Next(0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/context.go:161 +0x3b github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xc00053a000, 0xc00017c0f0) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/gin.go:409 +0x67a github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xc00053a000, 0x5487700, 0xc0006281c0, 0xc000372600) /Users/zhouhaowei/Documents/Project/eam/vendor/github.com/gin-gonic/gin/gin.go:367 +0x14d net/http.serverHandler.ServeHTTP(0xc0006280e0, 0x5487700, 0xc0006281c0, 0xc000372600) /usr/local/Cellar/go/1.15/libexec/src/net/http/server.go:2843 +0xa3 net/http.(*conn).serve(0xc0000e0140, 0x548c680, 0xc0005643c0) /usr/local/Cellar/go/1.15/libexec/src/net/http/server.go:1925 +0x8ad created by net/http.(*Server).Serve /usr/local/Cellar/go/1.15/libexec/src/net/http/server.go:2969 +0x36c 
http.headers.user-agent=PostmanRuntime/7.26.3
http.headers.x-forwarded-for=
http.method=GET
http.request_body=
http.server.mode=debug
http.status_code=200
http.url=/ping
internal.span.format=proto
request.errors=
request.time=2020-09-01T20:55:34+08:00
sampler.param=1
sampler.type=probabilistic
span.kind=server

Process:
client-uuid=8060d5ba910b166
hostname=zhouhaoweis-MacBook-Pro.localip=192.168.31.83
jaeger.version=Go-2.25.0

0x08 延伸: 从配置初始化 Jaeger

package jaeger

import (
    "fmt"
    "strings"
    "time"

    "github.com/google/wire"
    "github.com/opentracing/opentracing-go"
    "github.com/pkg/errors"
    "github.com/spf13/viper"
    "github.com/uber/jaeger-client-go"
    "github.com/uber/jaeger-client-go/config"

    // 使用 zap 作为日志输出
    jaegerZap "github.com/uber/jaeger-client-go/log/zap"
    "go.uber.org/zap"
)

type Sampler struct {
    SamplerType string
    Probability float64
    Rate        int64
}

type Reporter struct {
    QueueSize           int
    LogSpans            bool
    BufferFlushInterval time.Duration
    Username            string
    Password            string
}

type Options struct {
    Enable      bool
    ServiceName string
    Host        string
    Port        uint16
    Sampler     *Sampler
    Reporter    *Reporter
}

const defaultSamplingLimitRate = 1

var NoDefaultTracingConfigError = errors.New("unable to load tracing config")

func NewOptions(v *viper.Viper, logger *zap.Logger) (opts *Options, err error) {
    cfg := v.GetString("tracing.default")
    if len(cfg) <= 0 {
        return nil, NoDefaultTracingConfigError
    }

    opts = new(Options)
    key := fmt.Sprintf("tracing.%s", cfg)
    if err = v.UnmarshalKey(key, opts); err != nil {
        return nil, errors.Wrap(err, "unmarshal tracing options error")
    }

    logger.With(zap.String("type", "tracing")).Info(fmt.Sprintf("load %s tracing options success", cfg))
    return
}

func NewTracer(logger *zap.Logger, opts *Options) (opentracing.Tracer, error) {
    sampleCfg := &config.SamplerConfig{}
    switch strings.ToLower(opts.Sampler.SamplerType) {
    case jaeger.SamplerTypeConst:
        sampleCfg.Type = jaeger.SamplerTypeConst
    case jaeger.SamplerTypeRateLimiting:
        sampleCfg.Type = jaeger.SamplerTypeRateLimiting
        if opts.Sampler.Rate > 0 {
            sampleCfg.Param = float64(opts.Sampler.Rate)
        } else {
            sampleCfg.Param = defaultSamplingLimitRate
        }
    case jaeger.SamplerTypeRemote:
        sampleCfg.Type = jaeger.SamplerTypeRemote
    default:
        sampleCfg.Type = jaeger.SamplerTypeProbabilistic
        if opts.Sampler.Probability > 0 {
            sampleCfg.Param = opts.Sampler.Probability
        }
    }

    reporterCfg := &config.ReporterConfig{
        QueueSize:           opts.Reporter.QueueSize,
        LogSpans:            opts.Reporter.LogSpans,
        BufferFlushInterval: opts.Reporter.BufferFlushInterval,
    }

    if len(opts.Reporter.Username) > 0 && len(opts.Reporter.Password) > 0 {
        reporterCfg.User = opts.Reporter.Username
        reporterCfg.Password = opts.Reporter.Password
    }

    cfg := config.Configuration{
        ServiceName: opts.ServiceName,
        Sampler:     sampleCfg,
        Reporter:    reporterCfg,
    }

    addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port)
    sender, err := jaeger.NewUDPTransport(addr, 0)
    if err != nil {
        return nil, err
    }

    reporter := jaeger.NewRemoteReporter(sender)
    tracer, _, err := cfg.NewTracer(
        config.Reporter(reporter),
        config.Logger(jaegerZap.NewLogger(logger.With(zap.String("type", "tracing")))),
        config.MaxTagValueLength(65535),
    )

    opentracing.SetGlobalTracer(tracer)

    return tracer, err
}

var ProviderSet = wire.NewSet(NewOptions, NewTracer)

0 条评论

发表评论

电子邮件地址不会被公开。 必填项已用*标注