Why Nginx tracing

Nginx是一个广泛使用的web原件，常用于反向代理、负载均衡服务器、API网关。在日常的使用中，nginx常常要负责各种redirect、rewrite等服务，对于业务或者调用关系负责的情况，tracing能够帮助开发者直观分析请求链路，快速定位性能瓶颈，逐渐优化服务间依赖，也有助于开发者从更宏观的角度更好地理解整个系统。

同时，如果开发者能够在代码内部也加入对tracing的支持，那么对于自己开发的服务整体从将可以以上帝视角直观地看到。

环境要求

nginx stable 1.2+

ngonx-opentracing module

jaeger环境

配置详情

1. nginx.conf

load_module modules/ngx_http_opentracing_module.so;
user  nobody nobody;
worker_processes  auto;
working_directory /search/odin/nginx/core;
worker_rlimit_core 2G;

error_log  logs/error.log error;

pid        /var/run/nginx.pid;


events {
    use epoll;
    worker_connections  51200;
}


http {
    include       /etc/nginx/mime.types;
    default_type  Application/octet-stream;

    opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.linux_amd64.so /etc/nginx/conf.d/jaeger-config.json;
    opentracing on;
    opentracing_propagate_context;
    opentracing_operation_name nginx-$host;
    opentracing_tag request.id $request_id;

    index index.html index.htm index.shtml;

    set_real_ip_from   10.0.0.0/8;
    set_real_ip_from   192.168.0.0/16;
    real_ip_header     X-Real-IP;

    log_format mAIn '$host $remote_addr [$time_local] "$request" '
                        '$status $body_bytes_sent "$http_referer" "$http_user_agent" '
                        '$cookie_SUID $request_time $cookie_SSUID '
                        '$http_x_forwarded_for $request_length $cookie_YYID '
                        '$connection_requests "$upstream_addr" $request_id';
    log_format mini '$time_local $status $body_bytes_sent $request_time $upstream_cache_status $server_name';
    access_log "logs/${server_name}_access_log" main;
    access_log logs/status_log mini;


    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    gzip  on;
    gzip_min_length  1024;
    gzip_comp_level  6;
    gzip_types       text/plain text/xml application/x-JAVAscript text/css application/xml;


    include /etc/nginx/conf.d/*.conf;
}

首先，加载nginx 模块：
ngx_http_opentracing_module.so

load_module modules/ngx_http_opentracing_module.so;

其次，加载并配置jaeger plugin

opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.linux_amd64.so /etc/nginx/conf.d/jaeger-config.json;
opentracing on; # 启用opentracing，默认为off
opentracing_propagate_context; # 启用传递context
opentracing_operation_name nginx-$host; # 为请求起个name
opentracing_tag request.id $request_id; # 设置span tag request.id，需要nginx 1.11以上

其中专门设置了一个request.id的tag，这个request.id串起来从nginx到code的所有请求，方便检索请求

最后，附加一个jaeger的配置，localAgentHostPort需要配置成jaeger-agent所在host:port

{
  "service_name": "nginx",
  "diabled": false,
  "reporter": {
    "logSpans": true,
    "localAgentHostPort": "jaeger:6831"
  },
  "sampler": {
    "type": "const",
    "param": "1"
  }
}

2. server.conf

upstream api{
    server 127.0.0.1:8080;
}

server
{
    listen *:80 default_server;
    server_name xx.test;

    opentracing_location_operation_name $request;
    opentracing_propagate_context;
    proxy_set_header X-Request-Id $request_id;

    proxy_set_header  Host $host;
    proxy_http_version 1.1;

    location ~ ^/api/ {
      proxy_pass http://api;
    }
}

以下三条配置分别设置了location name、启用传递context、设置header X-Request-Id为 $request_id

opentracing_location_operation_name $request;
opentracing_propagate_context;
proxy_set_header X-Request-Id $request_id;

3. 代码接入opentracing

import (
.....
opentracing "Github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
)

func (c *AlertManagerController) Webhook() {
	spanCtx, _ := opentracing.GlobalTracer().Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(c.Ctx.Request.Header))
	span := opentracing.GlobalTracer().StartSpan("Webhook", ext.RPCServerOption(spanCtx)) // 关联为nginx的child
	uuid := c.Ctx.Input.Header("X-Request-Id")
	span.SetTag("request.id", uuid) // 设置request tag
	span.SetBaggageItem("request.id", uuid) // 向后续请求加入request.id
	span.LogKV("request.body", string(c.Ctx.Input.RequestBody))
        ......
}

​

​

为什么要用nginx 1.2以上？

nginx opentracing require nginx 1.9.13+

request_id require nginx 1.11.0

nginx 偶数版本是stable

附模块下载地址

ngx_http_opentracing_module.so https://github.com/opentracing-contrib/nginx-opentracing/releases

jaeger plugin https://github.com/jaegertracing/jaeger-client-cpp/releases

参考

https://github.com/opentracing-contrib/nginx-opentracing

https://medium.com/opentracing/how-to-enable-nginx-for-distributed-tracing-9479df18b22c

https://github.com/opentracing-contrib/nginx-opentracing/tree/master/example/trivial/jaeger