How to push traces Observability

Create Traces

To push traces to Observability we first need to create them with an app.

We need to do the following steps:

• Extract the span context from the request header.
• Create a new span.
• Finish the span.
• Inject the new span.

Below are examples for Golangs Fiber framework and Python Falcon framework:

Golang Fiber Tracing Middleware

package middlewares

import (
  "observability-limit-api/internal/settings"
  "fmt"
  "github.com/opentracing/opentracing-go/ext"
  "strings"

  fiber "github.com/gofiber/fiber/v2"
  opentracing "github.com/opentracing/opentracing-go"
  _ "github.com/opentracing/opentracing-go/ext"
  jaeger "github.com/uber/jaeger-client-go"
  jaegercfg "github.com/uber/jaeger-client-go/config"
  jaegerlog "github.com/uber/jaeger-client-go/log"
  "github.com/uber/jaeger-lib/metrics/prometheus"
)

// init jaeger tracer.
var cfg = jaegercfg.Configuration{
  Sampler: &jaegercfg.SamplerConfig{
    Type: jaeger.SamplerTypeConst,
    Param: 1,
  },
  Reporter: &jaegercfg.ReporterConfig{
    LogSpans: true,
    LocalAgentHostPort: fmt.Sprintf("%s:%s", settings.JaegerService, settings.JaegerPort),
  },
}

var (
  jLogger = jaegerlog.StdLogger
  MetricsFactory = prometheus.New()
)

var _, _ = cfg.InitGlobalTracer(
  "test-app",
  jaegercfg.Logger(jLogger),
  jaegercfg.Metrics(MetricsFactory),
)
var tracer = opentracing.GlobalTracer()

// TracingMiddleware to trace requests.
func TracingMiddleware(c *fiber.Ctx) error {
  // build a header map from fiber req headers to extract jaeger header
  headerMap:= getHeaderMap(c.GetReqHeaders())

  // build carrier to build tracing chain
  carrier:= opentracing.HTTPHeadersCarrier(headerMap)

  spanCtx, _:= tracer.Extract(opentracing.HTTPHeaders, carrier)

  // start span
  span:= tracer.StartSpan("tracing middleware", ext.RPCServerOption(spanCtx))

  err:= tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier)
  if err!= nil {
    return err
  }

  // get carrier header to set it in response headers
  for key, el:= range carrier {
    if strings.ToLower(key) == strings.ToLower(jaeger.TraceContextHeaderName) {
      if len(el) > 0 {
        c.Response().Header.Add(jaeger.TraceContextHeaderName, el[0])
        c.Request().Header.Add(jaeger.TraceContextHeaderName, el[0])
      }
    }
  }

  // send request to route
  err = c.Next()
  if err!= nil {
    return err
  }

  // append meta data to span
  ext.HTTPMethod.Set(span, c.Method())
  ext.HTTPUrl.Set(span, c.OriginalURL())
  ext.HTTPStatusCode.Set(span, uint16(c.Response().StatusCode()))

  // finish and inject span
  span.Finish()
  err = tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier)
  if err!= nil {
    return err
  }

  return nil
}

// getHeaderMap get http headers format.
func getHeaderMap(headers map[string]string) map[string][]string {
  headerMap:= make(map[string][]string)

  for key, element:= range headers {
    if strings.EqualFold(strings.ToLower(key), strings.ToLower(jaeger.TraceContextHeaderName)) {
      headerMap[jaeger.TraceContextHeaderName] = []string{element}
    }
  }

  return headerMap
}

Python Falcon Tracing Middleware

"""Tracing middleware."""
import os
import time
from typing import Dict

import falcon
import jaeger_client
from jaeger_client.constants import TRACE_ID_HEADER
from opentracing import Format, tags

class TracingMiddleware:
 """Tracing middleware."""

 now = time.time()
 tracer = jaeger_client.Config(
 config={
 "sampler": {"type": "const", "param": 1},
 "local_agent": {
 "reporting_port": int(
 os.environ.get(
 "JAEGER_AGENT_PORT", jaeger_client.config.DEFAULT_REPORTING_PORT
 )
 ),
 "reporting_host": os.environ.get("JAEGER_AGENT_HOST", "jaeger"),
 },
 "logging": bool(os.environ.get("JAEGER_LOGGING", True)),
 },
 service_name=os.environ.get("JAEGER_SERVICE_NAME", "test-app"),
 validate=True,
 ).initialize_tracer()

 def process_request(self, req: falcon.Request, resp: falcon.Response):
 """Process request before route enter."""
 self.now = time.time()

 def process_response(
 self, req: falcon.Request, resp: falcon.Response, resource, req_succeeded
 ):
 """Process response."""
 span_ctx = self.tracer.extract(Format.HTTP_HEADERS, req.headers)
 span_tags = {tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER}
 with self.tracer.start_span(
 operation_name="test-app-response",
 child_of=span_ctx,
 tags=span_tags,
 start_time=self.now,
 ) as gate_span:
 self.inject_span(gate_span, req, resp)

 def inject_span(self, span, request: falcon.Request, resp: falcon.Response):
 """Inject span."""
 carrier: Dict[str, str] = {}
 span.set_tag(tags.HTTP_METHOD, request.method)
 span.set_tag(tags.HTTP_URL, request.path)
 span.set_tag(tags.SPAN_KIND, tags.SPAN_KIND_RPC_CLIENT)
 span.set_tag(tags.HTTP_STATUS_CODE, resp.status)
 self.tracer.inject(span, Format.HTTP_HEADERS, carrier)
 resp.set_header(TRACE_ID_HEADER, carrier[TRACE_ID_HEADER])

Setup an Agent

Next we need an agent in the same network as the app since traces from the app are sent via UDP to the agent. A good option is the Opentelemetry Exporter since it can receive all kinds of traces and export them in all kind of formats.

Example:

Opentelemetry Exporter Config

receivers:
 jaeger:
 protocols:
 thrift_compact:

processors:
 batch:

exporters:
 otlp:
 # To get the URL, make a GET request to the observability API with the path /v1/projects/{projectId}/instances/{instanceId}. The URL can be found in the "otlpTracesUrl" key of the "instance" dictionary in the response
    endpoint: <instance>-op.traces.testing.observability.eu01.stackit.cloud:443
     headers:
 authorization: Basic dXNlcm5hbWU6cGFzc3dvcmQ=

service:
 pipelines:
 traces:
 receivers: [jaeger]
 processors: [batch]
 exporters: [otlp]

Opentelemetry Config Zipkin

receivers:
 jaeger:
 protocols:
 thrift_compact:

processors:
 batch:

exporters:
 zipkin:
 endpoint: "https://<instance>-zk.traces.testing.observability.eu01.stackit.cloud/api/v2/spans"
 headers:
 authorization: Basic dXNlcm5hbWU6cGFzc3dvcmQ=

service:
 pipelines:
 traces:
 receivers: [jaeger]
 processors: [batch]
 exporters: [zipkin]

Currently Observability supports sending traces with the jaeger, otlp or zipkin format. The above examples shows how to send traces over otlp with grpc and zipkin with http. We can recommend sending traces over grpc for better performance.

Example docker-compose:

docker-compose.yaml

version: "3"

services:
 hotrod:
 image: jaegertracing/example-hotrod:latest
 ports:
 - "8080:8080"
 command: ["all"]
 environment:
 - JAEGER_AGENT_HOST=collector
 - JAEGER_AGENT_PORT=6831

 collector:
 image: otel/opentelemetry-collector:0.23.0
 command: "--config /etc/otel-config.yaml"
 volumes:
 -./otel-config.yaml:/etc/otel-config.yaml

In this example we are using the hotrod example to generate example traces and send them to the collector. otel-config.yaml is the yaml from the examples above.

Check Traces

Now you can go to Grafana and select the datasource “Tempo” or “Jaeger”. In the Jaeger datasource it is important to set a limit to 20. Observability uses Tempo as traces collector and this tool does not have indexing of traces. This makes it hard to find the traces. The recommended way to check traces is to use the “Tempo” datasource and log the traces. These logs need to go to the Loki of Observability.