From d0b608af3178bc674936f4b1c6cce00591ebbf09 Mon Sep 17 00:00:00 2001 From: Andrii Kushch Date: Tue, 8 Mar 2022 20:18:32 +0100 Subject: tracing: New OpenTelemetry module (#4361) * opentelemetry: create a new module * fix imports * fix test * Update modules/caddyhttp/opentelemetry/README.md Co-authored-by: Dave Henderson * Update modules/caddyhttp/opentelemetry/README.md Co-authored-by: Dave Henderson * Update modules/caddyhttp/opentelemetry/README.md Co-authored-by: Dave Henderson * Update modules/caddyhttp/opentelemetry/tracer.go Co-authored-by: Dave Henderson * rename error ErrUnsupportedTracesProtocol * replace spaces with tabs in the test data * Update modules/caddyhttp/opentelemetry/README.md Co-authored-by: Francis Lavoie * Update modules/caddyhttp/opentelemetry/README.md Co-authored-by: Francis Lavoie * replace spaces with tabs in the README.md * use default values for a propagation and exporter protocol * set http attributes with helper * simplify code * Cleanup modules/caddyhttp/opentelemetry/README.md Co-authored-by: Dave Henderson * Update link in README.md Co-authored-by: Dave Henderson * Update documentation in README.md Co-authored-by: Dave Henderson * Update link to naming spec in README.md Co-authored-by: Dave Henderson * Rename module from opentelemetry to tracing Co-authored-by: Dave Henderson * Rename span_name to span Co-authored-by: Dave Henderson * Rename span_name to span Co-authored-by: Dave Henderson * Simplify otel resource creation Co-authored-by: Dave Henderson * handle extra attributes Co-authored-by: Dave Henderson * update go.opentelemetry.io/otel/semconv to 1.7.0 Co-authored-by: Dave Henderson * update go.opentelemetry.io/otel version * remove environment variable handling * always use tracecontext,baggage as propagators * extract tracer name into variable * rename OpenTelemetry to Tracing * simplify resource creation * update go.mod * rename package from opentelemetry to tracing * cleanup tests * update Caddyfile example in README.md * update README.md * fix test * fix module name in README.md * fix module name in README.md * change names in README.md and tests * order imports * remove redundant tests * Update documentation README.md Co-authored-by: Dave Henderson * Fix grammar Co-authored-by: Dave Henderson * Update comments Co-authored-by: Dave Henderson * Update comments Co-authored-by: Dave Henderson * update go.sum * update go.sum * Add otelhttp instrumentation, update OpenTelemetry libraries. * Use otelhttp instrumentation for instrumenting HTTP requests. This change uses context.WithValue to inject the next handler into the request context via a "nextCall" carrier struct, and pass it on to a standard Go HTTP handler returned by otelhttp.NewHandler. The underlying handler will extract the next handler from the context, call it and pass the returned error to the carrier struct. * use zap.Error() for the error log * remove README.md * update dependencies * clean up the code * change comment * move serveHTTP method from separate file * add syntax to the UnmarshalCaddyfile comment * go import the file * admin: Write proper status on invalid requests (#4569) (fix #4561) * update dependencies Co-authored-by: Dave Henderson Co-authored-by: Francis Lavoie Co-authored-by: Vibhav Pant Co-authored-by: Alok Naushad Co-authored-by: Cedric Ziel --- modules/caddyhttp/tracing/module.go | 121 +++++++++++++++ modules/caddyhttp/tracing/module_test.go | 182 +++++++++++++++++++++++ modules/caddyhttp/tracing/tracer.go | 108 ++++++++++++++ modules/caddyhttp/tracing/tracer_test.go | 27 ++++ modules/caddyhttp/tracing/tracerprovider.go | 63 ++++++++ modules/caddyhttp/tracing/tracerprovider_test.go | 43 ++++++ 6 files changed, 544 insertions(+) create mode 100644 modules/caddyhttp/tracing/module.go create mode 100644 modules/caddyhttp/tracing/module_test.go create mode 100644 modules/caddyhttp/tracing/tracer.go create mode 100644 modules/caddyhttp/tracing/tracer_test.go create mode 100644 modules/caddyhttp/tracing/tracerprovider.go create mode 100644 modules/caddyhttp/tracing/tracerprovider_test.go (limited to 'modules/caddyhttp/tracing') diff --git a/modules/caddyhttp/tracing/module.go b/modules/caddyhttp/tracing/module.go new file mode 100644 index 0000000..7cce669 --- /dev/null +++ b/modules/caddyhttp/tracing/module.go @@ -0,0 +1,121 @@ +package tracing + +import ( + "fmt" + "net/http" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" + "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "go.uber.org/zap" +) + +func init() { + caddy.RegisterModule(Tracing{}) + httpcaddyfile.RegisterHandlerDirective("tracing", parseCaddyfile) +} + +// Tracing implements an HTTP handler that adds support for distributed tracing, +// using OpenTelemetry. This module is responsible for the injection and +// propagation of the trace context. Configure this module via environment +// variables (see https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md). +// Some values can be overwritten in the configuration file. +type Tracing struct { + // SpanName is a span name. It should follow the naming guidelines here: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span + SpanName string `json:"span"` + + // otel implements opentelemetry related logic. + otel openTelemetryWrapper + + logger *zap.Logger +} + +// CaddyModule returns the Caddy module information. +func (Tracing) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.handlers.tracing", + New: func() caddy.Module { return new(Tracing) }, + } +} + +// Provision implements caddy.Provisioner. +func (ot *Tracing) Provision(ctx caddy.Context) error { + ot.logger = ctx.Logger(ot) + + var err error + ot.otel, err = newOpenTelemetryWrapper(ctx, ot.SpanName) + + return err +} + +// Cleanup implements caddy.CleanerUpper and closes any idle connections. It +// calls Shutdown method for a trace provider https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#shutdown. +func (ot *Tracing) Cleanup() error { + if err := ot.otel.cleanup(ot.logger); err != nil { + return fmt.Errorf("tracerProvider shutdown: %w", err) + } + return nil +} + +// ServeHTTP implements caddyhttp.MiddlewareHandler. +func (ot *Tracing) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { + return ot.otel.ServeHTTP(w, r, next) +} + +// UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax: +// +// tracing { +// [span ] +// } +// +func (ot *Tracing) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + setParameter := func(d *caddyfile.Dispenser, val *string) error { + if d.NextArg() { + *val = d.Val() + } else { + return d.ArgErr() + } + if d.NextArg() { + return d.ArgErr() + } + return nil + } + + // paramsMap is a mapping between "string" parameter from the Caddyfile and its destination within the module + paramsMap := map[string]*string{ + "span": &ot.SpanName, + } + + for d.Next() { + args := d.RemainingArgs() + if len(args) > 0 { + return d.ArgErr() + } + + for d.NextBlock(0) { + if dst, ok := paramsMap[d.Val()]; ok { + if err := setParameter(d, dst); err != nil { + return err + } + } else { + return d.ArgErr() + } + } + } + return nil +} + +func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { + var m Tracing + err := m.UnmarshalCaddyfile(h.Dispenser) + return &m, err +} + +// Interface guards +var ( + _ caddy.Provisioner = (*Tracing)(nil) + _ caddyhttp.MiddlewareHandler = (*Tracing)(nil) + _ caddyfile.Unmarshaler = (*Tracing)(nil) +) diff --git a/modules/caddyhttp/tracing/module_test.go b/modules/caddyhttp/tracing/module_test.go new file mode 100644 index 0000000..0fbc05b --- /dev/null +++ b/modules/caddyhttp/tracing/module_test.go @@ -0,0 +1,182 @@ +package tracing + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" +) + +func TestTracing_UnmarshalCaddyfile(t *testing.T) { + tests := []struct { + name string + spanName string + d *caddyfile.Dispenser + wantErr bool + }{ + { + name: "Full config", + spanName: "my-span", + d: caddyfile.NewTestDispenser(` +tracing { + span my-span +}`), + wantErr: false, + }, + { + name: "Only span name in the config", + spanName: "my-span", + d: caddyfile.NewTestDispenser(` +tracing { + span my-span +}`), + wantErr: false, + }, + { + name: "Empty config", + d: caddyfile.NewTestDispenser(` +tracing { +}`), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ot := &Tracing{} + if err := ot.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr { + t.Errorf("UnmarshalCaddyfile() error = %v, wantErrType %v", err, tt.wantErr) + } + + if ot.SpanName != tt.spanName { + t.Errorf("UnmarshalCaddyfile() SpanName = %v, want SpanName %v", ot.SpanName, tt.spanName) + } + }) + } +} + +func TestTracing_UnmarshalCaddyfile_Error(t *testing.T) { + tests := []struct { + name string + d *caddyfile.Dispenser + wantErr bool + }{ + { + name: "Unknown parameter", + d: caddyfile.NewTestDispenser(` + tracing { + foo bar + }`), + wantErr: true, + }, + { + name: "Missed argument", + d: caddyfile.NewTestDispenser(` +tracing { + span +}`), + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ot := &Tracing{} + if err := ot.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr { + t.Errorf("UnmarshalCaddyfile() error = %v, wantErrType %v", err, tt.wantErr) + } + }) + } +} + +func TestTracing_ServeHTTP_Propagation_Without_Initial_Headers(t *testing.T) { + ot := &Tracing{ + SpanName: "mySpan", + } + + req := httptest.NewRequest("GET", "https://example.com/foo", nil) + w := httptest.NewRecorder() + + var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error { + traceparent := request.Header.Get("Traceparent") + if traceparent == "" || strings.HasPrefix(traceparent, "00-00000000000000000000000000000000-0000000000000000") { + t.Errorf("Invalid traceparent: %v", traceparent) + } + + return nil + } + + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()}) + defer cancel() + + if err := ot.Provision(ctx); err != nil { + t.Errorf("Provision error: %v", err) + t.FailNow() + } + + if err := ot.ServeHTTP(w, req, handler); err != nil { + t.Errorf("ServeHTTP error: %v", err) + } +} + +func TestTracing_ServeHTTP_Propagation_With_Initial_Headers(t *testing.T) { + ot := &Tracing{ + SpanName: "mySpan", + } + + req := httptest.NewRequest("GET", "https://example.com/foo", nil) + req.Header.Set("traceparent", "00-11111111111111111111111111111111-1111111111111111-01") + w := httptest.NewRecorder() + + var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error { + traceparent := request.Header.Get("Traceparent") + if !strings.HasPrefix(traceparent, "00-11111111111111111111111111111111") { + t.Errorf("Invalid traceparent: %v", traceparent) + } + + return nil + } + + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()}) + defer cancel() + + if err := ot.Provision(ctx); err != nil { + t.Errorf("Provision error: %v", err) + t.FailNow() + } + + if err := ot.ServeHTTP(w, req, handler); err != nil { + t.Errorf("ServeHTTP error: %v", err) + } +} + +func TestTracing_ServeHTTP_Next_Error(t *testing.T) { + ot := &Tracing{ + SpanName: "mySpan", + } + + req := httptest.NewRequest("GET", "https://example.com/foo", nil) + w := httptest.NewRecorder() + + expectErr := errors.New("test error") + + var handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error { + return expectErr + } + + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()}) + defer cancel() + + if err := ot.Provision(ctx); err != nil { + t.Errorf("Provision error: %v", err) + t.FailNow() + } + + if err := ot.ServeHTTP(w, req, handler); err == nil || !errors.Is(err, expectErr) { + t.Errorf("expected error, got: %v", err) + } +} diff --git a/modules/caddyhttp/tracing/tracer.go b/modules/caddyhttp/tracing/tracer.go new file mode 100644 index 0000000..ce23944 --- /dev/null +++ b/modules/caddyhttp/tracing/tracer.go @@ -0,0 +1,108 @@ +package tracing + +import ( + "context" + "fmt" + "net/http" + + "github.com/caddyserver/caddy/v2" + + caddycmd "github.com/caddyserver/caddy/v2/cmd" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" + "go.uber.org/zap" +) + +const ( + webEngineName = "Caddy" + defaultSpanName = "handler" + nextCallCtxKey caddy.CtxKey = "nextCall" +) + +// nextCall store the next handler, and the error value return on calling it (if any) +type nextCall struct { + next caddyhttp.Handler + err error +} + +// openTelemetryWrapper is responsible for the tracing injection, extraction and propagation. +type openTelemetryWrapper struct { + propagators propagation.TextMapPropagator + + handler http.Handler + + spanName string +} + +// newOpenTelemetryWrapper is responsible for the openTelemetryWrapper initialization using provided configuration. +func newOpenTelemetryWrapper( + ctx context.Context, + spanName string, +) (openTelemetryWrapper, error) { + if spanName == "" { + spanName = defaultSpanName + } + + ot := openTelemetryWrapper{ + spanName: spanName, + } + + res, err := ot.newResource(webEngineName, caddycmd.CaddyVersion()) + if err != nil { + return ot, fmt.Errorf("creating resource error: %w", err) + } + + traceExporter, err := otlptracegrpc.New(ctx) + if err != nil { + return ot, fmt.Errorf("creating trace exporter error: %w", err) + } + + ot.propagators = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) + + tracerProvider := globalTracerProvider.getTracerProvider( + sdktrace.WithBatcher(traceExporter), + sdktrace.WithResource(res), + ) + + ot.handler = otelhttp.NewHandler(http.HandlerFunc(ot.serveHTTP), ot.spanName, otelhttp.WithTracerProvider(tracerProvider), otelhttp.WithPropagators(ot.propagators)) + return ot, nil +} + +// serveHTTP injects a tracing context and call the next handler. +func (ot *openTelemetryWrapper) serveHTTP(w http.ResponseWriter, r *http.Request) { + ot.propagators.Inject(r.Context(), propagation.HeaderCarrier(r.Header)) + next := r.Context().Value(nextCallCtxKey).(*nextCall) + next.err = next.next.ServeHTTP(w, r) +} + +// ServeHTTP propagates call to the by wrapped by `otelhttp` next handler. +func (ot *openTelemetryWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { + n := &nextCall{ + next: next, + err: nil, + } + ot.handler.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), nextCallCtxKey, n))) + + return n.err +} + +// cleanup flush all remaining data and shutdown a tracerProvider +func (ot *openTelemetryWrapper) cleanup(logger *zap.Logger) error { + return globalTracerProvider.cleanupTracerProvider(logger) +} + +// newResource creates a resource that describe current handler instance and merge it with a default attributes value. +func (ot *openTelemetryWrapper) newResource( + webEngineName, + webEngineVersion string, +) (*resource.Resource, error) { + return resource.Merge(resource.Default(), resource.NewSchemaless( + semconv.WebEngineNameKey.String(webEngineName), + semconv.WebEngineVersionKey.String(webEngineVersion), + )) +} diff --git a/modules/caddyhttp/tracing/tracer_test.go b/modules/caddyhttp/tracing/tracer_test.go new file mode 100644 index 0000000..36a32ff --- /dev/null +++ b/modules/caddyhttp/tracing/tracer_test.go @@ -0,0 +1,27 @@ +package tracing + +import ( + "context" + "testing" + + "github.com/caddyserver/caddy/v2" +) + +func TestOpenTelemetryWrapper_newOpenTelemetryWrapper(t *testing.T) { + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()}) + defer cancel() + + var otw openTelemetryWrapper + var err error + + if otw, err = newOpenTelemetryWrapper(ctx, + "", + ); err != nil { + t.Errorf("newOpenTelemetryWrapper() error = %v", err) + t.FailNow() + } + + if otw.propagators == nil { + t.Errorf("Propagators should not be empty") + } +} diff --git a/modules/caddyhttp/tracing/tracerprovider.go b/modules/caddyhttp/tracing/tracerprovider.go new file mode 100644 index 0000000..035425e --- /dev/null +++ b/modules/caddyhttp/tracing/tracerprovider.go @@ -0,0 +1,63 @@ +package tracing + +import ( + "context" + "fmt" + "sync" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.uber.org/zap" +) + +// globalTracerProvider stores global tracer provider and is responsible for graceful shutdown when nobody is using it. +var globalTracerProvider = &tracerProvider{} + +type tracerProvider struct { + mu sync.Mutex + tracerProvider *sdktrace.TracerProvider + tracerProvidersCounter int +} + +// getTracerProvider create or return an existing global TracerProvider +func (t *tracerProvider) getTracerProvider(opts ...sdktrace.TracerProviderOption) *sdktrace.TracerProvider { + t.mu.Lock() + defer t.mu.Unlock() + + t.tracerProvidersCounter++ + + if t.tracerProvider == nil { + t.tracerProvider = sdktrace.NewTracerProvider( + opts..., + ) + } + + return t.tracerProvider +} + +// cleanupTracerProvider gracefully shutdown a TracerProvider +func (t *tracerProvider) cleanupTracerProvider(logger *zap.Logger) error { + t.mu.Lock() + defer t.mu.Unlock() + + if t.tracerProvidersCounter > 0 { + t.tracerProvidersCounter-- + } + + if t.tracerProvidersCounter == 0 { + if t.tracerProvider != nil { + // tracerProvider.ForceFlush SHOULD be invoked according to https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#forceflush + if err := t.tracerProvider.ForceFlush(context.Background()); err != nil { + logger.Error("forcing flush", zap.Error(err)) + } + + // tracerProvider.Shutdown MUST be invoked according to https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#shutdown + if err := t.tracerProvider.Shutdown(context.Background()); err != nil { + return fmt.Errorf("tracerProvider shutdown error: %w", err) + } + } + + t.tracerProvider = nil + } + + return nil +} diff --git a/modules/caddyhttp/tracing/tracerprovider_test.go b/modules/caddyhttp/tracing/tracerprovider_test.go new file mode 100644 index 0000000..cb2e593 --- /dev/null +++ b/modules/caddyhttp/tracing/tracerprovider_test.go @@ -0,0 +1,43 @@ +package tracing + +import ( + "testing" + + "go.uber.org/zap" +) + +func Test_tracersProvider_getTracerProvider(t *testing.T) { + tp := tracerProvider{} + + tp.getTracerProvider() + tp.getTracerProvider() + + if tp.tracerProvider == nil { + t.Errorf("There should be tracer provider") + } + + if tp.tracerProvidersCounter != 2 { + t.Errorf("Tracer providers counter should equal to 2") + } +} + +func Test_tracersProvider_cleanupTracerProvider(t *testing.T) { + tp := tracerProvider{} + + tp.getTracerProvider() + tp.getTracerProvider() + + err := tp.cleanupTracerProvider(zap.NewNop()) + + if err != nil { + t.Errorf("There should be no error: %v", err) + } + + if tp.tracerProvider == nil { + t.Errorf("There should be tracer provider") + } + + if tp.tracerProvidersCounter != 1 { + t.Errorf("Tracer providers counter should equal to 1") + } +} -- cgit v1.2.3