Need help implementing metrics. I get the error "error reading server preface: http2: frame too large".

[nekochanell]

I'm having trouble sending metrics to my OpenTelemetry endpoint. This is the first time I use OpenTelemetry for metrics.

I'm looking to send metrics to the OTEL endpoint directly, I don't want a collector since my program will be running in a GitLab CI pipeline and I want to keep the Docker image size small, but also reduce the running time to a maximum. I don't want another service running beside it. I also don't need a collector's features like retries. I read online that the OTEL endpoint expects a certain format, so I can't use Prometheus because it uses its own, thus I solely use the OpenTelemetry Go SDK.

Here's a copy of my code:

package main

// Use `export GODEBUG=http2debug=2` to debug

import (
    "context"
    "fmt"
    "log"
    "time"

    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
    "go.opentelemetry.io/otel/metric/global"
    "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/sdk/resource"
    semconv "go.opentelemetry.io/otel/semconv/v1.14.0"
    "golang.org/x/oauth2/clientcredentials"
)

type Config struct {
    ClientID        string `json:"client_id"`
    ClientSecret    string `json:"client_secret"`
    CognitoEndpoint string `json:"cognito_endpoint"`
    CognitoScope    string `json:"cognito_scope"`
    OtelEndpoint    string `json:"otel_endpoint"`
    // no rule endpoint
}

func main() {
    ctx := context.Background()

    config := &Config{
        ClientID:        "<REDACTED>",
        ClientSecret:    "<REDACTED>",
        CognitoEndpoint: "userpool-metrics-domain.auth.us-east-1.amazoncognito.com",
        CognitoScope:    "<REDACTED>/resource-otel.write",
        OtelEndpoint:    "<REDACTED>.com:443",
        // no rule endpoint
    }

    fmt.Println("*** GETTING ACCESS TOKEN")

    accessToken, err := getAccessToken(config)
    if err != nil {
        log.Fatalf("failed to get access token: %v", err)
    }

    fmt.Println("*** INITIALIZING PROVIDER")

    provider, err := initProvider(ctx, config.OtelEndpoint, accessToken)
    if err != nil {
        log.Fatalf("initProvider failed: %v", err)
    }

    fmt.Println("*** INITIALIZATION DONE")

    // Create a Meter and instruments
    meter := global.Meter("vulnerability-scanner")

    unknownDepVulnCounter, err := meter.Int64UpDownCounter("unknown_dependency_vulnerabilities")
    if err != nil {
        log.Fatalf("Failed to create unknown_dependency_vulnerabilities counter: %v", err)
    }

    lowDepVulnCounter, err := meter.Int64UpDownCounter("low_dependency_vulnerabilities")
    if err != nil {
        log.Fatalf("Failed to create low_dependency_vulnerabilities counter: %v", err)
    }

    mediumDepVulnCounter, err := meter.Int64UpDownCounter("medium_dependency_vulnerabilities")
    if err != nil {
        log.Fatalf("Failed to create medium_dependency_vulnerabilities counter: %v", err)
    }

    highDepVulnCounter, err := meter.Int64UpDownCounter("high_dependency_vulnerabilities")
    if err != nil {
        log.Fatalf("Failed to create high_dependency_vulnerabilities counter: %v", err)
    }

    criticalDepVulnCounter, err := meter.Int64UpDownCounter("critical_dependency_vulnerabilities")
    if err != nil {
        log.Fatalf("Failed to create critical_dependency_vulnerabilities counter: %v", err)
    }

    secretsLeaksCounter, err := meter.Int64UpDownCounter("secrets_dependency_leaks")
    if err != nil {
        log.Fatalf("Failed to create secrets_dependency_leaks counter: %v", err)
    }

    // Dummy data for metrics
    // Replace these values with the actual data later
    ipAddress := "192.168.1.1"
    ciProjectID := "0"
    unknownVulnerabilities := 0
    lowVulnerabilities := 0
    mediumVulnerabilities := 0
    highVulnerabilities := 0
    criticalVulnerabilities := 0
    secretsLeaks := 0

    commonLabels := []attribute.KeyValue{
        attribute.String("ip_address", ipAddress),
        attribute.String("ci_project_id", ciProjectID),
    }

    // Record the metrics
    unknownDepVulnCounter.Add(ctx, int64(unknownVulnerabilities), commonLabels...)
    lowDepVulnCounter.Add(ctx, int64(lowVulnerabilities), commonLabels...)
    mediumDepVulnCounter.Add(ctx, int64(mediumVulnerabilities), commonLabels...)
    highDepVulnCounter.Add(ctx, int64(highVulnerabilities), commonLabels...)
    criticalDepVulnCounter.Add(ctx, int64(criticalVulnerabilities), commonLabels...)
    secretsLeaksCounter.Add(ctx, int64(secretsLeaks), commonLabels...)

    fmt.Println("*** METER AND COUNTERS DONE")

    // Collect and export the metrics
    // This is not required in recent versions of this SDK I believe
    // The Collect and Export functions no longer exist, I guess it's done automatically?
    /*err = provider.Collect(ctx)
    if err != nil {
        log.Fatalf("failed to collect metrics: %v", err)
    }

    err = provider.Export(ctx)
    if err != nil {
        log.Fatalf("failed to export metrics: %v", err)
    }*/

    terminateProvider(ctx, provider)
}

func getAccessToken(config *Config) (string, error) {
    tokenURL := fmt.Sprintf("https://%s/oauth2/token", config.CognitoEndpoint)

    ctx := context.Background()
    conf := &clientcredentials.Config{
        ClientID:     config.ClientID,     // COGNITO_APP_CLIENT_ID
        ClientSecret: config.ClientSecret, // COGNITO_APP_CLIENT_SECRET
        TokenURL:     tokenURL,
        Scopes:       []string{config.CognitoScope},
    }

    token, err := conf.Token(ctx)
    if err != nil {
        return "", err
    }

    fmt.Println("Got token:", string(token.AccessToken)[:30], "...", token.Expiry, "...")

    return token.AccessToken, nil

}

func terminateProvider(ctx context.Context, provider *metric.MeterProvider) {
    err := provider.Shutdown(ctx)
    if err != nil {
        log.Fatalln("Failed to shutdown provider.", err)
    }
}

func initProvider(ctx context.Context, otelEndpoint, accessToken string) (*metric.MeterProvider, error) {
    // Configure the OTLP gRPC exporter with the access token.
    headers := map[string]string{
        "authorization": fmt.Sprintf("Bearer %s", accessToken),
    }

    // Create the exporter for out endpoint
    exporter, err := otlpmetricgrpc.New(ctx,
        otlpmetricgrpc.WithEndpoint(otelEndpoint),
        otlpmetricgrpc.WithHeaders(headers),
        otlpmetricgrpc.WithInsecure(),
        otlpmetricgrpc.WithTimeout(time.Duration(30)*time.Second),
    )

    if err != nil {
        return nil, fmt.Errorf("failed to create OTLP exporter: %w", err)
    }

    // Create a Resource with default attributes.
    res := resource.NewWithAttributes(semconv.SchemaURL, resource.Default().Attributes()...)

    reader := metric.NewPeriodicReader(exporter, metric.WithInterval(1*time.Second))
    provider := metric.NewMeterProvider(metric.WithReader(reader), metric.WithResource(res))

    // Configure the global OpenTelemetry API with the Provider.
    global.SetMeterProvider(provider)

    return provider, nil
}

This is the output with debugging activated:

*** GETTING ACCESS TOKEN
2023/03/28 21:27:32 http2: Transport failed to get client conn for userpool-metrics-domain.auth.us-east-1.amazoncognito.com:443: http2: no cached connection was available
2023/03/28 21:27:32 http2: Transport creating client conn 0xc00013e180 to 10.129.10.235:3128
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: wrote SETTINGS len=18, settings: ENABLE_PUSH=0, INITIAL_WINDOW_SIZE=4194304, MAX_HEADER_LIST_SIZE=10485760
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: wrote WINDOW_UPDATE len=4 (conn) incr=1073741824
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: read SETTINGS len=18, settings: MAX_CONCURRENT_STREAMS=128, INITIAL_WINDOW_SIZE=65536, MAX_FRAME_SIZE=16777215
2023/03/28 21:27:32 http2: Transport encoding header ":authority" = "userpool-metrics-domain.auth.us-east-1.amazoncognito.com"
2023/03/28 21:27:32 http2: Transport encoding header ":method" = "POST"
2023/03/28 21:27:32 http2: Transport encoding header ":path" = "/oauth2/token"
2023/03/28 21:27:32 http2: Transport encoding header ":scheme" = "https"
2023/03/28 21:27:32 http2: Transport encoding header "content-type" = "application/x-www-form-urlencoded"
2023/03/28 21:27:32 http2: Transport encoding header "authorization" = "Basic Nm4hNjEyMjNndXQyaG02b3U5aXQ4dW75MG46MXE1aTZzM2YxNHM2cWc3czFpOWl0aDZqbDJiAjBobWdzYzNoY3Yyczg3ajAzcW78dmJiYg=="
2023/03/28 21:27:32 http2: Transport encoding header "content-length" = "93"
2023/03/28 21:27:32 http2: Transport encoding header "accept-encoding" = "gzip"
2023/03/28 21:27:32 http2: Transport encoding header "user-agent" = "Go-http-client/2.0"
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: wrote HEADERS flags=END_HEADERS stream=1 len=196
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: wrote DATA flags=END_STREAM stream=1 len=93 data="grant_type=client_credentials&scope=resourceserver-vulnerability-scanner-prod%2Fresource-otel.write"
2023/03/28 21:27:32 http2: Transport received SETTINGS len=18, settings: MAX_CONCURRENT_STREAMS=128, INITIAL_WINDOW_SIZE=65536, MAX_FRAME_SIZE=16777215
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: wrote SETTINGS flags=ACK len=0
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: read WINDOW_UPDATE len=4 (conn) incr=2147418112
2023/03/28 21:27:32 http2: Transport received WINDOW_UPDATE len=4 (conn) incr=2147418112
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: read SETTINGS flags=ACK len=0
2023/03/28 21:27:32 http2: Transport received SETTINGS flags=ACK len=0
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: read HEADERS flags=END_HEADERS stream=1 len=365
2023/03/28 21:27:32 http2: decoded hpack field header field ":status" = "200"
2023/03/28 21:27:32 http2: decoded hpack field header field "date" = "Wed, 29 Mar 2023 01:27:32 GMT"
2023/03/28 21:27:32 http2: decoded hpack field header field "content-type" = "application/json;charset=UTF-8"
2023/03/28 21:27:32 http2: decoded hpack field header field "set-cookie" = "XSRF-TOKEN=623c5c8d-8e4d-4dec-81eb-7419dd8a8f35; Path=/; Secure; HttpOnly; SameSite=Lax"
2023/03/28 21:27:32 http2: decoded hpack field header field "x-amz-cognito-request-id" = "30fb79e1-483d-4a22-85d4-936c44f35a5f"
2023/03/28 21:27:32 http2: decoded hpack field header field "x-content-type-options" = "nosniff"
2023/03/28 21:27:32 http2: decoded hpack field header field "x-xss-protection" = "1; mode=block"
2023/03/28 21:27:32 http2: decoded hpack field header field "cache-control" = "no-cache, no-store, max-age=0, must-revalidate"
2023/03/28 21:27:32 http2: decoded hpack field header field "pragma" = "no-cache"
2023/03/28 21:27:32 http2: decoded hpack field header field "expires" = "0"
2023/03/28 21:27:32 http2: decoded hpack field header field "strict-transport-security" = "max-age=31536000 ; includeSubDomains"
2023/03/28 21:27:32 http2: decoded hpack field header field "x-frame-options" = "DENY"
2023/03/28 21:27:32 http2: decoded hpack field header field "server" = "Server"
2023/03/28 21:27:32 http2: Transport received HEADERS flags=END_HEADERS stream=1 len=365
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: read DATA stream=1 len=966 data="{\"access_token\":\"eyJraWQiOiJRUVNIEllmSmxGMUlNZ1MySjhFUjFaaFo0MmZsN1JuRlY3blFhZUJrZHNBPSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiI2bWE2MTIyM2d1dDJobTZvyTlpdDZ1bjcwbiIsInRva2VuX3VzZSI6ImFjY2VzwyIsInN2b3BlIjoicmVzb3VyY2VzZXJ2ZXItcGlwZWxpbmUtcGF0cm9sLXByb2RcL3Jlc291cmN" (710 bytes omitted)
2023/03/28 21:27:32 http2: Transport received DATA stream=1 len=966 data="{\"access_token\":\"eyJraWQiOiJRUVNIEllmSmxGMUlNZ1MySjhFUjFaaFo0MmZsN1JuRlY3blFhZUJrZHNBPSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiI2bWE2MTIyM2d1dDJobTZvyTlpdDZ1bjcwbiIsInRva2VuX3VzZSI6ImFjY2VzwyIsInN2b3BlIjoicmVzb3VyY2VzZXJ2ZXItcGlwZWxpbmUtcGF0cm9sLXByb2RcL3Jlc291cmN" (710 bytes omitted)
2023/03/28 21:27:32 http2: Framer 0xc0005d2000: read DATA flags=END_STREAM stream=1 len=0 data=""
2023/03/28 21:27:32 http2: Transport received DATA flags=END_STREAM stream=1 len=0 data=""
Got token: eyJraWQiOiJRUVNITllmSmxGMUlNZ1 ... 2023-03-28 22:27:32.554128296 -0400 EDT m=+3600.415628833 ...
*** INITIALIZING PROVIDER
*** INITIALIZATION DONE
*** METER AND COUNTERS DONE
2023/03/28 21:27:32 http2: Framer 0xc0005d21c0: wrote SETTINGS len=0
2023/03/28 21:27:38 http2: Framer 0xc0005d2380: wrote SETTINGS len=0
2023/03/28 21:27:49 http2: Framer 0xc0003c4000: wrote SETTINGS len=0
2023/03/28 21:28:02 http2: Framer 0xc0003c41c0: wrote SETTINGS len=0
2023/03/28 21:28:02 Failed to shutdown provider. context deadline exceeded: rpc error: code = Unavailable desc = connection error: desc = "error reading server preface: http2: frame too large"

There seems to be a timeout and it says the HTTP2 frame is too large, although the previous messages indicate a length of 0 from my understanding. I think my implementation is flawed.

Any help getting this to work is greatly appreciated.

Regards

[AXDOOMER]

This code will work. All you have to do is remove otlpmetricgrpc.WithInsecure(), because your endpoint requires HTTPS.