export otel metrics for gensign panic (#218)

Contributing.md

@@ -18,6 +18,14 @@ If you encounter any bugs while using this software, or want to request a new fe
 ### Contribute Code
 Pull requests are welcome for bug fixes. If you want to implement something new, please [request a feature first](#report-a-bug-or-request-a-feature) so we can discuss it.
 
+#### Add license
+
+Please use license headers in all code files. You can apply the `./license_header` template by utilizing [addlicense](https://github.com/google/addlicense).
+
+```sh
+addlicense -f ./license_header ./
+```
+
 #### Creating a Pull Request
 Before you submit any code, we need you to agree to our [Contributor License Agreement](https://yahoocla.herokuapp.com/); this ensures we can continue to protect your contributions under an open source license well into the future.
 

cmd/gensign/main.go

@@ -6,22 +6,23 @@ package main
 import (
 	"context"
 	"io"
+	golog "log"
 	"os"
 
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
-	otellib "github.com/theparanoids/crypki/otel"
-	"github.com/theparanoids/ysshra/tlsutils"
-	"go.opentelemetry.io/otel/sdk/resource"
-	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
-
+	"github.com/theparanoids/crypki/otellib"
 	"github.com/theparanoids/ysshra/agent/ssh"
 	"github.com/theparanoids/ysshra/config"
 	"github.com/theparanoids/ysshra/crypki"
 	"github.com/theparanoids/ysshra/csr"
 	"github.com/theparanoids/ysshra/gensign"
 	"github.com/theparanoids/ysshra/gensign/regular"
 	"github.com/theparanoids/ysshra/internal/logkey"
+	ysshra_otellib "github.com/theparanoids/ysshra/otellib"
+	"github.com/theparanoids/ysshra/tlsutils"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
 )
 
 const (
@@ -44,6 +45,10 @@ func main() {
 	}
 	defer file.Close()
 	log.Logger = log.Logger.Output(io.MultiWriter(file, os.Stderr))
+	fileLogger := log.Output(file)
+
+	// Ensure all the 3rd party libraries, e.g. oTel, wouldn't log to os.Stderr since it will also go to user's console.
+	golog.SetOutput(file)
 
 	conf, err := config.NewGensignConfig(confPath)
 	if err != nil {
@@ -92,21 +97,22 @@ func main() {
 			resource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceNameKey.String("gensign")),
 		)
 		if err != nil {
-			log.Fatal().Err(err).Msg("failed to create otel resource")
+			fileLogger.Warn().Err(err).Msg("failed to create oTel resource")
 		}
-		otelTLSConf, err := tlsutils.TLSClientConfiguration(conf.OTel.CACertPath, conf.OTel.ClientCertPath,
-			[]string{conf.OTel.ClientKeyPath})
+		otelTLSConf, err := tlsutils.TLSClientConfiguration(conf.OTel.ClientCertPath, conf.OTel.ClientKeyPath,
+			[]string{conf.OTel.CACertPath})
 		if err != nil {
-			log.Fatal().Err(err).Msg("failed to create oTel TLS config")
+			fileLogger.Warn().Err(err).Msg("failed to create oTel TLS config")
 		}
 		shutdownProvider := otellib.InitOTelSDK(context.Background(),
 			conf.OTel.OTELCollectorEndpoint, otelTLSConf, otelResource)
 
 		defer func() {
 			if err := shutdownProvider(context.Background()); err != nil {
-				log.Fatal().Err(err).Msg("failed to shut down oTel provider")
+				fileLogger.Warn().Err(err).Msg("failed to shut down oTel provider")
 			}
 		}()
+		ysshra_otellib.InitMeter()
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), conf.RequestTimeout)
@@ -115,8 +121,8 @@ func main() {
 		if gensign.IsErrorOfType(err, gensign.Panic) {
 			// gensign will return debug stack in err when panic.
 			// We do not want it to be printed to os.Stderr since it will also go to user's console.
-			log.Logger = log.Output(file)
+			log.Logger = fileLogger
 		}
-		log.Fatal().Str(logkey.TransIDField, reqParam.TransID).Err(err).Msg("failed to run gensign")
+		log.Error().Str(logkey.TransIDField, reqParam.TransID).Err(err).Msg("failed to run gensign")
 	}
 }

docker/Dockerfile

@@ -1,7 +1,7 @@
 # Copyright 2022 Yahoo Inc.
 # Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms.
 
-FROM golang:1.20
+FROM golang:1.23
 ENV YSSHRA_DIR /go/src/github.com/theparanoids/ysshra
 COPY . ${YSSHRA_DIR}
 WORKDIR ${YSSHRA_DIR}

gensign/gensign.go

@@ -13,6 +13,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/theparanoids/ysshra/csr"
 	"github.com/theparanoids/ysshra/internal/logkey"
+	"github.com/theparanoids/ysshra/otellib"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -22,6 +23,7 @@ func Run(ctx context.Context, params *csr.ReqParam, handlers []Handler, signer c
 	// Prepare for panic logs
 	defer func() {
 		if r := recover(); r != nil {
+			otellib.ExportPanicMetric(ctx, params, fmt.Sprintf("%v", r))
 			err = NewError(Panic, "", fmt.Errorf(`unexpected crash: %q`, string(debug.Stack())))
 		}
 	}()

go.mod

@@ -13,9 +13,10 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/rs/zerolog v1.33.0
 	github.com/square/certstrap v1.3.0
-	github.com/theparanoids/crypki v1.20.2
+	github.com/theparanoids/crypki v1.20.4
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0
 	go.opentelemetry.io/otel v1.32.0
+	go.opentelemetry.io/otel/metric v1.32.0
 	go.opentelemetry.io/otel/sdk v1.32.0
 	go.uber.org/multierr v1.11.0
 	golang.org/x/crypto v0.29.0
@@ -27,6 +28,7 @@ require (
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -42,8 +44,8 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/urfave/cli v1.22.9 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 // indirect
-	go.opentelemetry.io/otel/metric v1.32.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect
 	go.opentelemetry.io/otel/trace v1.32.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect

go.sum

@@ -21,6 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -111,15 +113,17 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/theparanoids/crypki v1.20.2 h1:MT8PwuQOpNEpTe3E1oKclFPNXQCAyivKNtINVLCN8mE=
-github.com/theparanoids/crypki v1.20.2/go.mod h1:IKLbf/ug5i1LraCtZfDjue9M1V1aGNzNaYQNscRXkgg=
+github.com/theparanoids/crypki v1.20.4 h1:VGbWqs8hIoxTyjXo+8a1OCGt9YOhV37A5EwQwnNNXcE=
+github.com/theparanoids/crypki v1.20.4/go.mod h1:bQe3v9cOlKmHHY8LLjFGOtAKAKCz42Nb7/KpU5/pcqI=
 github.com/urfave/cli v1.22.9 h1:cv3/KhXGBGjEXLC4bH0sLuJ9BewaAbpk5oyMOveu4pw=
 github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 h1:yMkBS9yViCc7U7yeLzJPM2XizlfdVvBRSmsQDWu6qc0=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0/go.mod h1:n8MR6/liuGB5EmTETUBeU5ZgqMOlqKRxUaqPQBOANZ8=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
 go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
 go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 h1:t/Qur3vKSkUCcDVaSumWF2PKHt85pc7fRvFuoVT8qFU=

otellib/meter.go

@@ -0,0 +1,40 @@
+package otellib
+
+import (
+	"context"
+	"log"
+
+	"github.com/theparanoids/ysshra/csr"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+const (
+	scopeName   = "github.com/theparanoids/ysshra/otellib"
+	ysshraPanic = "ysshra.panic"
+)
+
+var meter metric.Meter
+
+// InitMeter initializes the oTel meter for ysshra.
+func InitMeter() {
+	meter = otel.GetMeterProvider().Meter(scopeName)
+}
+
+// ExportPanicMetric exports a panic metric to the oTel meter.
+func ExportPanicMetric(ctx context.Context, param *csr.ReqParam, msg string) {
+	var err error
+	panicCounter, err := meter.Int64Counter(
+		ysshraPanic,
+		metric.WithUnit("1"),
+		metric.WithDescription("Count the number of HTTP handler panic"),
+	)
+	if err != nil {
+		log.Printf("Error creating metric for panic: %v\n", err)
+	}
+	panicCounter.Add(ctx, 1, metric.WithAttributes(
+		attribute.String("gensign.transid", param.TransID),
+		attribute.String("panic.message", msg),
+	))
+}