diff --git a/plugins/ptp_operator/metrics/logparser.go b/plugins/ptp_operator/metrics/logparser.go index 894ea137..c416aabd 100644 --- a/plugins/ptp_operator/metrics/logparser.go +++ b/plugins/ptp_operator/metrics/logparser.go @@ -92,7 +92,7 @@ func extractSummaryMetrics(processName, output string) (iface string, ptpOffset, return } -func extractRegularMetrics(processName, output string) (interfaceName string, ptpOffset, maxPtpOffset, frequencyAdjustment, delay float64, clockState ptp.SyncState) { +func extractRegularMetrics(processName, output string) (interfaceName string, nmeaPpsStatus, ptpOffset, maxPtpOffset, frequencyAdjustment, delay float64, clockState ptp.SyncState) { // remove everything before the rms string // This makes the out to equal // ptp4l[5196819.100]: [ptp4l.0.config] master offset -2162130 s2 freq +22451884 path delay 374976 @@ -102,10 +102,11 @@ func extractRegularMetrics(processName, output string) (interfaceName string, pt // ts2phc[82674.465]: [ts2phc.0.cfg] nmea delay: 88403525 ns // ts2phc[82674.465]: [ts2phc.0.cfg] ens2f1 extts index 0 at 1673031129.000000000 corr 0 src 1673031129.911642976 diff 0 // ts2phc[82674.465]: [ts2phc.0.cfg] ens2f1 master offset 0 s2 freq -0 - + // ts2phc[1699929121]:[ts2phc.0.config] ens2f0 nmea_status 0 offset 999999 s0 // 0 1 2 3 4 5 6 7 8 9 10 11 // 1 2 3 4 5 6 7 8 9 // ptp4l 5196819.100 ptp4l.0.config master offset -2162130 s2 freq +22451884 path delay 374976 + var err error index := FindInLogForCfgFileIndex(output) if index == -1 { log.Errorf("config name is not found in log outpt") @@ -122,11 +123,12 @@ func extractRegularMetrics(processName, output string) (interfaceName string, pt // 0 1 2 3 4 5 6 7 8 // ptp4l.0.config master offset -2162130 s2 freq +22451884 delay 374976 // ts2phc.0.cfg ens2f1 master offset 0 s2 freq -0 + // ts2phc.0.config ens2f0 nmea_status 0 offset 999999 s0 // (ts2phc.0.cfg master offset 0 s2 freq -0) if len(fields) < 7 { return } - // either master or clock_realtime + // either master, clock_realtime, or nmea_status interfaceName = fields[1] if fields[2] != offset && processName == ts2phcProcessName { // Remove the element at index 1 from fields. @@ -134,12 +136,26 @@ func extractRegularMetrics(processName, output string) (interfaceName string, pt // ts2phc.0.cfg master offset 0 s2 freq -0 fields = fields[:len(fields)-1] // Truncate slice. } + + // .0.config nmea_status 0 offset 999999 s0 + if fields[1] == "nmea_status" || fields[1] == "pps_status" { + nmeaPpsStatus, err = strconv.ParseFloat(fields[2], 64) + if err != nil { + log.Errorf("%s failed to parse nmea/pps status from master output %s error %v", processName, fields[2], err) + } + ptpOffset, err = strconv.ParseFloat(fields[4], 64) + if err != nil { + log.Errorf("%s failed to parse nmea/pps offset from master output %s error %v", processName, fields[4], err) + } + return + } + if fields[2] != offset { log.Errorf("%s failed to parse offset from master output %s error %s", processName, fields[2], "offset is not in right order") return } - ptpOffset, err := strconv.ParseFloat(fields[3], 64) + ptpOffset, err = strconv.ParseFloat(fields[3], 64) if err != nil { log.Errorf("%s failed to parse offset from master output %s error %v", processName, fields[3], err) } diff --git a/plugins/ptp_operator/metrics/manager.go b/plugins/ptp_operator/metrics/manager.go index 0f6b139d..621ea253 100644 --- a/plugins/ptp_operator/metrics/manager.go +++ b/plugins/ptp_operator/metrics/manager.go @@ -283,7 +283,7 @@ func (p *PTPEventManager) GenPTPEvent(ptpProfileName string, oStats *stats.Stats case ptp.HOLDOVER: // do nothing, the timeout will switch holdover to FREE-RUN default: // not yet used states - log.Warnf("unknown %s sync state %s ,has last ptp state %s", eventResourceName, clockState, lastClockState) + log.Warnf("%s sync state %s, last ptp state is unknown: %s", eventResourceName, clockState, lastClockState) if !isOffsetInRange(ptpOffset, threshold.MaxOffsetThreshold, threshold.MinOffsetThreshold) { clockState = ptp.FREERUN } diff --git a/plugins/ptp_operator/metrics/metrics.go b/plugins/ptp_operator/metrics/metrics.go index edd98872..47bb6b7b 100644 --- a/plugins/ptp_operator/metrics/metrics.go +++ b/plugins/ptp_operator/metrics/metrics.go @@ -65,6 +65,11 @@ const ( PtpProcessDown int64 = 0 // PtpProcessUp process is up PtpProcessUp int64 = 1 + + // UNAVAILABLE Nmea and Pps status + UNAVAILABLE float64 = 0 + // AVAILABLE Nmea and Pps status + AVAILABLE float64 = 1 ) // ExtractMetrics ... extract metrics from ptp logs. @@ -132,6 +137,7 @@ func (p *PTPEventManager) ExtractMetrics(msg string) { } return } + switch processName { case gnssProcessName: p.ParseGNSSLogs(processName, configName, output, fields, ptp4lCfg, ptpStats) @@ -176,9 +182,12 @@ func (p *PTPEventManager) ExtractMetrics(msg string) { // ts2phc[82674.465]: [ts2phc.0.cfg] nmea delay: 88403525 ns // ts2phc[82674.465]: [ts2phc.0.cfg] ens2f1 extts index 0 at 1673031129.000000000 corr 0 src 1673031129.911642976 diff 0 // ts2phc[82674.465]: [ts2phc.0.cfg] ens2f1 master offset 0 s2 freq -0 + // ts2phc[82674.465]: [ts2phc.0.cfg] ens7f1 master offset 0 s2 freq -0 + // ts2phc[1699894580]:[ts2phc.0.config] ens2fx nmea_status 1 offset 0 s2 + // Use threshold to CLOCK_REALTIME==SLAVE, rest send clock state to metrics no events - interfaceName, ptpOffset, _, frequencyAdjustment, delay, syncState := extractRegularMetrics(processName, output) + interfaceName, nmeaPpsStatus, ptpOffset, _, frequencyAdjustment, delay, syncState := extractRegularMetrics(processName, output) if interfaceName == "" { return // don't do if iface not known } @@ -260,6 +269,19 @@ func (p *PTPEventManager) ExtractMetrics(msg string) { alias = string(r[:len(r)-1]) + "x" ptpStats[master].SetAlias(alias) masterResource := fmt.Sprintf("%s/%s", alias, MasterClockType) + if interfaceName == "nmea_status" || interfaceName == "pps_status" { + if nmeaPpsStatus == UNAVAILABLE { + p.GenPTPEvent(profileName, ptpStats[interfaceType], masterResource, int64(ptpOffset), ptp.FREERUN, ptp.PtpStateChange) + } else { + UpdatePTPOffsetMetrics(offsetSource, processName, alias, ptpOffset) + } + if interfaceName == "nmea_status" { + UpdateNmeaStatusMetrics(processName, alias, nmeaPpsStatus) + } else { + UpdatePpsStatusMetrics(processName, alias, nmeaPpsStatus) + } + return + } p.GenPTPEvent(profileName, ptpStats[interfaceType], masterResource, int64(ptpOffset), syncState, ptp.PtpStateChange) UpdateSyncStateMetrics(processName, alias, ptpStats[interfaceType].LastSyncState()) UpdatePTPMetrics(offsetSource, processName, alias, ptpOffset, float64(ptpStats[interfaceType].MaxAbs()), diff --git a/plugins/ptp_operator/metrics/metrics_test.go b/plugins/ptp_operator/metrics/metrics_test.go new file mode 100644 index 00000000..0f909de8 --- /dev/null +++ b/plugins/ptp_operator/metrics/metrics_test.go @@ -0,0 +1,102 @@ +// Copyright 2023 The Cloud Native Events Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build unittests +// +build unittests + +package metrics_test + +import ( + "os" + "testing" + + "github.com/redhat-cne/cloud-event-proxy/plugins/ptp_operator/config" + "github.com/redhat-cne/cloud-event-proxy/plugins/ptp_operator/metrics" + "github.com/redhat-cne/cloud-event-proxy/plugins/ptp_operator/ptp4lconf" + "github.com/redhat-cne/cloud-event-proxy/plugins/ptp_operator/stats" + "github.com/redhat-cne/cloud-event-proxy/plugins/ptp_operator/types" +) + +type metricsTestCase struct { + ptpLog string +} + +var ptp4lConfig = &ptp4lconf.PTP4lConfig{ + Name: "ptp4l.0.config", + Profile: "grandmaster", + Interfaces: []*ptp4lconf.PTPInterface{ + { + Name: "ens2f0", + PortID: 1, + PortName: "port 1", + Role: 2, + }, + { + Name: "ens7f0", + PortID: 2, + PortName: "port 3", + Role: 2, + }, + }, +} + +var ptpEventManager = &metrics.PTPEventManager{ + Stats: make(map[types.ConfigName]map[types.IFace]*stats.Stats), + Ptp4lConfigInterfaces: make(map[types.ConfigName]*ptp4lconf.PTP4lConfig), +} + +var testCase = []string{ + "ts2phc[1699929121]:[ts2phc.0.config] ens2f0 nmea_status 0 offset 999999 s0" + "\n", + "ts2phc[1699929171]:[ts2phc.0.config] ens2fx nmea_status 1 offset 0 s2" + "\n", + "ts2phc[1699929121]:[ts2phc.0.config] ens7f0 pps_status 0 offset 999999 s0" + "\n", + "ts2phc[1699929171]:[ts2phc.0.config] ens7fx pps_status 1 offset 0 s2" + "\n", + "GM[1699929086]:[ts2phc.0.config] ens2f0 T-GM-STATUS s0" + "\n", + "ts2phc[441664.291]: [ts2phc.0.config] ens2f0 master offset 0 s2 freq -0" + "\n", + "ts2phc[441664.304]: [ts2phc.0.config] ens7f0 master offset 0 s2 freq +0" + "\n", +} + +func setup() { + ptpEventManager.AddPTPConfig(types.ConfigName(ptp4lConfig.Name), ptp4lConfig) + + stats_master := stats.NewStats(ptp4lConfig.Name) + stats_master.SetOffsetSource("master") + stats_master.SetProcessName("ts2phc") + stats_master.SetAlias("ens2fx") + + stats_slave := stats.NewStats(ptp4lConfig.Name) + stats_slave.SetOffsetSource("phc") + stats_slave.SetProcessName("phc2sys") + stats_slave.SetLastSyncState("LOCKED") + stats_slave.SetClockClass(0) + + ptpEventManager.Stats[types.ConfigName(ptp4lConfig.Name)] = make(map[types.IFace]*stats.Stats) + ptpEventManager.Stats[types.ConfigName(ptp4lConfig.Name)][types.IFace("master")] = stats_master + ptpEventManager.Stats[types.ConfigName(ptp4lConfig.Name)][types.IFace("CLOCK_REALTIME")] = stats_slave + ptpEventManager.PtpConfigMapUpdates = config.NewLinuxPTPConfUpdate() +} + +func teardown() { +} + +func TestMain(m *testing.M) { + setup() + code := m.Run() + teardown() + os.Exit(code) +} +func Test_ExtractMetrics(t *testing.T) { + for _, tc := range testCase { + ptpEventManager.ExtractMetrics(tc) + } +} diff --git a/plugins/ptp_operator/metrics/registry.go b/plugins/ptp_operator/metrics/registry.go index 90dd5794..4e147bae 100644 --- a/plugins/ptp_operator/metrics/registry.go +++ b/plugins/ptp_operator/metrics/registry.go @@ -57,6 +57,24 @@ var ( Help: "0 = FREERUN, 1 = LOCKED, 2 = HOLDOVER", }, []string{"process", "node", "iface"}) + // NmeaStatus metrics to show current nmea status + NmeaStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: ptpNamespace, + Subsystem: ptpSubsystem, + Name: "nmea_status", + Help: "0 = UNAVAILABLE, 1 = AVAILABLE", + }, []string{"process", "node", "iface"}) + + // PpsStatus metrics to show current pps status + PpsStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: ptpNamespace, + Subsystem: ptpSubsystem, + Name: "pps_status", + Help: "0 = UNAVAILABLE, 1 = AVAILABLE", + }, []string{"process", "node", "iface"}) + // Threshold metrics to show current ptp threshold Threshold = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -113,6 +131,8 @@ func RegisterMetrics(nodeName string) { prometheus.MustRegister(PtpFrequencyAdjustment) prometheus.MustRegister(PtpDelay) prometheus.MustRegister(SyncState) + prometheus.MustRegister(NmeaStatus) + prometheus.MustRegister(PpsStatus) prometheus.MustRegister(Threshold) prometheus.MustRegister(InterfaceRole) prometheus.MustRegister(ClockClassMetrics) @@ -142,6 +162,12 @@ func UpdatePTPMetrics(metricsType, process, eventResourceName string, offset, ma "process": process, "node": ptpNodeName, "iface": eventResourceName}).Set(delay) } +// UpdatePTPOffsetMetrics ... update ptp offset metrics +func UpdatePTPOffsetMetrics(metricsType, process, eventResourceName string, offset float64) { + PtpOffset.With(prometheus.Labels{"from": metricsType, + "process": process, "node": ptpNodeName, "iface": eventResourceName}).Set(offset) +} + // DeletedPTPMetrics ... update metrics for deleted ptp config func DeletedPTPMetrics(clockType, processName, eventResourceName string) { PtpOffset.Delete(prometheus.Labels{"from": clockType, @@ -178,13 +204,25 @@ func UpdateSyncStateMetrics(process, iface string, state ptp.SyncState) { } // prevent reporting wrong metrics if iface == master && process == phc2sysProcessName { - log.Errorf("wrong meterics are processed, ignoring interface %s with process %s", iface, process) + log.Errorf("wrong metrics are processed, ignoring interface %s with process %s", iface, process) return } SyncState.With(prometheus.Labels{ "process": process, "node": ptpNodeName, "iface": iface}).Set(clockState) } +// UpdateNmeaStatusMetrics ... update nmea status metrics +func UpdateNmeaStatusMetrics(process, iface string, status float64) { + NmeaStatus.With(prometheus.Labels{ + "process": process, "node": ptpNodeName, "iface": iface}).Set(status) +} + +// UpdatePpsStatusMetrics ... update pps status metrics +func UpdatePpsStatusMetrics(process, iface string, status float64) { + PpsStatus.With(prometheus.Labels{ + "process": process, "node": ptpNodeName, "iface": iface}).Set(status) +} + // UpdateInterfaceRoleMetrics ... update interface role metrics func UpdateInterfaceRoleMetrics(process, ptpInterface string, role types.PtpPortRole) { InterfaceRole.With(prometheus.Labels{