diff --git a/grafana/dashboards/deep_dive.json b/grafana/dashboards/deep_dive.json index b3f0cc274..426c5be23 100644 --- a/grafana/dashboards/deep_dive.json +++ b/grafana/dashboards/deep_dive.json @@ -116,7 +116,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, "links": [], "panels": [ { @@ -591,7 +590,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { @@ -1365,7 +1364,7 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -1373,698 +1372,679 @@ "y": 32 }, "id": 48, - "panels": [ - { - "datasource": { - "type": "jaeger", - "uid": "${DS_JAEGER}" - }, - "description": "Total number of traces recorded in the selected time range.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 33 + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "datasource": { + "type": "jaeger", + "uid": "${DS_JAEGER}" + }, + "description": "Total number of traces recorded in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "id": 60, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "count" - ], - "fields": "/^Trace ID$/", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "jaeger", - "uid": "${DS_JAEGER}" - }, - "limit": 1000, - "queryType": "search", - "refId": "A", - "service": "${trace_service}", - "tags": "${trace_host}" - } + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 60, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "count" ], - "title": "Total Traces", - "type": "stat" + "fields": "/^Trace ID$/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "jaeger", "uid": "${DS_JAEGER}" }, - "description": "Number of traces with errors in the selected time range.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "limit": 1000, + "queryType": "search", + "refId": "A", + "service": "${trace_service}", + "tags": "${trace_host}" + } + ], + "title": "Total Traces", + "type": "stat" + }, + { + "datasource": { + "type": "jaeger", + "uid": "${DS_JAEGER}" + }, + "description": "Number of traces with errors in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 10 - } - ] + { + "color": "yellow", + "value": 1 }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 33 - }, - "id": 61, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "count" - ], - "fields": "/^Trace ID$/", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + { + "color": "red", + "value": 10 + } + ] }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "jaeger", - "uid": "${DS_JAEGER}" - }, - "limit": 1000, - "queryType": "search", - "refId": "A", - "service": "${trace_service}", - "tags": "error=true ${trace_host}" - } + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 33 + }, + "id": 61, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "count" ], - "title": "Error Traces", - "type": "stat" + "fields": "/^Trace ID$/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "jaeger", "uid": "${DS_JAEGER}" }, - "description": "Average trace duration across all traces in the selected time range.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 100000 - }, - { - "color": "red", - "value": 500000 - } - ] - }, - "unit": "µs" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 33 - }, - "id": 62, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "/^Duration$/", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "limit": 1000, + "queryType": "search", + "refId": "A", + "service": "${trace_service}", + "tags": "error=true ${trace_host}" + } + ], + "title": "Error Traces", + "type": "stat" + }, + { + "datasource": { + "type": "jaeger", + "uid": "${DS_JAEGER}" + }, + "description": "Average trace duration across all traces in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "jaeger", - "uid": "${DS_JAEGER}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "limit": 1000, - "queryType": "search", - "refId": "A", - "service": "${trace_service}", - "tags": "${trace_host}" - } - ], - "title": "Avg Duration", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "substring", - "options": { - "value": "pprof" - } - }, - "fieldName": "Trace name" - }, - { - "config": { - "id": "substring", - "options": { - "value": "metrics" - } - }, - "fieldName": "Trace name" - } - ], - "match": "any", - "type": "exclude" - } - }, - { - "id": "organize", - "options": { - "renameByName": { - "duration": "Duration" - } + { + "color": "yellow", + "value": 100000 + }, + { + "color": "red", + "value": 500000 } - } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 33 + }, + "id": 62, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" ], - "type": "stat" + "fields": "/^Duration$/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "jaeger", "uid": "${DS_JAEGER}" }, - "description": "Maximum trace duration (slowest request) in the selected time range.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 500000 - }, - { - "color": "red", - "value": 1000000 - } - ] - }, - "unit": "µs" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 33 - }, - "id": 63, + "limit": 1000, + "queryType": "search", + "refId": "A", + "service": "${trace_service}", + "tags": "${trace_host}" + } + ], + "title": "Avg Duration", + "transformations": [ + { + "id": "filterByValue", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "max" - ], - "fields": "/^Duration$/", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "jaeger", - "uid": "${DS_JAEGER}" + "filters": [ + { + "config": { + "id": "substring", + "options": { + "value": "pprof" + } + }, + "fieldName": "Trace name" }, - "limit": 1000, - "queryType": "search", - "refId": "A", - "service": "${trace_service}", - "tags": "${trace_host}" - } - ], - "title": "Max Duration", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "substring", - "options": { - "value": "pprof" - } - }, - "fieldName": "Trace name" - }, - { - "config": { - "id": "substring", - "options": { - "value": "metrics" - } - }, - "fieldName": "Trace name" + { + "config": { + "id": "substring", + "options": { + "value": "metrics" } - ], - "match": "any", - "type": "exclude" - } - }, - { - "id": "organize", - "options": { - "renameByName": { - "duration": "Duration" - } + }, + "fieldName": "Trace name" } - } - ], - "type": "stat" + ], + "match": "any", + "type": "exclude" + } }, + { + "id": "organize", + "options": { + "renameByName": { + "duration": "Duration" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "jaeger", + "uid": "${DS_JAEGER}" + }, + "description": "Maximum trace duration (slowest request) in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 500000 + }, + { + "color": "red", + "value": 1000000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 33 + }, + "id": 63, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "/^Duration$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "jaeger", "uid": "${DS_JAEGER}" }, - "description": "Recent traces from Ella Core. Click on a Trace ID to view the full span timeline.", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "footer": { - "reducers": [] - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - } - }, - "overrides": [ + "limit": 1000, + "queryType": "search", + "refId": "A", + "service": "${trace_service}", + "tags": "${trace_host}" + } + ], + "title": "Max Duration", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ { - "matcher": { - "id": "byName", - "options": "Trace ID" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "View Trace", - "url": "/explore?schemaVersion=1&panes=%7B%22trace%22:%7B%22datasource%22:%22jaeger%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22jaeger%22,%22uid%22:%22jaeger%22%7D,%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1" - } - ] + "config": { + "id": "substring", + "options": { + "value": "pprof" } - ] + }, + "fieldName": "Trace name" }, { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "µs" + "config": { + "id": "substring", + "options": { + "value": "metrics" } - ] + }, + "fieldName": "Trace name" } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 52, + ], + "match": "any", + "type": "exclude" + } + }, + { + "id": "organize", "options": { - "cellHeight": "sm", + "renameByName": { + "duration": "Duration" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "jaeger", + "uid": "${DS_JAEGER}" + }, + "description": "Recent traces from Ella Core. Click on a Trace ID to view the full span timeline.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "reducers": [] }, - "showHeader": true, - "sortBy": [ + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "desc": true, - "displayName": "Start Time" + "color": "green", + "value": 0 } ] - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "jaeger", - "uid": "${DS_JAEGER}" - }, - "limit": 50, - "queryType": "search", - "refId": "A", - "service": "${trace_service}", - "tags": "${trace_host}" - } - ], - "title": "Recent Traces", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "substring", - "options": { - "value": "pprof" - } - }, - "fieldName": "Trace name" - }, + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "links", + "value": [ { - "config": { - "id": "substring", - "options": { - "value": "metrics" - } - }, - "fieldName": "Trace name" + "title": "View Trace", + "url": "/explore?schemaVersion=1&panes=%7B%22trace%22:%7B%22datasource%22:%22jaeger%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22jaeger%22,%22uid%22:%22jaeger%22%7D,%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1" } - ], - "match": "any", - "type": "exclude" + ] } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "duration": "Duration", - "startTime": "Start Time", - "traceID": "Trace ID", - "traceName": "Operation" - } + "properties": [ + { + "id": "unit", + "value": "µs" } - } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 52, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" ], - "type": "table" + "show": false }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Start Time" + } + ] + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "jaeger", "uid": "${DS_JAEGER}" }, - "description": "Traces exceeding 1ms duration. These may indicate performance bottlenecks. Click on a Trace ID to investigate.", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "footer": { - "reducers": [] + "limit": 50, + "queryType": "search", + "refId": "A", + "service": "${trace_service}", + "tags": "${trace_host}" + } + ], + "title": "Recent Traces", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "substring", + "options": { + "value": "pprof" + } }, - "inspect": false + "fieldName": "Trace name" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": 0 - }, - { - "color": "orange", - "value": 500000 - }, - { - "color": "red", - "value": 1000000 + { + "config": { + "id": "substring", + "options": { + "value": "metrics" } - ] + }, + "fieldName": "Trace name" } + ], + "match": "any", + "type": "exclude" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "duration": "Duration", + "startTime": "Start Time", + "traceID": "Trace ID", + "traceName": "Operation" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "jaeger", + "uid": "${DS_JAEGER}" + }, + "description": "Traces exceeding 1ms duration. These may indicate performance bottlenecks. Click on a Trace ID to investigate.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" }, - "overrides": [ + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "Trace ID" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "View Trace", - "url": "/explore?schemaVersion=1&panes=%7B%22trace%22:%7B%22datasource%22:%22jaeger%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22jaeger%22,%22uid%22:%22jaeger%22%7D,%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1" - } - ] - } - ] + "color": "yellow", + "value": 0 }, { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "µs" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, + "color": "orange", + "value": 500000 + }, + { + "color": "red", + "value": 1000000 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "links", + "value": [ { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": 0 - }, - { - "color": "orange", - "value": 500000 - }, - { - "color": "red", - "value": 1000000 - } - ] - } + "title": "View Trace", + "url": "/explore?schemaVersion=1&panes=%7B%22trace%22:%7B%22datasource%22:%22jaeger%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22jaeger%22,%22uid%22:%22jaeger%22%7D,%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1" } ] } ] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 37 - }, - "id": 64, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + { + "matcher": { + "id": "byName", + "options": "Duration" }, - "showHeader": true, - "sortBy": [ + "properties": [ { - "desc": true, - "displayName": "Duration" - } - ] - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "jaeger", - "uid": "${DS_JAEGER}" + "id": "unit", + "value": "µs" }, - "limit": 50, - "minDuration": "1ms", - "queryType": "search", - "refId": "A", - "service": "${trace_service}", - "tags": "${trace_host}" - } - ], - "title": "Slow-ish Traces (>1ms)", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "substring", - "options": { - "value": "pprof" - } + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": 0 }, - "fieldName": "Trace name" - }, - { - "config": { - "id": "substring", - "options": { - "value": "metrics" - } + { + "color": "orange", + "value": 500000 }, - "fieldName": "Trace name" - } - ], - "match": "any", - "type": "exclude" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "duration": "Duration", - "startTime": "Start Time", - "traceID": "Trace ID", - "traceName": "Operation" + { + "color": "red", + "value": 1000000 + } + ] } } - } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 64, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" ], - "type": "table" + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Duration" + } + ] + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "${DS_JAEGER}" + }, + "limit": 50, + "minDuration": "1ms", + "operation": "NGAP receive", + "queryType": "search", + "refId": "A", + "service": "${trace_service}", + "tags": "${trace_host}" } ], - "title": "Traces", - "type": "row" + "title": "Slow NGAP Traces (>1ms)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Trace name": true + }, + "includeByName": {}, + "indexByName": {}, + "orderByMode": "manual", + "renameByName": { + "Duration": "", + "Start time": "", + "Trace ID": "", + "Trace name": "", + "duration": "Duration", + "startTime": "Start Time", + "traceID": "Trace ID", + "traceName": "Operation" + } + } + } + ], + "type": "table" }, { "collapsed": true, @@ -2072,7 +2052,7 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 45 }, "id": 15, "panels": [ @@ -2098,14 +2078,12 @@ "detailsMode": "inline", "enableInfiniteScrolling": false, "enableLogDetails": true, - "fontSize": "default", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showLabels": true, "showTime": true, "sortOrder": "Descending", - "timestampResolution": "ms", "wrapLogMessage": false }, "pluginVersion": "12.3.1", @@ -2117,7 +2095,7 @@ }, "direction": "backward", "editorMode": "builder", - "expr": "{job=~\"$job\", instance=~\"$instance\"} | json | line_format `[{{.component}}] {{.msg}}`", + "expr": "{job=~\"$job\", instance=~\"$instance\", component=~\"$component\", level=~\"$level\"} | json | line_format \"[{{.component}}] {{.msg}}\"", "queryType": "range", "refId": "A" } @@ -2135,7 +2113,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 46 }, "id": 3, "panels": [ @@ -2153,7 +2131,7 @@ "h": 12, "w": 12, "x": 0, - "y": 47 + "y": 83 }, "id": 2, "options": {}, @@ -2187,7 +2165,7 @@ "h": 12, "w": 12, "x": 12, - "y": 47 + "y": 83 }, "id": 1, "options": {}, @@ -2267,7 +2245,7 @@ "type": "datasource" }, { - "allValue": ".+", + "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", @@ -2288,7 +2266,7 @@ "type": "query" }, { - "allValue": ".+", + "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", @@ -2364,17 +2342,62 @@ ], "query": "", "type": "textbox" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "definition": "label_values({job=~\"$job\", instance=~\"$instance\"}, component)", + "description": "Filter logs by component (amf, smf, upf, api, etc.)", + "includeAll": true, + "label": "Component", + "multi": true, + "name": "component", + "options": [], + "query": { + "query": "label_values({job=~\"$job\", instance=~\"$instance\"}, component)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "definition": "label_values({job=~\"$job\", instance=~\"$instance\"}, level)", + "description": "Filter logs by severity (info, warn, error)", + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [], + "query": { + "query": "label_values({job=~\"$job\", instance=~\"$instance\"}, level)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" } ] }, "time": { - "from": "now-3h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Ella Core Deep Dive", "uid": "ella-deep-dive", - "version": 3, - "weekStart": "" + "version": 8, + "weekStart": "", + "id": null } \ No newline at end of file diff --git a/grafana/dashboards/network_health.json b/grafana/dashboards/network_health.json index 2365f563d..1a8304bf3 100644 --- a/grafana/dashboards/network_health.json +++ b/grafana/dashboards/network_health.json @@ -52,10 +52,10 @@ } ] }, - "editable": false, + "description": "Monitor your Ella Core deployment.", + "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": null, "links": [], "panels": [ { @@ -123,15 +123,15 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "builder", "expr": "app_connected_radios{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "Connected Radios", @@ -189,16 +189,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "100 * (\n sum(rate(app_registration_attempts_total{job=~\"$job\",instance=~\"$instance\",result=\"accept\"}[$__rate_interval]))\n /\n sum(rate(app_registration_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))\n)\nand\nsum(rate(app_registration_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) > 0", "interval": "1m", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "Registration Success Rate", @@ -264,16 +264,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "100 * (\n sum(rate(app_pdu_session_establishment_attempts_total{job=~\"$job\",instance=~\"$instance\",result=\"accept\"}[$__rate_interval]))\n /\n sum(rate(app_pdu_session_establishment_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))\n)\nand\nsum(rate(app_pdu_session_establishment_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) > 0", "interval": "1m", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "PDU Session Establishment Success Rate", @@ -331,15 +331,15 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "builder", "expr": "app_pdu_sessions_total{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "Active PDU Sessions", @@ -397,16 +397,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "(rate(app_uplink_bytes{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) * 8) + (rate(app_downlink_bytes{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) * 8)", "interval": "1m", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "Live Throughput", @@ -474,17 +474,17 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "(sum(rate(app_api_requests_total{job=~\"$job\",instance=~\"$instance\",status=~\"5..\"}[$__rate_interval])) or vector(0)) / \n(sum(rate(app_api_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) > 0 or vector(1)) * 100", "instant": false, "interval": "1m", "legendFormat": " ", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "API Error Rate", @@ -555,15 +555,15 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "time() - process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "System Uptime", @@ -622,16 +622,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", "interval": "1m", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "CPU Cores", @@ -697,15 +697,15 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "builder", "expr": "process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "Memory Usage", @@ -777,15 +777,15 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "builder", "expr": "app_database_storage_bytes{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "Database Size", @@ -852,15 +852,15 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "100 * app_ip_addresses_allocated_total{job=~\"$job\",instance=~\"$instance\"} / app_ip_addresses_total{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "Allocated", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "IP Address Pool Utilization", @@ -907,13 +907,13 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 2, "pointSize": 0, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { @@ -965,16 +965,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "rate(app_ngap_messages_total{job=~\"$job\",instance=~\"$instance\",}[$__rate_interval])", "interval": "1m", "legendFormat": "{{type}}", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "NGAP Message Rate", @@ -1015,9 +1015,9 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "showValues": false, - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -1067,15 +1067,15 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "builder", "expr": "app_registered_subscribers{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "Registered UEs", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "Registered Subscribers", @@ -1194,16 +1194,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "sum(rate(app_registration_attempts_total{job=~\"$job\",instance=~\"$instance\",result=\"accept\"}[$__rate_interval])) by (type)", "interval": "1m", "legendFormat": "{{type}} (accept)", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" }, { "datasource": { @@ -1258,9 +1258,9 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "showValues": false, - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -1310,15 +1310,15 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "builder", "expr": "app_pdu_sessions_total{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "Active PDU Sessions", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "Active PDU Sessions", @@ -1440,16 +1440,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "rate(app_pdu_session_establishment_attempts_total{job=~\"$job\",instance=~\"$instance\",result=\"accept\"}[$__rate_interval])", "interval": "1m", "legendFormat": "accept", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" }, { "datasource": { @@ -1714,16 +1714,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "rate(app_xdp_action_total{job=~\"$job\",instance=~\"$instance\",interface=\"n3\",action=\"XDP_PASS\"}[$__rate_interval])", "interval": "1m", "legendFormat": "PASS - Forwarded to network stack", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" }, { "datasource": { @@ -1871,16 +1871,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "rate(app_xdp_action_total{job=~\"$job\",instance=~\"$instance\",interface=\"n6\",action=\"XDP_PASS\"}[$__rate_interval])", "interval": "1m", "legendFormat": "PASS - Forwarded to network stack", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" }, { "datasource": { @@ -1981,7 +1981,7 @@ "h": 9, "w": 3, "x": 0, - "y": 69 + "y": 85 }, "id": 12, "options": { @@ -2004,16 +2004,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "sum(rate(app_api_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "interval": "1m", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "API Request Rate", @@ -2128,7 +2128,7 @@ "h": 9, "w": 13, "x": 3, - "y": 69 + "y": 85 }, "id": 4, "options": { @@ -2153,16 +2153,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "rate(app_api_requests_total{job=~\"$job\",instance=~\"$instance\",endpoint!~\"/api/v1/(pprof|metrics)\"}[$__rate_interval])", "interval": "1m", "legendFormat": "{{method}} {{endpoint}} ({{status}})", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "API Request Rate by Endpoint", @@ -2277,7 +2277,7 @@ "h": 9, "w": 8, "x": 16, - "y": 69 + "y": 85 }, "id": 9, "options": { @@ -2300,16 +2300,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "histogram_quantile(0.50, \n sum(rate(app_api_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",endpoint!~\"/api/v1/(pprof|metrics)\"}[$__rate_interval])) by (le)\n)", "interval": "1m", "legendFormat": "p50 (median)", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" }, { "datasource": { @@ -2373,7 +2373,7 @@ "h": 9, "w": 3, "x": 0, - "y": 78 + "y": 94 }, "id": 18, "options": { @@ -2396,16 +2396,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "100 *\nsum(rate(app_api_authentication_attempts_total{job=~\"$job\",instance=~\"$instance\",result=\"success\"}[$__rate_interval]))\n/\nsum(rate(app_api_authentication_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))\nand\nsum(rate(app_api_authentication_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) > 0\n", "interval": "1m", "legendFormat": "__auto", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" } ], "title": "API Authentication Success Rate", @@ -2505,7 +2505,7 @@ "h": 9, "w": 21, "x": 3, - "y": 78 + "y": 94 }, "id": 26, "options": { @@ -2528,16 +2528,16 @@ "pluginVersion": "12.3.1", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "editorMode": "code", "expr": "sum(rate(app_api_authentication_attempts_total{job=~\"$job\",instance=~\"$instance\",result=\"success\"}[$__rate_interval]))", "interval": "1m", "legendFormat": "Success", "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "refId": "A" }, { "datasource": { @@ -2570,689 +2570,690 @@ "y": 47 }, "id": 23, - "panels": [], - "title": "System Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Memory In-Use Bytes (Heap)", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "description": "Memory In-Use Bytes (Heap)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "bytes" }, - "thresholdsStyle": { - "mode": "off" - } + "overrides": [] }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 48 }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 48 - }, - "id": 41, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "max" + "id": 41, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Heap", + "range": true, + "refId": "A" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "title": "Memory Heap", + "type": "timeseries" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ { - "editorMode": "builder", - "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\",instance=~\"$instance\"}", - "legendFormat": "Heap", - "range": true, - "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" - } - } - ], - "title": "Memory Heap", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of CPU cores used by the Ella Core process", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 0, - "scaleDistribution": { - "type": "linear" + "description": "Number of CPU cores used by the Ella Core process", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 0, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 48 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "none" + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",}[$__rate_interval])", + "interval": "1m", + "legendFormat": "CPU Usage", + "range": true, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 48 - }, - "id": 42, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ { - "editorMode": "code", - "expr": "rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\",}[$__rate_interval])", - "interval": "1m", - "legendFormat": "CPU Usage", - "range": true, - "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" - } - } - ], - "title": "CPU Cores", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of goroutines", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepBefore", - "lineWidth": 2, - "pointSize": 0, - "scaleDistribution": { - "type": "linear" + "description": "Number of goroutines", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineWidth": 2, + "pointSize": 0, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "none" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 48 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 48 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "go_goroutines{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ { - "editorMode": "builder", - "expr": "go_goroutines{job=~\"$job\",instance=~\"$instance\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" - } - } - ], - "title": "Goroutines", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total Database file size", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "description": "Total Database file size", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 0, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "bytes" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 0, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] + "overrides": [] }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 56 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "editorMode": "builder", - "expr": "app_database_storage_bytes{job=~\"$job\",instance=~\"$instance\"}", - "legendFormat": " ", - "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } - } - ], - "title": "DB Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of database queries by table and operation type", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 56 }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 0, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 56 - }, - "id": 44, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "max" + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "app_database_storage_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": " ", + "range": true, + "refId": "A" + } ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true + "title": "DB Size", + "type": "timeseries" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ { - "editorMode": "code", - "expr": "rate(app_database_queries_total{job=~\"$job\",instance=~\"$instance\",}[$__rate_interval])", - "interval": "1m", - "legendFormat": "{{operation}} {{table}} ", - "range": true, - "refId": "A", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" - } - } - ], - "title": "DB Query Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "DB latency percentiles per bucket. For example, a datapoint for p99 at 3.987ms means that 99% of requests take less than 3.987ms.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 0, - "scaleDistribution": { - "type": "linear" + "description": "Rate of database queries by table and operation type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 0, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 56 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(app_database_queries_total{job=~\"$job\",instance=~\"$instance\",}[$__rate_interval])", + "interval": "1m", + "legendFormat": "{{operation}} {{table}} ", + "range": true, + "refId": "A" } + ], + "title": "DB Query Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "decimals": 3, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "description": "DB latency percentiles per bucket. For example, a datapoint for p99 at 3.987ms means that 99% of requests take less than 3.987ms.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 0, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 3, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.2 + } + ] + }, + "unit": "s" + }, + "overrides": [ { - "color": "green", - "value": 0 + "matcher": { + "id": "byName", + "options": "p50 (median)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] }, { - "color": "yellow", - "value": 0.05 + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] }, { - "color": "red", - "value": 0.2 + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] } ] }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "p50 (median)" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 56 }, - { - "matcher": { - "id": "byName", - "options": "p95" + "id": 40, + "options": { + "legend": { + "calcs": [ + "last", + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - { - "matcher": { - "id": "byName", - "options": "p99" + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(app_database_query_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le))", + "interval": "1m", + "legendFormat": "p50 (median)", + "range": true, + "refId": "A" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 56 - }, - "id": 40, - "options": { - "legend": { - "calcs": [ - "last", - "mean", - "max" + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(app_database_query_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le))", + "interval": "1m", + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(app_database_query_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le))", + "interval": "1m", + "legendFormat": "p99", + "range": true, + "refId": "C" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(app_database_query_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le))", - "interval": "1m", - "legendFormat": "p50 (median)", - "range": true, - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } - }, - { - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(app_database_query_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le))", - "interval": "1m", - "legendFormat": "p95", - "range": true, - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } - }, - { - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(app_database_query_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) by (le))", - "interval": "1m", - "legendFormat": "p99", - "range": true, - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } + "title": "DB Latency Percentiles", + "type": "timeseries" } ], - "title": "DB Latency Percentiles", - "type": "timeseries" + "title": "System Resources", + "type": "row" } ], "preload": false, @@ -3266,11 +3267,7 @@ "templating": { "list": [ { - "current": { - "text": "", - "value": "${DS_PROMETHEUS}", - "selected": true - }, + "current": {}, "includeAll": false, "label": "Data source", "name": "DS_PROMETHEUS", @@ -3325,13 +3322,14 @@ ] }, "time": { - "from": "now-6h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Ella Core Network Health", "uid": "adbqjzm", - "version": 1, - "weekStart": "" + "version": 9, + "weekStart": "", + "id": null } \ No newline at end of file diff --git a/internal/amf/nas/gmm/message/send.go b/internal/amf/nas/gmm/message/send.go index f9fa84d6e..f50bb28b2 100644 --- a/internal/amf/nas/gmm/message/send.go +++ b/internal/amf/nas/gmm/message/send.go @@ -33,7 +33,7 @@ func SendDLNASTransport(ctx context.Context, ue *amfContext.RanUe, payloadContai attribute.Int("pduSessionID", int(pduSessionID)), attribute.Int("cause", int(cause)), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -65,7 +65,7 @@ func SendIdentityRequest(ctx context.Context, ue *amfContext.RanUe, typeOfIdenti attribute.String("supi", ue.AmfUe.Supi), attribute.Int("typeOfIdentity", int(typeOfIdentity)), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -91,7 +91,7 @@ func SendAuthenticationRequest(ctx context.Context, amf *amfContext.AMF, ue *amf trace.WithAttributes( attribute.String("supi", ue.AmfUe.Supi), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -141,7 +141,7 @@ func SendServiceAccept(ctx context.Context, ue *amfContext.RanUe, pDUSessionStat attribute.Int("pduSessionIDErrorCount", len(errPduSessionID)), attribute.Int("causeErrorCount", len(errCause)), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -167,7 +167,7 @@ func SendAuthenticationReject(ctx context.Context, ue *amfContext.RanUe) error { trace.WithAttributes( attribute.String("supi", ue.AmfUe.Supi), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -194,7 +194,7 @@ func SendServiceReject(ctx context.Context, ue *amfContext.RanUe, cause uint8) e attribute.String("supi", ue.AmfUe.Supi), attribute.Int("cause", int(cause)), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -222,7 +222,7 @@ func SendRegistrationReject(ctx context.Context, ue *amfContext.RanUe, cause5GMM attribute.String("supi", ue.AmfUe.Supi), attribute.Int("cause", int(cause5GMM)), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -248,7 +248,7 @@ func SendSecurityModeCommand(ctx context.Context, amf *amfContext.AMF, ue *amfCo trace.WithAttributes( attribute.String("supi", ue.AmfUe.Supi), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -295,7 +295,7 @@ func SendDeregistrationAccept(ctx context.Context, ue *amfContext.RanUe) error { trace.WithAttributes( attribute.String("supi", ue.AmfUe.Supi), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -332,7 +332,7 @@ func SendRegistrationAccept( trace.WithAttributes( attribute.String("supi", ue.Supi), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() @@ -436,7 +436,7 @@ func SendConfigurationUpdateCommand(ctx context.Context, amf *amfContext.AMF, am trace.WithAttributes( attribute.String("supi", amfUe.Supi), ), - trace.WithSpanKind(trace.SpanKindServer), + trace.WithSpanKind(trace.SpanKindInternal), ) defer span.End() diff --git a/internal/amf/nas/handler.go b/internal/amf/nas/handler.go index 06385beb5..7c3e29451 100644 --- a/internal/amf/nas/handler.go +++ b/internal/amf/nas/handler.go @@ -68,10 +68,11 @@ func HandleNAS(ctx context.Context, amf *amfContext.AMF, ue *amfContext.RanUe, n msgTypeName := messageName(msg.GmmHeader.GetMessageType()) - ctx, span := tracer.Start(ctx, fmt.Sprintf("AMF NAS %s", msgTypeName), + ctx, span := tracer.Start(ctx, "NAS receive", + trace.WithSpanKind(trace.SpanKindInternal), trace.WithAttributes( - attribute.String("nas.messageType", msgTypeName), - attribute.String("supi", ue.AmfUe.Supi), + attribute.String("nas.message_type", msgTypeName), + attribute.String("ue.supi", ue.AmfUe.Supi), ), ) defer span.End() diff --git a/internal/amf/ngap/dispatcher.go b/internal/amf/ngap/dispatcher.go index 84105f69c..053f2420f 100644 --- a/internal/amf/ngap/dispatcher.go +++ b/internal/amf/ngap/dispatcher.go @@ -9,7 +9,6 @@ package ngap import ( "context" - "fmt" "reflect" amfContext "github.com/ellanetworks/core/internal/amf/context" @@ -66,34 +65,37 @@ func Dispatch(ctx context.Context, conn *sctp.SCTPConn, msg []byte) { return } + messageType := getMessageType(pdu) + + ctx, span := tracer.Start(ctx, "NGAP receive", + trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes( + attribute.String("ngap.message_type", messageType), + attribute.Int("ngap.pdu_category", pdu.Present), + attribute.Int("ngap.message_size", len(msg)), + attribute.String("network.protocol.name", "ngap"), + attribute.String("network.transport", "sctp"), + attribute.String("network.peer.address", remoteAddress.String()), + attribute.String("network.local.address", localAddress.String()), + ), + ) + defer span.End() + logger.LogNetworkEvent( ctx, logger.NGAPNetworkProtocol, - getMessageType(pdu), + messageType, logger.DirectionInbound, localAddress.String(), remoteAddress.String(), msg, ) - DispatchNgapMsg(amf, ran, pdu) + dispatchNgapMsg(ctx, amf, ran, pdu) } -func DispatchNgapMsg(amf *amfContext.AMF, ran *amfContext.Radio, pdu *ngapType.NGAPPDU) { - messageType := getMessageType(pdu) - - spanName := fmt.Sprintf("AMF NGAP %s", messageType) - - ctx, span := tracer.Start(context.Background(), spanName, - trace.WithAttributes( - attribute.String("ngap.pdu_present", fmt.Sprintf("%d", pdu.Present)), - attribute.String("ngap.messageType", messageType), - ), - trace.WithSpanKind(trace.SpanKindServer), - ) - defer span.End() - - NGAPMessages.WithLabelValues(messageType).Inc() +func dispatchNgapMsg(ctx context.Context, amf *amfContext.AMF, ran *amfContext.Radio, pdu *ngapType.NGAPPDU) { + NGAPMessages.WithLabelValues(getMessageType(pdu)).Inc() switch pdu.Present { case ngapType.NGAPPDUPresentInitiatingMessage: diff --git a/internal/amf/ngap/send/send.go b/internal/amf/ngap/send/send.go index 33bcc65af..90813d2f8 100644 --- a/internal/amf/ngap/send/send.go +++ b/internal/amf/ngap/send/send.go @@ -24,11 +24,14 @@ type RealNGAPSender struct { } func (s *RealNGAPSender) SendToRan(ctx context.Context, packet []byte, msgType NGAPProcedure) error { - ctx, span := tracer.Start(ctx, "Send To RAN", + ctx, span := tracer.Start(ctx, "NGAP send", + trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes( - attribute.String("ngap.messageType", string(msgType)), + attribute.String("ngap.message_type", string(msgType)), + attribute.Int("ngap.message_size", len(packet)), + attribute.String("network.protocol.name", "ngap"), + attribute.String("network.transport", "sctp"), ), - trace.WithSpanKind(trace.SpanKindClient), ) defer span.End() diff --git a/internal/api/server/authentication_middleware.go b/internal/api/server/authentication_middleware.go index d9dc2cb7a..ddf48a530 100644 --- a/internal/api/server/authentication_middleware.go +++ b/internal/api/server/authentication_middleware.go @@ -65,8 +65,8 @@ func authenticateRequest(r *http.Request, jwtSecret []byte, store *db.Database) } }() - ctx, span := tracer.Start(r.Context(), "Authenticate", - trace.WithAttributes(), + ctx, span := tracer.Start(r.Context(), "API authenticate", + trace.WithSpanKind(trace.SpanKindServer), ) defer span.End() diff --git a/internal/ausf/ue_authentication.go b/internal/ausf/ue_authentication.go index f2c6f2644..e1fc36df4 100644 --- a/internal/ausf/ue_authentication.go +++ b/internal/ausf/ue_authentication.go @@ -21,11 +21,11 @@ import ( var tracer = otel.Tracer("ella-core/ausf") func UeAuthPostRequestProcedure(ctx context.Context, suci string, snName string, resyncInfo *models.ResynchronizationInfo) (*models.Av5gAka, error) { - ctx, span := tracer.Start( - ctx, - "AUSF UEAuthentication PostRequest", + ctx, span := tracer.Start(ctx, "AUSF authenticate", + trace.WithSpanKind(trace.SpanKindInternal), trace.WithAttributes( attribute.String("ue.suci", suci), + attribute.String("ausf.serving_network", snName), ), ) defer span.End() diff --git a/internal/smf/pdusession/handle_create_sm_context.go b/internal/smf/pdusession/handle_create_sm_context.go index 1afd5cdbe..d340f347a 100644 --- a/internal/smf/pdusession/handle_create_sm_context.go +++ b/internal/smf/pdusession/handle_create_sm_context.go @@ -28,10 +28,12 @@ import ( var tracer = otel.Tracer("ella-core/smf") func CreateSmContext(ctx context.Context, supi string, pduSessionID uint8, dnn string, snssai *models.Snssai, n1Msg []byte) (string, []byte, error) { - ctx, span := tracer.Start(ctx, "SMF Create SmContext", + ctx, span := tracer.Start(ctx, "SMF create session", + trace.WithSpanKind(trace.SpanKindInternal), trace.WithAttributes( - attribute.String("supi", supi), - attribute.Int("pduSessionID", int(pduSessionID)), + attribute.String("ue.supi", supi), + attribute.Int("smf.pdu_session_id", int(pduSessionID)), + attribute.String("smf.dnn", dnn), ), ) defer span.End() diff --git a/internal/smf/pdusession/handle_deactivate_sm_context.go b/internal/smf/pdusession/handle_deactivate_sm_context.go index b08827314..fc462baf0 100644 --- a/internal/smf/pdusession/handle_deactivate_sm_context.go +++ b/internal/smf/pdusession/handle_deactivate_sm_context.go @@ -13,11 +13,10 @@ import ( ) func DeactivateSmContext(ctx context.Context, smContextRef string) error { - ctx, span := tracer.Start( - ctx, - "SMF Deactivate SmContext", + ctx, span := tracer.Start(ctx, "SMF deactivate session", + trace.WithSpanKind(trace.SpanKindInternal), trace.WithAttributes( - attribute.String("smf.smContextRef", smContextRef), + attribute.String("smf.context_ref", smContextRef), ), ) defer span.End() diff --git a/internal/smf/pdusession/handle_release_sm_context.go b/internal/smf/pdusession/handle_release_sm_context.go index 8998b1ff4..a21d5f3d6 100644 --- a/internal/smf/pdusession/handle_release_sm_context.go +++ b/internal/smf/pdusession/handle_release_sm_context.go @@ -12,11 +12,10 @@ import ( ) func ReleaseSmContext(ctx context.Context, smContextRef string) error { - ctx, span := tracer.Start( - ctx, - "SMF Release SmContext", + ctx, span := tracer.Start(ctx, "SMF release session", + trace.WithSpanKind(trace.SpanKindInternal), trace.WithAttributes( - attribute.String("smf.smContextRef", smContextRef), + attribute.String("smf.context_ref", smContextRef), ), ) defer span.End() diff --git a/internal/upf/core/pfcp_session_handlers.go b/internal/upf/core/pfcp_session_handlers.go index b1d44e0d7..0692305fc 100644 --- a/internal/upf/core/pfcp_session_handlers.go +++ b/internal/upf/core/pfcp_session_handlers.go @@ -12,6 +12,7 @@ import ( "github.com/wmnsk/go-pfcp/ie" "github.com/wmnsk/go-pfcp/message" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -37,7 +38,9 @@ func (u UpfPfcpHandler) HandlePfcpSessionModificationRequest(ctx context.Context } func HandlePfcpSessionEstablishmentRequest(ctx context.Context, msg *message.SessionEstablishmentRequest) (*message.SessionEstablishmentResponse, error) { - ctx, span := tracer.Start(ctx, "UPF Session Establishment") + ctx, span := tracer.Start(ctx, "UPF establish session", + trace.WithSpanKind(trace.SpanKindInternal), + ) defer span.End() conn := GetConnection() @@ -185,7 +188,9 @@ func HandlePfcpSessionEstablishmentRequest(ctx context.Context, msg *message.Ses } func HandlePfcpSessionDeletionRequest(ctx context.Context, msg *message.SessionDeletionRequest) (*message.SessionDeletionResponse, error) { - _, span := tracer.Start(ctx, "UPF Session Delete") + _, span := tracer.Start(ctx, "UPF delete session", + trace.WithSpanKind(trace.SpanKindInternal), + ) defer span.End() conn := GetConnection() @@ -232,7 +237,9 @@ func HandlePfcpSessionDeletionRequest(ctx context.Context, msg *message.SessionD } func HandlePfcpSessionModificationRequest(ctx context.Context, msg *message.SessionModificationRequest) (*message.SessionModificationResponse, error) { - ctx, span := tracer.Start(ctx, "UPF Session Modify") + ctx, span := tracer.Start(ctx, "UPF modify session", + trace.WithSpanKind(trace.SpanKindInternal), + ) defer span.End() conn := GetConnection()