diff --git a/agents/prometheus.go b/agents/prometheus.go index f4f0cc25a..5f237d455 100644 --- a/agents/prometheus.go +++ b/agents/prometheus.go @@ -27,7 +27,6 @@ import ( "github.com/cgrates/cgrates/cores" "github.com/cgrates/cgrates/engine" "github.com/cgrates/cgrates/utils" - "github.com/cgrates/ltcache" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -165,7 +164,7 @@ func NewPrometheusAgent(cfg *config.CGRConfig, cm *engine.ConnManager) *Promethe Subsystem: "cache", Name: "groups_total", Help: "Total number of cache groups", - }, []string{"cache"}) + }, []string{"cache", "node_id"}) reg.MustRegister(cacheGroupsMetric) cacheItemsMetric := prometheus.NewGaugeVec( @@ -174,7 +173,7 @@ func NewPrometheusAgent(cfg *config.CGRConfig, cm *engine.ConnManager) *Promethe Subsystem: "cache", Name: "items_total", Help: "Total number of cache items", - }, []string{"cache"}) + }, []string{"cache", "node_id"}) reg.MustRegister(cacheItemsMetric) statMetrics := prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -255,24 +254,26 @@ func (pa *PrometheusAgent) updateCacheStats() { return } for _, connID := range pa.cfg.PrometheusAgentCfg().CacheSConns { - var cacheStats map[string]*ltcache.CacheStats + var reply engine.CacheStatsWithMetadata if err := pa.cm.Call(context.Background(), []string{connID}, - utils.CacheSv1GetCacheStats, + utils.CacheSv1GetStats, &utils.AttrCacheIDsWithAPIOpts{ CacheIDs: pa.cfg.PrometheusAgentCfg().CacheIDs, - }, &cacheStats); err != nil { + }, &reply); err != nil { utils.Logger.Err(fmt.Sprintf( "<%s> failed to retrieve cache stats (connID=%q): %v", utils.PrometheusAgent, connID, err)) continue } + cacheStats := reply.CacheStatistics + nodeID := utils.IfaceAsString(reply.Metadata[utils.NodeID]) for cacheID, stats := range cacheStats { if stats == nil { continue } - pa.cacheGroupsMetric.WithLabelValues(cacheID).Set(float64(stats.Groups)) - pa.cacheItemsMetric.WithLabelValues(cacheID).Set(float64(stats.Items)) + pa.cacheGroupsMetric.WithLabelValues(cacheID, nodeID).Set(float64(stats.Groups)) + pa.cacheItemsMetric.WithLabelValues(cacheID, nodeID).Set(float64(stats.Items)) } } } diff --git a/apier/v1/caches.go b/apier/v1/caches.go index 23f997525..0e51b1baf 100644 --- a/apier/v1/caches.go +++ b/apier/v1/caches.go @@ -90,6 +90,12 @@ func (chSv1 *CacheSv1) GetCacheStats(ctx *context.Context, args *utils.AttrCache return chSv1.cacheS.V1GetCacheStats(ctx, args, rply) } +// GetStats returns CacheStats filtered by cacheIDs +func (chSv1 *CacheSv1) GetStats(ctx *context.Context, args *utils.AttrCacheIDsWithAPIOpts, + rply *engine.CacheStatsWithMetadata) error { + return chSv1.cacheS.V1GetStats(ctx, args, rply) +} + // PrecacheStatus checks status of active precache processes func (chSv1 *CacheSv1) PrecacheStatus(ctx *context.Context, args *utils.AttrCacheIDsWithAPIOpts, rply *map[string]string) error { return chSv1.cacheS.V1PrecacheStatus(ctx, args, rply) diff --git a/docs/prometheus.rst b/docs/prometheus.rst index 01ea4ce5d..6fc739360 100644 --- a/docs/prometheus.rst +++ b/docs/prometheus.rst @@ -7,7 +7,7 @@ PrometheusAgent 1. **Core metrics** - collected from configured CGRateS engines via CoreSv1.Status API 2. **StatQueue metrics** - values from CGRateS :ref:`StatS ` component, collected via StatSv1.GetQueueFloatMetrics API -3. **Cache statistics** - collected from configured :ref:`CacheS ` components via CacheSv1.GetCacheStats API +3. **Cache statistics** - collected from configured :ref:`CacheS ` components via CacheSv1.GetStats API For core metrics, the agent computes real-time values on each Prometheus scrape request. For StatQueue metrics, it retrieves the current state of the stored StatQueues without additional calculations. For cache statistics, it collects current cache utilization data from the configured cache partitions. @@ -103,9 +103,10 @@ The PrometheusAgent exposes the following metrics: go_memstats_alloc_bytes{node_id="e94160b"} 1.1360808e+07 3. **Cache Metrics** (when caches_conns is configured) - - Two separate metrics for cache statistics: ``cgrates_cache_groups_total`` and ``cgrates_cache_items_total`` with cache partition ID label + - Two separate metrics for cache statistics: ``cgrates_cache_groups_total`` and ``cgrates_cache_items_total`` with cache partition ID and node_id labels - Obtained from CacheS services on each scrape request - Useful for identifying memory usage patterns and potential performance issues + - Includes node_id labels for multi-engine environments, allowing collection from multiple CGRateS engines Example of cache metrics output: @@ -113,15 +114,15 @@ The PrometheusAgent exposes the following metrics: # HELP cgrates_cache_groups_total Total number of cache groups # TYPE cgrates_cache_groups_total gauge - cgrates_cache_groups_total{cache="*attribute_filter_indexes"} 2 - cgrates_cache_groups_total{cache="*charger_profiles"} 0 - cgrates_cache_groups_total{cache="*rpc_connections"} 0 + cgrates_cache_groups_total{cache="*attribute_filter_indexes",node_id="dc2cb63"} 2 + cgrates_cache_groups_total{cache="*charger_profiles",node_id="dc2cb63"} 0 + cgrates_cache_groups_total{cache="*rpc_connections",node_id="dc2cb63"} 0 # HELP cgrates_cache_items_total Total number of cache items # TYPE cgrates_cache_items_total gauge - cgrates_cache_items_total{cache="*attribute_filter_indexes"} 6 - cgrates_cache_items_total{cache="*charger_profiles"} 2 - cgrates_cache_items_total{cache="*rpc_connections"} 1 + cgrates_cache_items_total{cache="*attribute_filter_indexes",node_id="dc2cb63"} 6 + cgrates_cache_items_total{cache="*charger_profiles",node_id="dc2cb63"} 2 + cgrates_cache_items_total{cache="*rpc_connections",node_id="dc2cb63"} 1 How It Works diff --git a/engine/caches.go b/engine/caches.go index 8cc7285f6..f9e055df2 100644 --- a/engine/caches.go +++ b/engine/caches.go @@ -443,6 +443,22 @@ func (chS *CacheS) V1GetCacheStats(ctx *context.Context, args *utils.AttrCacheID return } +type CacheStatsWithMetadata struct { + CacheStatistics map[string]*ltcache.CacheStats + Metadata map[string]any +} + +func (chS *CacheS) V1GetStats(ctx *context.Context, args *utils.AttrCacheIDsWithAPIOpts, + rply *CacheStatsWithMetadata) error { + *rply = CacheStatsWithMetadata{ + CacheStatistics: chS.tCache.GetCacheStats(args.CacheIDs), + Metadata: map[string]any{ + utils.NodeID: chS.cfg.GeneralCfg().NodeID, + }, + } + return nil +} + func (chS *CacheS) V1PrecacheStatus(ctx *context.Context, args *utils.AttrCacheIDsWithAPIOpts, rply *map[string]string) (err error) { if len(args.CacheIDs) == 0 { args.CacheIDs = utils.CachePartitions.AsSlice() diff --git a/general_tests/prometheus_it_test.go b/general_tests/prometheus_it_test.go index 4c13ec2e0..4773c0da5 100644 --- a/general_tests/prometheus_it_test.go +++ b/general_tests/prometheus_it_test.go @@ -61,8 +61,13 @@ func TestPrometheusAgentIT(t *testing.T) { "prometheus_agent": { "enabled": true, "path": "/metrics", - "collect_go_metrics": false, - "collect_process_metrics": false, + "caches_conns": ["*localhost", "external"], + "cache_ids": [ + "*statqueue_profiles", + "*statqueues", + "*stat_filter_indexes", + "*rpc_connections" + ], "stats_conns": ["*localhost", "external"], "stat_queue_ids": ["cgrates.org:SQ_1","SQ_2"] } diff --git a/utils/consts.go b/utils/consts.go index 921e6625f..abbd58e05 100644 --- a/utils/consts.go +++ b/utils/consts.go @@ -2010,6 +2010,7 @@ const ( const ( CacheSv1 = "CacheSv1" CacheSv1GetCacheStats = "CacheSv1.GetCacheStats" + CacheSv1GetStats = "CacheSv1.GetStats" CacheSv1GetItemIDs = "CacheSv1.GetItemIDs" CacheSv1HasItem = "CacheSv1.HasItem" CacheSv1GetItem = "CacheSv1.GetItem"