From ceb3f6e783eba649e9412483b32274d3822a8de3 Mon Sep 17 00:00:00 2001 From: Michael Mraka Date: Fri, 30 Oct 2026 12:01:42 +0100 Subject: [PATCH] remove unused vmaas_cache simplify code remove feature unused since 2024 due to memory issues --- ...hts-patchman-engine-general.configmap.yaml | 220 ---------- evaluator/evaluate.go | 39 +- evaluator/evaluate_test.go | 389 ++++-------------- evaluator/metrics.go | 17 +- evaluator/template_advisory_e2e_test.go | 1 - evaluator/template_advisory_test.go | 76 +--- evaluator/vmaas_cache.go | 84 ---- go.mod | 1 - go.sum | 2 - 9 files changed, 104 insertions(+), 725 deletions(-) delete mode 100644 evaluator/vmaas_cache.go diff --git a/dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml b/dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml index e45fddaee..9564898a3 100644 --- a/dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml +++ b/dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml @@ -4925,226 +4925,6 @@ data: "title": "Package cache size", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Hit ratio", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - - ], - "mappings": [ - - ], - "max": 1, - "min": 0, - "noValue": "0%", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 90 - }, - "id": 92, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(patchman_engine_evaluator_vmaas_cache{type=\"hit\"})/sum(patchman_engine_evaluator_vmaas_cache)", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "VMaaS cache hit ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "items", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - - ], - "mappings": [ - - ], - "noValue": "0%", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 90 - }, - "id": 93, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(patchman_engine_evaluator_vmaas_cache_size) by (container)", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "VMaaS cache size", - "type": "timeseries" - }, { "collapsed": false, "gridPos": { diff --git a/evaluator/evaluate.go b/evaluator/evaluate.go index 9a66bfea6..bace192f1 100644 --- a/evaluator/evaluate.go +++ b/evaluator/evaluate.go @@ -11,7 +11,6 @@ import ( "app/base/types" "app/base/utils" "app/base/vmaas" - "app/tasks" "context" "net/http" "sync" @@ -19,7 +18,6 @@ import ( "github.com/bytedance/sonic" "github.com/google/uuid" - "github.com/jinzhu/copier" "github.com/pkg/errors" "go.opentelemetry.io/otel/trace" "gorm.io/gorm" @@ -53,9 +51,6 @@ var ( preloadPackageCache bool packageCacheSize int packageNameCacheSize int - enableVmaasCache bool - vmaasCacheSize int - vmaasCacheCheckDuration time.Duration vmaasCallMaxRetries int vmaasCallUseExpRetry bool vmaasCallUseOptimisticUpdates bool @@ -120,12 +115,6 @@ func configureEvaluator() { packageCacheSize = utils.PodConfig.GetInt("package_cache_size", 1000000) // Size of package name cache packageNameCacheSize = utils.PodConfig.GetInt("package_name_cache_size", 60000) - // Toggle caching of vmaas responses - enableVmaasCache = utils.PodConfig.GetBool("vmaas_cache", true) - // Size of vmaas response cache - vmaasCacheSize = utils.PodConfig.GetInt("vmaas_cache_size", 10000) - // Interval to check vmaas API if there was a data change thus if cache is still valid - vmaasCacheCheckDuration = time.Duration(utils.PodConfig.GetInt("vmaas_cache_check_duration_sec", 60)) * time.Second // How many tries before we consider vmaad API call failed vmaasCallMaxRetries = utils.PodConfig.GetInt("vmaas_call_max_retries", 8) // Use exponential delay between two vmaas calls @@ -362,17 +351,6 @@ func getUpdatesData(ctx context.Context, system *models.SystemPlatformV2) (*vmaa func getVmaasUpdates(ctx context.Context, system *models.SystemPlatformV2) (*vmaas.UpdatesV3Response, error) { defer utils.ObserveSecondsSince(time.Now(), evaluationPartDuration.WithLabelValues("vmaas-updates-prepare")) - var vmaasDataCopy vmaas.UpdatesV3Response - // first check if we have data in cache - vmaasData, ok := memoryVmaasCache.Get(system.Inventory.JSONChecksum) - if ok { - // return copy of vmaasData to avoid modification of cached data e.g. by templates - err := copier.CopyWithOption(&vmaasDataCopy, vmaasData, copier.Option{DeepCopy: true}) - if err != nil { - return nil, err - } - return &vmaasDataCopy, nil - } updatesReq, err := tryGetVmaasRequest(system) if err != nil { return nil, errors.Wrap(err, "unable to get vmaas request") @@ -388,20 +366,11 @@ func getVmaasUpdates(ctx context.Context, system *models.SystemPlatformV2) (*vma updatesReq.OptimisticUpdates = utils.PtrBool(useOptimisticUpdates) updatesReq.EpochRequired = utils.PtrBool(true) - vmaasData, err = callVMaas(ctx, updatesReq) + vmaasData, err := callVMaas(ctx, updatesReq) if err != nil { evaluationCnt.WithLabelValues("error-call-vmaas-updates").Inc() return nil, errors.Wrap(err, "vmaas API call failed") } - - if memoryVmaasCache.enabled { - // store copy of vmaasData to cache to avoid modification of cached data e.g. by templates - err := copier.CopyWithOption(&vmaasDataCopy, vmaasData, copier.Option{DeepCopy: true}) - if err != nil { - return nil, err - } - memoryVmaasCache.Add(system.Inventory.JSONChecksum, &vmaasDataCopy) - } return vmaasData, nil } @@ -824,12 +793,6 @@ func evaluateHandler(ctx context.Context, m mqueue.KafkaMessage) error { func loadCache() { memoryPackageCache = NewPackageCache(enablePackageCache, preloadPackageCache, packageCacheSize, packageNameCacheSize) memoryPackageCache.Load() - tx := tasks.CancelableDB() - memoryVmaasCache = NewVmaasPackageCache(tx, enableVmaasCache, vmaasCacheSize, vmaasCacheCheckDuration) - if memoryVmaasCache.enabled { - // no need to check cache validity when cache is not enabled - go memoryVmaasCache.CheckValidity(tx) - } } func run(wg *sync.WaitGroup, readerBuilder mqueue.CreateReader) { diff --git a/evaluator/evaluate_test.go b/evaluator/evaluate_test.go index cc1a46ad6..198f68c43 100644 --- a/evaluator/evaluate_test.go +++ b/evaluator/evaluate_test.go @@ -55,13 +55,7 @@ func TestEvaluate(t *testing.T) { utils.SkipWithoutDB(t) utils.SkipWithoutPlatform(t) core.SetupTestEnvironment() - // don't use vmaas-cache since tests here are not using vmaas_json - // so it will always get the same result from cache for empty vmaas_json configure() - - enableVmaasCache = false - defer func() { enableVmaasCache = true }() - loadCache() mockWriter := mqueue.MockKafkaWriter{} remediationsPublisher = &mockWriter @@ -235,292 +229,116 @@ func TestGetYumUpdates(t *testing.T) { assert.Equal(t, 2, len(updateList.GetAvailableUpdates())) } -// nolint:funlen -func TestSatelliteSystemAdvisories(t *testing.T) { - utils.SkipWithoutDB(t) - utils.SkipWithoutPlatform(t) - core.SetupTestEnvironment() - - ogYumUpdatesEval := enableYumUpdatesEval - enableYumUpdatesEval = true - defer func() { enableYumUpdatesEval = ogYumUpdatesEval }() - - configure() - loadCache() - mockWriter := mqueue.MockKafkaWriter{} - remediationsPublisher = &mockWriter - - vmaasJSON := ` - { - "package_list": [ - "git-2.30.1-1.el8_8.x86_64", - "sqlite-3.21.0-1.el8_6.x86_64" - ], - "repository_list": [ - "rhel-8-for-x86_64-appstream-rpms" - ], - "releasever": "8", - "basearch": "x86_64", - "latest_only": true - } - ` - // this satellite system has 2 git and 1 sqlite advisories reported by vmaas (APPLICABLE) - vmaasDataResp := ` - { - "update_list": { - "git-2.30.1-1.el8_8.x86_64": { - "available_updates": [ - { - "erratum": "RHSA-2023:3246", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "git-2.39.3-1.el8_8.x86_64", - "package_name": "git", - "evra": "0:2.39.3-1.el8_8.x86_64" - }, - { - "erratum": "RHSA-2023:3240", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "git-2.39.4-1.el8_8.x86_64", - "package_name": "git", - "evra": "0:2.39.4-1.el8_8.x86_64" - } - ] - }, - "sqlite-3.21.0-1.el8_6.x86_64": { - "available_updates": [ - { - "erratum": "RHSA-2022:7100", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "sqlite-3.26.0-16.el8_6.x86_64", - "package_name": "sqlite", - "evra": "0:3.26.0-16.el8_6.x86_64" - } - ] - } +// platform mock /updates returns RH-1 and RH-2 for firefox, and RH-100 for kernel. +const platformPackagesVmaasJSON = `{ + "package_list": [ + "firefox-0:76.0.1-1.fc31.x86_64", + "kernel-0:5.6.13-200.fc31.x86_64" + ], + "repository_list": ["repo1"] +}` + +// RH-1 matches the mock firefox update, so the merge keeps the yum copy. +// RH-200 is yum-only. +var platformYumUpdates = []byte(`{ + "update_list": { + "firefox-0:76.0.1-1.fc31.x86_64": { + "available_updates": [ + { + "erratum": "RH-1", + "basearch": "i686", + "releasever": "ser1", + "repository": "repo1", + "package": "firefox-0:77.0.1-1.fc31.x86_64" + } + ] + }, + "kernel-0:5.6.13-200.fc31.x86_64": { + "available_updates": [ + { + "erratum": "RH-200", + "basearch": "i686", + "releasever": "ser1", + "repository": "repo1", + "package": "kernel-0:5.10.0-1.fc31.x86_64" + } + ] } } - ` - - var vmaasData vmaas.UpdatesV3Response - err := sonic.Unmarshal([]byte(vmaasDataResp), &vmaasData) - assert.Nil(t, err) +}`) - // lets add the checksum to the cache, so we do not actually call vmaas - vmaasJSONChecksum := "1337" - memoryVmaasCache.Add(&vmaasJSONChecksum, &vmaasData) - - // this satellite system has 1 git installable advisory which is the same as the applicable one from vmaas - // and 1 sqlite different installable advisory - yumUpdatesRaw := []byte(` - { - "update_list": { - "git-2.30.1-1.el8_8.x86_64": { - "available_updates": [ - { - "erratum": "RHSA-2023:3246", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "git-0:2.39.3-1.el8_8.x86_64" - } - ] - }, - "sqlite-3.21.0-1.el8_6.x86_64": { - "available_updates": [ - { - "erratum": "RHSA-2022:7108", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "sqlite-3.26.0-16.el8_6.x86_64" - } - ] - } - } +func updateStatuses(resp *vmaas.UpdatesV3Response) (int, map[string]int) { + statuses := map[string]int{} + total := 0 + for _, updates := range resp.GetUpdateList() { + for _, update := range updates.GetAvailableUpdates() { + total++ + statuses[update.GetErratum()] = update.StatusID } - `) + } + return total, statuses +} - system := &models.SystemPlatformV2{ +func platformUpdatesSystem(inventoryID, displayName string, satellite, bootc bool) *models.SystemPlatformV2 { + vmaasJSON := platformPackagesVmaasJSON + return &models.SystemPlatformV2{ Inventory: models.SystemInventory{ - InventoryID: uuid.MustParse("99999999-0000-0000-0000-000000000015"), - JSONChecksum: &vmaasJSONChecksum, + InventoryID: uuid.MustParse(inventoryID), VmaasJSON: &vmaasJSON, - YumUpdates: yumUpdatesRaw, - DisplayName: "satellite_system_test1", + YumUpdates: platformYumUpdates, + DisplayName: displayName, RhAccountID: 1, BuiltPkgcache: true, - SatelliteManaged: true, + SatelliteManaged: satellite, + Bootc: bootc, }, - Patch: models.SystemPatch{}, - } - - result, err := getUpdatesData(base.Context, system) - assert.Nil(t, err) - - // result should have 2 git advisories, 1 is installable (taken from yum updates and vmaas, merged) - // 1 is applicable (taken from vmaas) - // 2 sqlite advisories, 1 is installable (taken from yum updates) - // 1 is applicable (taken from vmaas) - var installableCnt, applicableCnt int - for _, updates := range result.GetUpdateList() { - for _, update := range updates.GetAvailableUpdates() { - switch update.StatusID { - case INSTALLABLE: - installableCnt++ - case APPLICABLE: - applicableCnt++ - } - } } - assert.Equal(t, 2, installableCnt) - assert.Equal(t, 2, applicableCnt) } -// nolint:funlen -func TestBootcSystemAdvisories(t *testing.T) { +func checkPlatformUpdates(t *testing.T, system *models.SystemPlatformV2, expected map[string]int) { + t.Helper() utils.SkipWithoutDB(t) utils.SkipWithoutPlatform(t) core.SetupTestEnvironment() ogYumUpdatesEval := enableYumUpdatesEval enableYumUpdatesEval = true - defer func() { enableYumUpdatesEval = ogYumUpdatesEval }() + t.Cleanup(func() { enableYumUpdatesEval = ogYumUpdatesEval }) configure() loadCache() - mockWriter := mqueue.MockKafkaWriter{} - remediationsPublisher = &mockWriter - - vmaasJSON := ` - { - "package_list": [ - "git-2.30.1-1.el8_8.x86_64", - "sqlite-3.21.0-1.el8_6.x86_64" - ], - "repository_list": [ - "rhel-8-for-x86_64-appstream-rpms" - ], - "releasever": "8", - "basearch": "x86_64", - "latest_only": true - } - ` - vmaasDataResp := ` - { - "update_list": { - "git-2.30.1-1.el8_8.x86_64": { - "available_updates": [ - { - "erratum": "RHSA-2023:3246", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "git-2.39.3-1.el8_8.x86_64", - "package_name": "git", - "evra": "0:2.39.3-1.el8_8.x86_64" - }, - { - "erratum": "RHSA-2023:3240", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "git-2.39.4-1.el8_8.x86_64", - "package_name": "git", - "evra": "0:2.39.4-1.el8_8.x86_64" - } - ] - }, - "sqlite-3.21.0-1.el8_6.x86_64": { - "available_updates": [ - { - "erratum": "RHSA-2022:7100", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "sqlite-3.26.0-16.el8_6.x86_64", - "package_name": "sqlite", - "evra": "0:3.26.0-16.el8_6.x86_64" - } - ] - } - } - } - ` - - var vmaasData vmaas.UpdatesV3Response - err := sonic.Unmarshal([]byte(vmaasDataResp), &vmaasData) - assert.Nil(t, err) - - vmaasJSONChecksum := "bootc-1337" - memoryVmaasCache.Add(&vmaasJSONChecksum, &vmaasData) - - yumUpdatesRaw := []byte(` - { - "update_list": { - "git-2.30.1-1.el8_8.x86_64": { - "available_updates": [ - { - "erratum": "RHSA-2023:3246", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "git-0:2.39.3-1.el8_8.x86_64" - } - ] - }, - "sqlite-3.21.0-1.el8_6.x86_64": { - "available_updates": [ - { - "erratum": "RHSA-2022:7108", - "basearch": "x86_64", - "releasever": "8", - "repository": "rhel-8-for-x86_64-appstream-rpms", - "package": "sqlite-3.26.0-16.el8_6.x86_64" - } - ] - } - } - } - `) - - system := &models.SystemPlatformV2{ - Inventory: models.SystemInventory{ - InventoryID: uuid.MustParse("99999999-0000-0000-0000-000000000016"), - JSONChecksum: &vmaasJSONChecksum, - VmaasJSON: &vmaasJSON, - YumUpdates: yumUpdatesRaw, - DisplayName: "bootc_system_test1", - RhAccountID: 1, - BuiltPkgcache: true, - Bootc: true, - }, - Patch: models.SystemPatch{}, - } + remediationsPublisher = &mqueue.MockKafkaWriter{} result, err := getUpdatesData(base.Context, system) assert.Nil(t, err) assert.NotNil(t, result) + total, statuses := updateStatuses(result) + assert.Equal(t, len(expected), total) + assert.Equal(t, expected, statuses) +} - // bootc systems cannot be patched via Insights: every update must be applicable - var installableCnt, applicableCnt int - for _, updates := range result.GetUpdateList() { - for _, update := range updates.GetAvailableUpdates() { - switch update.StatusID { - case INSTALLABLE: - installableCnt++ - case APPLICABLE: - applicableCnt++ - } - } - } - assert.Equal(t, 0, installableCnt) - assert.Equal(t, 4, applicableCnt) +func TestSatelliteSystemAdvisories(t *testing.T) { + // VMaaS updates are applicable. Matching yum RH-1 stays installable; RH-200 is yum-only. + checkPlatformUpdates(t, + platformUpdatesSystem("99999999-0000-0000-0000-000000000015", "satellite_system_test1", true, false), + map[string]int{ + "RH-1": INSTALLABLE, + "RH-2": APPLICABLE, + "RH-100": APPLICABLE, + "RH-200": INSTALLABLE, + }) +} + +func TestBootcSystemAdvisories(t *testing.T) { + // Image-mode systems cannot be patched via Insights, so every update is applicable. + checkPlatformUpdates(t, + platformUpdatesSystem("99999999-0000-0000-0000-000000000016", "bootc_system_test1", false, true), + map[string]int{ + "RH-1": APPLICABLE, + "RH-2": APPLICABLE, + "RH-100": APPLICABLE, + "RH-200": APPLICABLE, + }) } func TestCallVmaas400(t *testing.T) { @@ -548,44 +366,3 @@ func TestGetUpdatesDataVmaas400(t *testing.T) { assert.Nil(t, err) assert.Nil(t, res) } - -func TestGetVmaasDataCached(t *testing.T) { - utils.SkipWithoutDB(t) - utils.SkipWithoutPlatform(t) - configure() - loadCache() - req := vmaas.UpdatesV3Request{ - PackageList: []string{"firefox-0:76.0.1-1.fc31.x86_64"}, RepositoryList: []string{"repo"}, - } - reqJSON, _ := sonic.Marshal(req) - reqString := string(reqJSON) - chsum := "123" - sp := &models.SystemPlatformV2{Inventory: models.SystemInventory{VmaasJSON: &reqString, JSONChecksum: &chsum}} - - var assertInstallable = func() (*vmaas.UpdatesV3Response, []vmaas.UpdatesV3ResponseAvailableUpdates) { - vmaasData, _ := getVmaasUpdates(base.Context, sp) - updates := (*vmaasData.UpdateList)["firefox-0:76.0.1-1.fc31.x86_64"].GetAvailableUpdates() - assert.Equal(t, INSTALLABLE, updates[0].StatusID) - return vmaasData, updates - } - vmaasData, updates := assertInstallable() - - // modify update status id, change data returned by vmaas - var setApplicable = func() { - updates[0].StatusID = APPLICABLE - modifiedUpdateList := vmaas.UpdatesV3ResponseUpdateList{AvailableUpdates: &updates} - (*vmaasData.UpdateList)["firefox-0:76.0.1-1.fc31.x86_64"] = &modifiedUpdateList - updates = (*vmaasData.UpdateList)["firefox-0:76.0.1-1.fc31.x86_64"].GetAvailableUpdates() - assert.Equal(t, APPLICABLE, updates[0].StatusID) - } - setApplicable() - - // cached value mustn't be changed - vmaasData, updates = assertInstallable() - - // modify data again to change data returned from cache - setApplicable() - - // cached value mustn't be changed - assertInstallable() -} diff --git a/evaluator/metrics.go b/evaluator/metrics.go index 11c085fa1..54f75b6c2 100644 --- a/evaluator/metrics.go +++ b/evaluator/metrics.go @@ -68,26 +68,11 @@ var ( Subsystem: "evaluator", Name: "package_cache_size", }, []string{"by"}) - - vmaasCacheCnt = prometheus.NewCounterVec(prometheus.CounterOpts{ - Help: "How many vmaas checksums hit/miss cache", - Namespace: "patchman_engine", - Subsystem: "evaluator", - Name: "vmaas_cache", - }, []string{"type"}) - - vmaasCacheGauge = prometheus.NewGauge(prometheus.GaugeOpts{ - Help: "VMaaS cache size", - Namespace: "patchman_engine", - Subsystem: "evaluator", - Name: "vmaas_cache_size", - }) ) func RunMetrics() { prometheus.MustRegister(evaluationCnt, updatesCnt, evaluationDuration, evaluationPartDuration, - uploadEvaluationDelay, twoEvaluationsInterval, packageCacheCnt, packageCacheGauge, - vmaasCacheCnt, vmaasCacheGauge) + uploadEvaluationDelay, twoEvaluationsInterval, packageCacheCnt, packageCacheGauge) // create web app app := gin.New() diff --git a/evaluator/template_advisory_e2e_test.go b/evaluator/template_advisory_e2e_test.go index eae6e2a86..50d710daf 100644 --- a/evaluator/template_advisory_e2e_test.go +++ b/evaluator/template_advisory_e2e_test.go @@ -21,7 +21,6 @@ import ( func prepareTemplateAdvisoryIntegrationTest() { configure() enableTemplateAdvisoryEval = true - enableVmaasCache = false loadCache() remediationsPublisher = &mqueue.MockKafkaWriter{} } diff --git a/evaluator/template_advisory_test.go b/evaluator/template_advisory_test.go index 8759acb4f..17e2fd9df 100644 --- a/evaluator/template_advisory_test.go +++ b/evaluator/template_advisory_test.go @@ -68,61 +68,28 @@ func TestLoadTemplateAdvisoryErrata(t *testing.T) { assert.True(t, ok) } -//nolint:funlen func TestGetUpdatesDataTemplateAdvisoryEval(t *testing.T) { - vmaasJSON := ` - { - "package_list": ["git-2.30.1-1.el8_8.x86_64"], - "repository_list": ["rhel-8-for-x86_64-appstream-rpms"], - "releasever": "8", - "basearch": "x86_64", - "latest_only": true - } - ` - vmaasDataResp := ` - { + // yum update must not be merged; template installability comes from VMaaS errata only + yumUpdatesRaw := []byte(`{ "update_list": { - "git-2.30.1-1.el8_8.x86_64": { + "firefox-0:76.0.1-1.fc31.x86_64": { "available_updates": [ { - "erratum": "RH-1", - "package": "git-2.39.3-1.el8_8.x86_64" - }, - { - "erratum": "RH-2", - "package": "git-2.39.4-1.el8_8.x86_64" + "erratum": "RH-YUM-ONLY", + "package": "firefox-9:9.9.9-1.fc31.x86_64" } ] } } - } - ` - yumUpdatesRaw := []byte(` - { - "update_list": { - "git-2.30.1-1.el8_8.x86_64": { - "available_updates": [ - { - "erratum": "RH-100", - "package": "git-9.9.9-1.el8_8.x86_64" - } - ] - } - } - } - `) - - var vmaasData vmaas.UpdatesV3Response - assert.Nil(t, sonic.Unmarshal([]byte(vmaasDataResp), &vmaasData)) - vmaasJSONChecksum := "template-advisory-eval-test" + }`) + vmaasJSON := platformPackagesVmaasJSON templateID := int64(1) system := &models.SystemPlatformV2{ Inventory: models.SystemInventory{ - RhAccountID: 1, - JSONChecksum: &vmaasJSONChecksum, - VmaasJSON: &vmaasJSON, - YumUpdates: yumUpdatesRaw, + RhAccountID: 1, + VmaasJSON: &vmaasJSON, + YumUpdates: yumUpdatesRaw, }, Patch: models.SystemPatch{ TemplateID: &templateID, @@ -130,6 +97,7 @@ func TestGetUpdatesDataTemplateAdvisoryEval(t *testing.T) { } utils.SkipWithoutDB(t) + utils.SkipWithoutPlatform(t) core.SetupTestEnvironment() configure() loadCache() @@ -143,7 +111,6 @@ func TestGetUpdatesDataTemplateAdvisoryEval(t *testing.T) { enableTemplateAdvisoryEval = ogTemplateEval }() - memoryVmaasCache.Add(&vmaasJSONChecksum, &vmaasData) database.CreateTemplateAdvisories(t, 1, templateID, []int64{1}) defer database.DeleteTemplateAdvisories(t, templateID, []int64{1}) @@ -151,19 +118,14 @@ func TestGetUpdatesDataTemplateAdvisoryEval(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, result) - var installableCnt, applicableCnt int - for _, updates := range result.GetUpdateList() { - for _, update := range updates.GetAvailableUpdates() { - switch update.StatusID { - case INSTALLABLE: - installableCnt++ - case APPLICABLE: - applicableCnt++ - } - } - } - assert.Equal(t, 1, installableCnt) - assert.Equal(t, 1, applicableCnt) + // platform /updates returns RH-1, RH-2, and RH-100. Template advisory id 1 is RH-1. + total, statuses := updateStatuses(result) + assert.Equal(t, 3, total) + assert.Equal(t, INSTALLABLE, statuses["RH-1"]) + assert.Equal(t, APPLICABLE, statuses["RH-2"]) + assert.Equal(t, APPLICABLE, statuses["RH-100"]) + _, yumMerged := statuses["RH-YUM-ONLY"] + assert.False(t, yumMerged) } func TestUseTemplateAdvisoryEval(t *testing.T) { diff --git a/evaluator/vmaas_cache.go b/evaluator/vmaas_cache.go deleted file mode 100644 index 24a3ebcd1..000000000 --- a/evaluator/vmaas_cache.go +++ /dev/null @@ -1,84 +0,0 @@ -package evaluator - -import ( - "app/base/types" - "app/base/utils" - "app/base/vmaas" - "app/tasks/vmaas_sync" - "time" - - lru "github.com/hashicorp/golang-lru/v2" - "gorm.io/gorm" -) - -var memoryVmaasCache *VmaasCache - -type VmaasCache struct { - enabled bool - size int - currentSize int - validity *types.Rfc3339TimestampWithZ - checkDuration time.Duration - data *lru.TwoQueueCache[string, *vmaas.UpdatesV3Response] -} - -func NewVmaasPackageCache(tx *gorm.DB, enabled bool, size int, checkDuration time.Duration) *VmaasCache { - c := new(VmaasCache) - - c.enabled = enabled - c.size = size - c.currentSize = 0 - c.validity = vmaas_sync.GetLastSync(tx, vmaas_sync.VmaasExported) - c.checkDuration = checkDuration - vmaasCacheGauge.Set(0) - - if c.enabled { - var err error - c.data, err = lru.New2Q[string, *vmaas.UpdatesV3Response](c.size) - if err != nil { - panic(err) - } - return c - } - return c -} - -func (c *VmaasCache) Get(checksum *string) (*vmaas.UpdatesV3Response, bool) { - if c.enabled && checksum != nil { - val, ok := c.data.Get(*checksum) - if ok { - vmaasCacheCnt.WithLabelValues("hit").Inc() - utils.LogTrace("checksum", *checksum, "VmaasCache.Get cache hit") - return val, true - } - } - vmaasCacheCnt.WithLabelValues("miss").Inc() - return nil, false -} - -func (c *VmaasCache) Add(checksum *string, response *vmaas.UpdatesV3Response) { - if c.enabled && checksum != nil { - c.data.Add(*checksum, response) - if c.currentSize <= c.size { - c.currentSize++ - vmaasCacheGauge.Inc() - } - } -} - -func (c *VmaasCache) Reset(ts *types.Rfc3339TimestampWithZ) { - if c.enabled { - c.data.Purge() - c.validity = ts - vmaasCacheGauge.Set(0) - } -} - -func (c *VmaasCache) CheckValidity(tx *gorm.DB) { - for range time.Tick(c.checkDuration) { - lastModifiedTS := vmaas_sync.GetLastSync(tx, vmaas_sync.VmaasExported) - if lastModifiedTS == nil || c.validity == nil || c.validity.Time().Before(*lastModifiedTS.Time()) { - c.Reset(lastModifiedTS) - } - } -} diff --git a/go.mod b/go.mod index fdc29b4dc..12e8df1a4 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/golang-migrate/migrate/v4 v4.20.1 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/jinzhu/copier v0.4.0 github.com/joho/godotenv v1.5.1 github.com/lestrrat-go/backoff/v2 v2.0.8 github.com/lib/pq v1.12.3 diff --git a/go.sum b/go.sum index f83bd1b05..a89b28b07 100644 --- a/go.sum +++ b/go.sum @@ -175,8 +175,6 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jeremija/gosubmit v0.2.8 h1:mmSITBz9JxVtu8eqbN+zmmwX7Ij2RidQxhcwRVI4wqA= github.com/jeremija/gosubmit v0.2.8/go.mod h1:Ui+HS073lCFREXBbdfrJzMB57OI/bdxTiLtrDHHhFPI= -github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= -github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=