Skip to content

Commit c9941b5

Browse files
YanniHu1996armru
andcommitted
fix: avoid write-read concurrency on cached cluster (#2884)
Closes: #2863 Signed-off-by: YanniHu1996 <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> (cherry picked from commit f77af89)
1 parent 59b4b4c commit c9941b5

5 files changed

Lines changed: 23 additions & 14 deletions

File tree

‎internal/management/cache/cache.go‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,22 @@ func LoadEnv(c string) ([]string, error) {
5959
return nil, ErrUnsupportedObject
6060
}
6161

62-
// LoadCluster loads a key from the local cache
63-
func LoadCluster() (*apiv1.Cluster, error) {
62+
// StoreCluster write a cluster object into the local cache
63+
func StoreCluster(cluster *apiv1.Cluster) {
64+
// We need to make a copy of the cluster object, because
65+
// the cluster object contains attribute with concurrent unsafe type
66+
// such as map, slice, etc.
67+
cache.Store(ClusterKey, cluster.DeepCopy())
68+
}
69+
70+
// LoadClusterUnsafe retrieves a cluster from the local cache.
71+
// Warning:
72+
//
73+
// The returned pointer can potentially be accessed concurrently.
74+
// Only use this function for read-only operations. If you need to
75+
// modify the cluster, always create a DeepCopy of the returned object
76+
// before writing to it.
77+
func LoadClusterUnsafe() (*apiv1.Cluster, error) {
6478
value, ok := cache.Load(ClusterKey)
6579
if !ok {
6680
return nil, ErrCacheMiss

‎internal/management/controller/cache.go‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,8 @@ import (
3333
// updateCacheFromCluster will update the internal cache with the cluster
3434
//
3535
// returns true if the update was not total, and should be retried
36-
func (r *InstanceReconciler) updateCacheFromCluster(
37-
ctx context.Context, cluster *apiv1.Cluster,
38-
) shoudRequeue {
39-
cache.Store(cache.ClusterKey, cluster)
36+
func (r *InstanceReconciler) updateCacheFromCluster(ctx context.Context, cluster *apiv1.Cluster) shoudRequeue {
37+
cache.StoreCluster(cluster)
4038

4139
var requeue shoudRequeue
4240

@@ -50,10 +48,7 @@ func (r *InstanceReconciler) updateCacheFromCluster(
5048
return requeue
5149
}
5250

53-
func (r *InstanceReconciler) updateWALRestoreSettingsCache(
54-
ctx context.Context,
55-
cluster *apiv1.Cluster,
56-
) {
51+
func (r *InstanceReconciler) updateWALRestoreSettingsCache(ctx context.Context, cluster *apiv1.Cluster) {
5752
_, env, barmanConfiguration, err := walrestore.GetRecoverConfiguration(cluster, r.instance.PodName)
5853
if errors.Is(err, walrestore.ErrNoBackupConfigured) {
5954
cache.Delete(cache.WALRestoreKey)

‎pkg/management/postgres/webserver/local.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func (ws *localWebserverEndpoints) serveCache(w http.ResponseWriter, r *http.Req
8383
var js []byte
8484
switch requestedObject {
8585
case cache.ClusterKey:
86-
response, err := cache.LoadCluster()
86+
response, err := cache.LoadClusterUnsafe()
8787
if errors.Is(err, cache.ErrCacheMiss) {
8888
w.WriteHeader(http.StatusNotFound)
8989
return

‎pkg/management/postgres/webserver/metricserver/pg_collector.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ func (e *Exporter) setTimestampMetric(
430430
errorLabel string,
431431
getTimestampFunc func(cluster *apiv1.Cluster) string,
432432
) {
433-
cluster, err := cache.LoadCluster()
433+
cluster, err := cache.LoadClusterUnsafe()
434434
// there isn't a cached object yet
435435
if errors.Is(err, cache.ErrCacheMiss) {
436436
return
@@ -472,7 +472,7 @@ func (e *Exporter) setTimestampMetric(
472472
func (e *Exporter) collectNodesUsed() {
473473
const notExtractedValue float64 = -1
474474

475-
cluster, err := cache.LoadCluster()
475+
cluster, err := cache.LoadClusterUnsafe()
476476
if err != nil {
477477
log.Error(err, "unable to collect metrics")
478478
e.Metrics.Error.Set(1)

‎pkg/management/postgres/webserver/metricserver/wal.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ func collectPGWalSettings(exporter *Exporter, db *sql.DB) error {
219219
}
220220

221221
func getWalVolumeSize() float64 {
222-
cluster, err := cache.LoadCluster()
222+
cluster, err := cache.LoadClusterUnsafe()
223223
if err != nil || !cluster.ShouldCreateWalArchiveVolume() {
224224
return 0
225225
}

0 commit comments

Comments
 (0)