From 89ec5c3921a663c5aaf7231583b47bff120e7150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 16:22:21 +0100 Subject: [PATCH 1/9] Suspected bug in informer cache access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../source/informer/ManagedInformerEventSource.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java index 978deda333..0811e57290 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java @@ -193,6 +193,11 @@ public void handleRecentResourceCreate(ResourceID resourceID, R resource) { public Optional get(ResourceID resourceID) { Optional resource = temporaryResourceCache.getResourceFromCache(resourceID); var res = cache.get(resourceID); + if (log.isDebugEnabled()) { + log.debug( + "Latest sync version: {}", + manager().lastSyncResourceVersion(resourceID.getNamespace().orElse(null))); + } if (comparableResourceVersions && resource.isPresent() && ReconcilerUtilsInternal.compareResourceVersions( @@ -203,8 +208,9 @@ public Optional get(ResourceID resourceID) { return resource; } log.debug( - "Resource not found, or older, in temporary cache. Found in informer cache {}, for" - + " Resource ID: {}", + "Resource found in temp cache: {}, or older, in temporary cache. Found in informer cache" + + " {}, for Resource ID: {}", + resource.isPresent(), res.isPresent(), resourceID); return res; From 9cda53561018290cd3a252664d64871cd62398b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 16:36:56 +0100 Subject: [PATCH 2/9] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../informer/ManagedInformerEventSource.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java index 0811e57290..f761a2061e 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java @@ -191,6 +191,7 @@ public void handleRecentResourceCreate(ResourceID resourceID, R resource) { @Override public Optional get(ResourceID resourceID) { + // order of getting those resource from cache matters Optional resource = temporaryResourceCache.getResourceFromCache(resourceID); var res = cache.get(resourceID); if (log.isDebugEnabled()) { @@ -200,10 +201,16 @@ public Optional get(ResourceID resourceID) { } if (comparableResourceVersions && resource.isPresent() - && ReconcilerUtilsInternal.compareResourceVersions( - resource.get().getMetadata().getResourceVersion(), - manager().lastSyncResourceVersion(resource.get().getMetadata().getNamespace())) - > 0) { + // it can happen here that we receive an event after we read the resource from the informer + // cache + // that bumps the lastSync version, but we read the resource before. In that case we want to + // return + // the resource from temp cache. + && (res.isEmpty() + || ReconcilerUtilsInternal.compareResourceVersions( + resource.get().getMetadata().getResourceVersion(), + manager().lastSyncResourceVersion(resource.get().getMetadata().getNamespace())) + > 0)) { log.debug("Latest resource found in temporary cache for Resource ID: {}", resourceID); return resource; } From aeffc779d76be432ce666f00105171cf8c59b3ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 16:37:48 +0100 Subject: [PATCH 3/9] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../event/source/informer/ManagedInformerEventSource.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java index f761a2061e..a139b1da8a 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java @@ -202,10 +202,8 @@ public Optional get(ResourceID resourceID) { if (comparableResourceVersions && resource.isPresent() // it can happen here that we receive an event after we read the resource from the informer - // cache - // that bumps the lastSync version, but we read the resource before. In that case we want to - // return - // the resource from temp cache. + // cache that bumps the lastSync version, but we read the resource before. In that case we + // want to return the resource from temp cache. && (res.isEmpty() || ReconcilerUtilsInternal.compareResourceVersions( resource.get().getMetadata().getResourceVersion(), From 7f2a3698c5d20cfb9c09fc38199c074bcd7c64e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 16:59:32 +0100 Subject: [PATCH 4/9] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../event/source/informer/ManagedInformerEventSource.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java index a139b1da8a..7bdeb2289a 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java @@ -194,11 +194,6 @@ public Optional get(ResourceID resourceID) { // order of getting those resource from cache matters Optional resource = temporaryResourceCache.getResourceFromCache(resourceID); var res = cache.get(resourceID); - if (log.isDebugEnabled()) { - log.debug( - "Latest sync version: {}", - manager().lastSyncResourceVersion(resourceID.getNamespace().orElse(null))); - } if (comparableResourceVersions && resource.isPresent() // it can happen here that we receive an event after we read the resource from the informer @@ -207,7 +202,7 @@ public Optional get(ResourceID resourceID) { && (res.isEmpty() || ReconcilerUtilsInternal.compareResourceVersions( resource.get().getMetadata().getResourceVersion(), - manager().lastSyncResourceVersion(resource.get().getMetadata().getNamespace())) + res.get().getMetadata().getResourceVersion()) > 0)) { log.debug("Latest resource found in temporary cache for Resource ID: {}", resourceID); return resource; From 12aadc75948b55414255e11b4a9200da187728f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 17:03:45 +0100 Subject: [PATCH 5/9] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../event/source/informer/ManagedInformerEventSource.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java index 7bdeb2289a..1828aa5533 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java @@ -196,9 +196,6 @@ public Optional get(ResourceID resourceID) { var res = cache.get(resourceID); if (comparableResourceVersions && resource.isPresent() - // it can happen here that we receive an event after we read the resource from the informer - // cache that bumps the lastSync version, but we read the resource before. In that case we - // want to return the resource from temp cache. && (res.isEmpty() || ReconcilerUtilsInternal.compareResourceVersions( resource.get().getMetadata().getResourceVersion(), From 834ae6a190f236b6eace32677b75248efcc9f9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 17:09:52 +0100 Subject: [PATCH 6/9] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../event/source/informer/ManagedInformerEventSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java index 1828aa5533..964af20fe9 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java @@ -191,7 +191,7 @@ public void handleRecentResourceCreate(ResourceID resourceID, R resource) { @Override public Optional get(ResourceID resourceID) { - // order of getting those resource from cache matters + // The order of reading from these caches matters Optional resource = temporaryResourceCache.getResourceFromCache(resourceID); var res = cache.get(resourceID); if (comparableResourceVersions From 33a2f3f862b405c4d0ffed236f5cee84df734237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 20:40:38 +0100 Subject: [PATCH 7/9] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../CachingFilteringUpdateReconciler.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/cachingfilteringupdate/CachingFilteringUpdateReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/cachingfilteringupdate/CachingFilteringUpdateReconciler.java index 1bd60eb2c9..9958bf8ad9 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/cachingfilteringupdate/CachingFilteringUpdateReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/cachingfilteringupdate/CachingFilteringUpdateReconciler.java @@ -43,19 +43,30 @@ public UpdateControl reconcile( CachingFilteringUpdateCustomResource resource, Context context) { - context.resourceOperations().serverSideApply(prepareCM(resource)); + context.resourceOperations().serverSideApply(prepareCM(resource, 1)); var cachedCM = context.getSecondaryResource(ConfigMap.class); if (cachedCM.isEmpty()) { issueFound.set(true); throw new IllegalStateException("Error for resource: " + ResourceID.fromResource(resource)); } + var updated = context.resourceOperations().serverSideApply(prepareCM(resource, 2)); + cachedCM = context.getSecondaryResource(ConfigMap.class); + if (!cachedCM + .orElseThrow() + .getMetadata() + .getResourceVersion() + .equals(updated.getMetadata().getResourceVersion())) { + issueFound.set(true); + throw new IllegalStateException("Error for resource: " + ResourceID.fromResource(resource)); + } + ensureStatusExists(resource); resource.getStatus().setUpdated(true); return UpdateControl.patchStatus(resource); } - private static ConfigMap prepareCM(CachingFilteringUpdateCustomResource p) { + private static ConfigMap prepareCM(CachingFilteringUpdateCustomResource p, int num) { var cm = new ConfigMapBuilder() .withMetadata( @@ -63,7 +74,7 @@ private static ConfigMap prepareCM(CachingFilteringUpdateCustomResource p) { .withName(p.getMetadata().getName()) .withNamespace(p.getMetadata().getNamespace()) .build()) - .withData(Map.of("name", p.getMetadata().getName())) + .withData(Map.of("name", p.getMetadata().getName(), "num", "" + num)) .build(); cm.addOwnerReference(p); return cm; From 2c7ba273e0ae2d8ae8cdeeb64ac108286d0f6349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 20:41:10 +0100 Subject: [PATCH 8/9] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../CachingFilteringUpdateReconciler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/cachingfilteringupdate/CachingFilteringUpdateReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/cachingfilteringupdate/CachingFilteringUpdateReconciler.java index 9958bf8ad9..0e9a953129 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/cachingfilteringupdate/CachingFilteringUpdateReconciler.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/cachingfilteringupdate/CachingFilteringUpdateReconciler.java @@ -58,7 +58,8 @@ public UpdateControl reconcile( .getResourceVersion() .equals(updated.getMetadata().getResourceVersion())) { issueFound.set(true); - throw new IllegalStateException("Error for resource: " + ResourceID.fromResource(resource)); + throw new IllegalStateException( + "Update error for resource: " + ResourceID.fromResource(resource)); } ensureStatusExists(resource); From d9b90ed93f2db4cc83976ae7628f5ae956549086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Mar 2026 21:48:55 +0100 Subject: [PATCH 9/9] fix: extremely rare race condition when getting the resource from cache - alt 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../informer/ManagedInformerEventSource.java | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java index 964af20fe9..7026ae86e2 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/ManagedInformerEventSource.java @@ -194,15 +194,28 @@ public Optional get(ResourceID resourceID) { // The order of reading from these caches matters Optional resource = temporaryResourceCache.getResourceFromCache(resourceID); var res = cache.get(resourceID); - if (comparableResourceVersions - && resource.isPresent() - && (res.isEmpty() - || ReconcilerUtilsInternal.compareResourceVersions( - resource.get().getMetadata().getResourceVersion(), - res.get().getMetadata().getResourceVersion()) - > 0)) { - log.debug("Latest resource found in temporary cache for Resource ID: {}", resourceID); - return resource; + if (comparableResourceVersions && resource.isPresent()) { + var comp = + ReconcilerUtilsInternal.compareResourceVersions( + resource.get().getMetadata().getResourceVersion(), + manager().lastSyncResourceVersion(resource.get().getMetadata().getNamespace())); + if (comp > 0) { + log.debug("Latest resource found in temporary cache for Resource ID: {}", resourceID); + return resource; + } else { + log.debug( + "Latest resource not found in temporary but was obsolete: {}. Re-reading the informer" + + " cache: {}", + resourceID, + res.isEmpty()); + if (res.isEmpty()) { + // re-reading resource from informer cache since an add even might have received + // after the first read. + return cache.get(resourceID); + } else { + return res; + } + } } log.debug( "Resource found in temp cache: {}, or older, in temporary cache. Found in informer cache"