Merge pull request #20 from lucannez64/fix/metal-iosurface-drawable-starvation

Fix Metal drawable starvation in mixed IOSurface layer trees

docs/TESTING.md

@@ -90,6 +90,7 @@ Useful probes (build tokenized Python helpers from `build/runtime/tools/`):
 - `m327-cgl-resolver-check`: first calls through dynamically looked-up GL functions retain floating-point arguments; expected `first_sampler_linear=1 first_color_arguments=1 errors=0,0`. Build with `build-m58f327-opengl.py` and run with `run-riot-native-check.py m327-cgl-resolver-check`.
 - `m327-cgl-direct-check`: with `RCLOL_GPU_DIRECT_OTHER_WINDOW_TEST=1` and an owned 337x339 native target, interleave 338x339 fallback surfaces; expected `active_presenter_preserved=1 fallback_frames=357`. This needs the native presentation fixture; it is not a `run-riot-native-check.py` choice.
 - `riot-metal-presentation-check`: Vulkan clear reaches the GL texture, including late window/context attachment. Set `RIOT_PRESENTATION_NESTED=1` to place the Metal layer beneath two ordinary layers and verify that its drawables are still presented and recycled past the three-drawable pool limit.
+- `riot-ca-render-policy-check`: headless check of the actual compositor render policy. A nested Metal layer must still be prepared when a full-window IOSurface is present; surface-only trees should retain the local-render shortcut. Built by `build-riot-metal-presentation.py`; expected `PASS: mixed-tree Metal preparation survives IOSurface shortcut`. Creates no window or GPU work and does not publish its test surface.
 - `riot-metal-concurrency-check`: submission ordering, resource lifetime and concurrent recording.
 
 `RIOT_PRESENTATION_STRESS=1` makes the presentation probe run 120 frames. With the game and clients closed, force async encoding for this standalone probe using the guest flag `/tmp/metal-async-encode`, then remove it. The tested old path leaked 120 synchronization FDs over 120 frames; the fixed path leaked zero. Inspect the probe's host `/proc/PID/fd` for `anon_inode:syncobj_file` if reproducing that test.

patches/cocotron.patch

@@ -8703,10 +8703,10 @@ index 5dcac115..5de4f03f 100644
 +
  @end
 diff --git a/QuartzCore/CALayerContext.m b/QuartzCore/CALayerContext.m
-index 5dce8c56..dee2b80c 100644
+index 5dce8c56..7f9125e3 100644
 --- a/QuartzCore/CALayerContext.m
 +++ b/QuartzCore/CALayerContext.m
-@@ -1,47 +1,58 @@
+@@ -1,47 +1,69 @@
  #import <Foundation/NSString.h>
 +#import <Foundation/NSMutableArray.h>
  #import <QuartzCore/CALayer.h>
@@ -8720,6 +8720,17 @@ index 5dce8c56..dee2b80c 100644
  
  @class CAMetalLayerInternal;
  
++/* A flattened IOSurface only replaces local rendering for trees without
++ * Metal content. Skipping a mixed tree also skips prepareRender, exhausting
++ * its drawable pool even though presentation callbacks continue to run. */
++static BOOL hasMetalLayer(CALayer *layer) {
++    if ([[layer class] isSubclassOfClass: [CAMetalLayerInternal class]])
++        return YES;
++    for (CALayer *child in layer.sublayers)
++        if (hasMetalLayer(child))return YES;
++    return NO;
++}
++
 +/* The X11 transport presents Chromium's flattened full-window surface.
 + * Avoid uploading and compositing that same window again into a covered GL
 + * child. Restrict this to the explicitly configured native presentation path. */
@@ -8784,7 +8795,7 @@ index 5dce8c56..dee2b80c 100644
      [super dealloc];
  }
  
-@@ -51,12 +62,22 @@
+@@ -51,12 +73,22 @@
  }
  
  - (void) setLayer: (CALayer *) layer {
@@ -8807,7 +8818,7 @@ index 5dce8c56..dee2b80c 100644
  }
  
  - (void) setSubwindow: (CGSubWindow*) subwindow
-@@ -68,16 +89,50 @@
+@@ -68,16 +100,50 @@
      }
  
      _subwindow = [subwindow retain];
@@ -8859,7 +8870,7 @@ index 5dce8c56..dee2b80c 100644
  }
  
  - (void) assignTextureIdsToLayerTree: (CALayer *) layer {
-@@ -93,7 +148,22 @@
+@@ -93,7 +159,22 @@
          [self assignTextureIdsToLayerTree: child];
  }
  
@@ -8882,7 +8893,7 @@ index 5dce8c56..dee2b80c 100644
      CGLContextMakeCurrentAndAttachToWindow(_glContext, _cglWindow);
  
      glEnable(GL_DEPTH_TEST);
-@@ -118,27 +188,63 @@
+@@ -118,27 +199,63 @@
  
      [_deleteTextureIds removeAllObjects];
  
@@ -8907,7 +8918,7 @@ index 5dce8c56..dee2b80c 100644
 +    if (_cglWindow == NULL || _layer == nil)
 +        return;
 +    if(getenv("DARLING_CA_SKIP_LOCAL_IOSURFACE_RENDER") &&
-+       hasFullWindowSurface(_layer,_frame.size))return;
++       !hasMetalLayer(_layer) && hasFullWindowSurface(_layer,_frame.size))return;
      [self renderLayer: _layer];
  }
  
@@ -8951,7 +8962,7 @@ index 5dce8c56..dee2b80c 100644
  }
  
  - (void) startTimerIfNeeded {
-@@ -147,7 +253,7 @@
+@@ -147,7 +264,7 @@
                                                     target: self
                                                   selector: @selector(timer:)
                                                   userInfo: nil
@@ -8960,13 +8971,13 @@ index 5dce8c56..dee2b80c 100644
  }
  
  - (void) deleteTextureId: (NSNumber *) textureId {
-@@ -155,7 +261,10 @@
+@@ -155,7 +272,10 @@
  }
  
  - (void) flush {
 -    CGLFlushDrawable(_glContext);
 +    if(getenv("DARLING_CA_SKIP_LOCAL_IOSURFACE_RENDER") && _layer &&
-+       hasFullWindowSurface(_layer,_frame.size))return;
++       !hasMetalLayer(_layer) && hasFullWindowSurface(_layer,_frame.size))return;
 +    if (_glContext != NULL && _cglWindow != NULL)
 +        CGLFlushDrawable(_glContext);
  }

tools/build-riot-metal-presentation.py

@@ -27,10 +27,12 @@ for name, suffix in [('CALayer', 'm'), ('CALayerContext', 'm'), ('CATransaction'
     link[link.index('CMakeFiles/QuartzCore.dir/' + name + '.' + suffix + '.o')] = str(obj)
 link[link.index('-o') + 1] = str(root / 'build/riot-QuartzCore')
 subprocess.run(link, cwd=folder, check=True)
-obj = root / 'build/riot-metal-presentation-check.o'
-subprocess.run(['clang++', *flags['CXX_DEFINES'], *flags['CXX_INCLUDES'], *flags['CXX_FLAGS'],
-                '-c', str(root / 'tools/riot-metal-presentation-check.mm'), '-o', str(obj)], check=True)
-subprocess.run(['clang++', '-target', 'x86_64-apple-macosx11.0', '-fuse-ld=lld', '-nostdlib',
-                '-isysroot', sdk, str(obj), '-lSystem', '-lc++', '-lobjc',
-                '-framework', 'AppKit', '-framework', 'OpenGL', '-framework', 'Foundation', str(root / 'build/riot-Metal'),
-                str(root / 'build/riot-QuartzCore'), '-o', str(root / 'build/riot-metal-presentation-check')], check=True)
+for name in ('riot-metal-presentation-check', 'riot-ca-render-policy-check'):
+    obj = root / f'build/{name}.o'
+    subprocess.run(['clang++', *flags['CXX_DEFINES'], *flags['CXX_INCLUDES'], *flags['CXX_FLAGS'],
+                    '-c', str(root / f'tools/{name}.mm'), '-o', str(obj)], check=True)
+    subprocess.run(['clang++', '-target', 'x86_64-apple-macosx11.0', '-fuse-ld=lld', '-nostdlib',
+                    '-isysroot', sdk, str(obj), '-lSystem', '-lc++', '-lobjc',
+                    '-framework', 'AppKit', '-framework', 'OpenGL', '-framework', 'Foundation',
+                    '-framework', 'IOSurface', str(root / 'build/riot-Metal'),
+                    str(root / 'build/riot-QuartzCore'), '-o', str(root / f'build/{name}')], check=True)

tools/riot-ca-render-policy-check.mm

@@ -0,0 +1,82 @@
+#import <Foundation/Foundation.h>
+#import <QuartzCore/CALayer.h>
+#import <QuartzCore/CALayerContext.h>
+#import "CAMetalLayerInternal.h"
+#import <IOSurface/IOSurface.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+@interface CALayerContext (RenderPolicyCheckPrivate)
+- (void)prepareMetalLayers:(CALayer*)layer;
+@end
+static unsigned prepared, renderCalls;
+@interface RenderPolicyMetalLayer : CAMetalLayerInternal
+@end
+@implementation RenderPolicyMetalLayer
+- (void)prepareRender { ++prepared; }
+@end
+@interface RenderPolicySurfaceLayer : CALayer { IOSurfaceRef _probeSurface; }
+- (id)initWithSurface:(IOSurfaceRef)surface;
+@end
+@implementation RenderPolicySurfaceLayer
+- (id)initWithSurface:(IOSurfaceRef)surface {
+ self=[super init]; _probeSurface=surface; return self;
+}
+- (id)contents { return (id)_probeSurface; }
+@end
+@interface RenderPolicyContext : CALayerContext
+- (id)initWithRoot:(CALayer*)root;
+@end
+@implementation RenderPolicyContext
+- (id)initWithRoot:(CALayer*)root {
+ self=[super initWithFrame:CGRectMake(0,0,16,16)];
+ _layer=[root retain]; _cglWindow=(void*)1; return self;
+}
+- (void)renderLayer:(CALayer*)layer { ++renderCalls; [self prepareMetalLayers:layer]; }
+@end
+int main() {
+ // No NSApplication, window, graphics context, GPU work, or published IOSurface.
+ // Objects intentionally live until process exit; the sentinel is never passed to CGL.
+ [NSAutoreleasePool new];
+ int dimensions[]={16,16,4,64};
+ const void* keys[]={kIOSurfaceWidth,kIOSurfaceHeight,kIOSurfaceBytesPerElement,kIOSurfaceBytesPerRow};
+ const void* values[4];
+ for(unsigned i=0;i<4;++i) values[i]=CFNumberCreate(NULL,kCFNumberIntType,&dimensions[i]);
+ CFDictionaryRef properties=CFDictionaryCreate(NULL,keys,values,4,&kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks);
+ IOSurfaceRef surface=IOSurfaceCreate(properties);
+ if(!surface) return 2;
+ CALayer* root=[CALayer new];
+ CALayer* container=[CALayer new];
+ RenderPolicyMetalLayer* metal=[RenderPolicyMetalLayer new];
+ [container addSublayer:metal];
+ [root addSublayer:container];
+ RenderPolicyContext* context=[[RenderPolicyContext alloc] initWithRoot:root];
+ setenv("DARLING_CA_SKIP_LOCAL_IOSURFACE_RENDER","1",1);
+ for(unsigned i=0;i<3;++i) [context render];
+ printf("metal_only: prepared=%u expected=3\n",prepared);
+ if(prepared!=3) return 3;
+ RenderPolicySurfaceLayer* surfaceLayer=[[RenderPolicySurfaceLayer alloc] initWithSurface:surface];
+ [root addSublayer:surfaceLayer];
+ prepared=0;
+ for(unsigned i=0;i<3;++i) [context render];
+ printf("mixed_tree_shortcut_enabled: prepared=%u expected=3\n",prepared);
+ if(prepared!=3) return 4;
+ [root setSublayers:@[surfaceLayer,container]];
+ prepared=0;
+ for(unsigned i=0;i<3;++i) [context render];
+ printf("mixed_tree_surface_first: prepared=%u expected=3\n",prepared);
+ if(prepared!=3) return 7;
+ unsetenv("DARLING_CA_SKIP_LOCAL_IOSURFACE_RENDER");
+ prepared=0;
+ for(unsigned i=0;i<3;++i) [context render];
+ printf("mixed_tree_shortcut_disabled: prepared=%u expected=3\n",prepared);
+ if(prepared!=3) return 5;
+ setenv("DARLING_CA_SKIP_LOCAL_IOSURFACE_RENDER","1",1);
+ RenderPolicyContext* surfaceOnly=[[RenderPolicyContext alloc] initWithRoot:surfaceLayer];
+ renderCalls=0;
+ [surfaceOnly render];
+ printf("surface_only_shortcut_enabled: renderCalls=%u expected=0\n",renderCalls);
+ if(renderCalls!=0) return 6;
+ printf("PASS: mixed-tree Metal preparation survives IOSurface shortcut\n");
+ return 0;
+}