Support adjusting GIL monitoring interval (#7650)

Author: milesgranger

distributed/distributed-schema.yaml

@@ -1141,9 +1141,19 @@ properties:
               host-cpu:
                 type: boolean
                 description: Should we include host-wide CPU usage, with very granular breakdown?
-              gil-contention:
-                type: boolean
-                description: Should we include GIL contention metrics, requires `gilknocker` to be installed.
+              gil:
+                type: object
+                description: |
+                  Should we include GIL contention metrics, requires `gilknocker` to be installed.
+                properties:
+                  enabled:
+                    type: boolean
+                    description: Enable monitoring of GIL contention
+                  interval:
+                    type: string
+                    description: |
+                      GIL polling interval. More frequent polling will reflect a more accurate GIL
+                      contention metric but will be more likely to impact runtime performance.
 
       rmm:
         type: object

distributed/distributed.yaml

@@ -310,7 +310,9 @@ distributed:
       interval: 500ms
       disk: true  # Monitor host-wide disk I/O
       host-cpu: false  # Monitor host-wide CPU usage, with very granular breakdown
-      gil-contention: false  # Monitor GIL contention
+      gil: 
+        enabled: false  # Monitor GIL contention
+        interval: "1ms"  # Frequency to poll GIL
     event-loop: tornado
   rmm:
     pool-size: null

distributed/system_monitor.py

@@ -95,7 +95,7 @@ def __init__(
 
         if monitor_gil_contention is None:
             monitor_gil_contention = dask.config.get(
-                "distributed.admin.system-monitor.gil-contention"
+                "distributed.admin.system-monitor.gil.enabled"
             )
         self.monitor_gil_contention = monitor_gil_contention
         if self.monitor_gil_contention:
@@ -108,7 +108,12 @@ def __init__(
                 self.monitor_gil_contention = False
             else:
                 self.quantities["gil_contention"] = deque(maxlen=maxlen)
-                self._gilknocker = KnockKnock(1_000)  # TODO(miles) param interval?
+                raw_interval = dask.config.get(
+                    "distributed.admin.system-monitor.gil.interval",
+                )
+                interval = dask.utils.parse_timedelta(raw_interval, default="us") * 1e6
+
+                self._gilknocker = KnockKnock(polling_interval_micros=int(interval))
                 self._gilknocker.start()
 
         if not WINDOWS:

distributed/tests/test_system_monitor.py

@@ -109,7 +109,7 @@ def test_gil_contention():
     a = sm.update()
     assert "gil_contention" in a
 
-    with dask.config.set({"distributed.admin.system-monitor.gil-contention": True}):
+    with dask.config.set({"distributed.admin.system-monitor.gil.enabled": True}):
         sm = SystemMonitor()
         a = sm.update()
         assert "gil_contention" in a