Basic pdb debugging for actor mesh (#217)

Summary:
Pull Request resolved: #217

This diff fleshes out suo's work in D75023066 to provide a basic pdb debugging experience for python actor meshes. The basic flow is:
- Call `init_debugging(python_actor_mesh)` to get a `debug_client`
- Call `debug_client.enter()` to enter a debugging session
- Input `list` to show the set of currently active breakpoints across your mesh
- Attach to a specific actor by inputting `attach`, then run arbitrary pdb commands for the attached actor
- Input `detach` to exit back into the outer debug loop, where you can attach to another rank
- Cast pdb commands to multiple actors at once, either via a comma separated list like `cast r0,r1,r2 <command>` or to all actors via `cast * <command>`
- Input `quit` to exit the debug session and continue

Reviewed By: suo

Differential Revision: D76086980

Author: Sam Lurye

python/monarch/actor_mesh.py

@@ -13,6 +13,7 @@
 import itertools
 import logging
 import random
+import sys
 import traceback
 
 from dataclasses import dataclass
@@ -34,6 +35,7 @@
     ParamSpec,
     Tuple,
     Type,
+    TYPE_CHECKING,
     TypeVar,
 )
 
@@ -54,6 +56,10 @@
 
 from monarch.common.pickle_flatten import flatten, unflatten
 from monarch.common.shape import MeshTrait, NDSlice
+from monarch.pdb_wrapper import remote_breakpointhook
+
+if TYPE_CHECKING:
+    from monarch.debugger import DebugClient
 
 logger = logging.getLogger(__name__)
 
@@ -610,6 +616,19 @@ def _new_with_shape(self, shape: Shape) -> "ActorMeshRef":
             "actor implementations are not meshes, but we can't convince the typechecker of it..."
         )
 
+    @endpoint
+    async def _set_debug_client(self, client: "DebugClient") -> None:
+        point = MonarchContext.get().point
+        # For some reason, using a lambda instead of functools.partial
+        # confuses the pdb wrapper implementation.
+        sys.breakpointhook = functools.partial(  # pyre-ignore
+            remote_breakpointhook,
+            point.rank,
+            point.shape.coordinates(point.rank),
+            MonarchContext.get().mailbox.actor_id,
+            client,
+        )
+
 
 class ActorMeshRef(MeshTrait):
     def __init__(