SetInteractorStyle fails to do anything

[spookylukey]

Describe the bug

Trying to use SetInteractorStyle always seems to fail, I always get vtkInteractorStyleTrackballCamera. Specifically I can't use a custom subclass.

To Reproduce

The script below was taken from https://examples.vtk.org/site/Python/Picking/HighlightWithSilhouette/ with minimal adaptation for trame.

When running, the custom code in MouseInteractorHighLightActor.onLeftButtonDown never executes, so the functionality does not work and there are no debug prints.

Code

#!/usr/bin/env python

# noinspection PyUnresolvedReferences
import vtkmodules.vtkRenderingOpenGL2
from vtkmodules.vtkCommonColor import vtkNamedColors
from vtkmodules.vtkCommonCore import vtkMinimalStandardRandomSequence
from vtkmodules.vtkFiltersHybrid import vtkPolyDataSilhouette
from vtkmodules.vtkFiltersSources import vtkSphereSource
from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera
from vtkmodules.vtkRenderingCore import (
    vtkActor,
    vtkPolyDataMapper,
    vtkPropPicker,
    vtkRenderWindow,
    vtkRenderWindowInteractor,
    vtkRenderer
)
from trame.app import get_server
from trame.ui.vuetify import SinglePageLayout
from trame.widgets import vtk, vuetify


def get_program_parameters():
    import argparse
    description = 'Highlighting a selected object with a silhouette.'
    epilogue = '''
Click on the object to highlight it.
The selected object is highlighted with a silhouette.
    '''
    parser = argparse.ArgumentParser(description=description, epilog=epilogue,
                                     formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('numberOfSpheres', nargs='?', type=int, default=10,
                        help='The number of spheres, default is 10.')
    args = parser.parse_args()
    return args.numberOfSpheres


class MouseInteractorHighLightActor(vtkInteractorStyleTrackballCamera):

    def __init__(self, silhouette=None, silhouetteActor=None):
        self.AddObserver("LeftButtonPressEvent", self.onLeftButtonDown)
        self.LastPickedActor = None
        self.Silhouette = silhouette
        self.SilhouetteActor = silhouetteActor

    def onLeftButtonDown(self, obj, event):
        print("In onLeftButtonDown")
        clickPos = self.GetInteractor().GetEventPosition()

        #  Pick from this location.
        picker = vtkPropPicker()
        picker.Pick(clickPos[0], clickPos[1], 0, self.GetDefaultRenderer())
        self.LastPickedActor = picker.GetActor()

        # If we picked something before, remove the silhouette actor and
        # generate a new one.
        if self.LastPickedActor:
            print(f"You picked: {self.LastPickedActor}")
            self.GetDefaultRenderer().RemoveActor(self.SilhouetteActor)

            # Highlight the picked actor by generating a silhouette
            self.Silhouette.SetInputData(self.LastPickedActor.GetMapper().GetInput())
            self.GetDefaultRenderer().AddActor(self.SilhouetteActor)
        else:
            print("Nothing picked")

        #  Forward events
        self.OnLeftButtonDown()
        return

    def SetSilhouette(self, silhouette):
        self.Silhouette = silhouette

    def SetSilhouetteActor(self, silhouetteActor):
        self.SilhouetteActor = silhouetteActor


def main():
    numberOfSpheres = get_program_parameters()
    colors = vtkNamedColors()

    # A renderer and render window
    renderer = vtkRenderer()
    renderer.SetBackground(colors.GetColor3d('SteelBlue'))

    renderWindow = vtkRenderWindow()
    renderWindow.SetSize(640, 480)
    renderWindow.AddRenderer(renderer)

    # An interactor
    interactor = vtkRenderWindowInteractor()
    interactor.SetRenderWindow(renderWindow)

    randomSequence = vtkMinimalStandardRandomSequence()
    # randomSequence.SetSeed(1043618065)
    # randomSequence.SetSeed(5170)
    randomSequence.SetSeed(8775070)
    # Add spheres to play with
    for i in range(numberOfSpheres):
        source = vtkSphereSource()

        # random position and radius
        x = randomSequence.GetRangeValue(-5.0, 5.0)
        randomSequence.Next()
        y = randomSequence.GetRangeValue(-5.0, 5.0)
        randomSequence.Next()
        z = randomSequence.GetRangeValue(-5.0, 5.0)
        randomSequence.Next()
        radius = randomSequence.GetRangeValue(0.5, 1.0)
        randomSequence.Next()

        source.SetRadius(radius)
        source.SetCenter(x, y, z)
        source.SetPhiResolution(11)
        source.SetThetaResolution(21)

        mapper = vtkPolyDataMapper()
        mapper.SetInputConnection(source.GetOutputPort())
        actor = vtkActor()
        actor.SetMapper(mapper)

        r = randomSequence.GetRangeValue(0.4, 1.0)
        randomSequence.Next()
        g = randomSequence.GetRangeValue(0.4, 1.0)
        randomSequence.Next()
        b = randomSequence.GetRangeValue(0.4, 1.0)
        randomSequence.Next()

        actor.GetProperty().SetDiffuseColor(r, g, b)
        actor.GetProperty().SetDiffuse(0.8)
        actor.GetProperty().SetSpecular(0.5)
        actor.GetProperty().SetSpecularColor(colors.GetColor3d('White'))
        actor.GetProperty().SetSpecularPower(30.0)

        renderer.AddActor(actor)
    renderer.ResetCamera()

    # Create the silhouette pipeline, the input data will be set in the
    # interactor
    silhouette = vtkPolyDataSilhouette()
    silhouette.SetCamera(renderer.GetActiveCamera())

    # Create mapper and actor for silhouette
    silhouetteMapper = vtkPolyDataMapper()
    silhouetteMapper.SetInputConnection(silhouette.GetOutputPort())

    silhouetteActor = vtkActor()
    silhouetteActor.SetMapper(silhouetteMapper)
    silhouetteActor.GetProperty().SetColor(colors.GetColor3d("Tomato"))
    silhouetteActor.GetProperty().SetLineWidth(5)

    # Set the custom type to use for interaction.
    style = MouseInteractorHighLightActor(silhouette, silhouetteActor)
    style.SetDefaultRenderer(renderer)

    interactor.SetInteractorStyle(style)

    server = get_server(client_type="vue2")
    ctrl = server.controller

    with SinglePageLayout(server) as layout:
        layout.title.set_text("Hello trame")

        with layout.content:
            with vuetify.VContainer(
                fluid=True,
                classes="pa-0 fill-height",
            ):
                view = vtk.VtkLocalView(renderWindow)

    server.start()
    


if __name__ == "__main__":
    main()

Expected behavior

The MouseInteractorHighLightActor sublcass should work like it does in the VTK example linked.

Platform:

I've checked only the following, but I imagine the bug is on all platforms.

Device:

• Desktop
• Mobile

OS:

• Windows
• MacOS
• Linux
• Android
• iOS

Browsers Affected:

• Chrome
• Firefox
• Microsoft Edge
• Safari
• Opera
• Brave
• IE 11

[jourdain]

VtkLocalView has its own set of interactors that don't match the server. You can see how to configure it here. Also even the VtkRemoteView don't match all the clients events. Some efforts have been made in trame-rca (remote-controlled-area) to properly propagate to the server those events so custom interactors could be used.

[resammc]

Hi @jourdain ,

is there any update on this issue? I can see that it has a WASM tag, so I tried a simple example with that repo, but it doesn't show the expected behavior. Switching to VtkRemoteView kind of works, but the view becomes very unresponsive and laggy.

from trame.app import get_server
from trame.ui.vuetify import SinglePageLayout
from trame_vtklocal.widgets import vtklocal
from vtkmodules.vtkFiltersSources import vtkLineSource
from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera
from vtkmodules.vtkRenderingCore import (
    vtkActor,
    vtkPolyDataMapper,
    vtkRenderer,
    vtkRenderWindow,
    vtkRenderWindowInteractor,
)
from trame.widgets import vtk as vtk_widgets

# Required for vtk factory
import vtkmodules.vtkRenderingOpenGL2  # noqa
from vtkmodules.vtkInteractionStyle import vtkInteractorStyleSwitch  # noqa

# VTK line
line_source = vtkLineSource()
line_source.SetPoint1(0, 0, 0)
line_source.SetPoint2(1, 0, 0)
line_source.Update()

# mappers and actors
line_mapper = vtkPolyDataMapper()
line_mapper.SetInputConnection(line_source.GetOutputPort())
line_actor = vtkActor()
line_actor.SetMapper(line_mapper)

# Create a renderer, render window, and interactor
renderer = vtkRenderer()
render_window = vtkRenderWindow()
render_window.AddRenderer(renderer)
interactor = vtkRenderWindowInteractor()
interactor.SetRenderWindow(render_window)

# Add actors to the renderer
renderer.AddActor(line_actor)

# Custom interactor style class
class MouseInteractorStyle(vtkInteractorStyleTrackballCamera):
    def __init__(self):
        self.AddObserver("LeftButtonPressEvent", self.left_button_press_event)
        self.AddObserver("MouseMoveEvent", self.mouse_move_event)

    def left_button_press_event(self, obj, event):
        print("left_button_press_event")

    def mouse_move_event(self, obj, event):
        print("mouse_move_event")

# Set the custom interactor style
style = MouseInteractorStyle()
style.SetDefaultRenderer(renderer)
interactor.SetInteractorStyle(style)

# Create a Trame server
server = get_server()
server.client_type="vue2"

# Define the layout
with SinglePageLayout(server) as layout:
    layout.root.style = "width: 100vw; height: 100vh; margin: 0; padding: 0;"
    layout.title.set_text("VTK Line")
    with layout.content:
        vtklocal.LocalView(render_window)
        # vtk_widgets.VtkRemoteView(render_window)
        

# Start the server
if __name__ == "__main__":
    server.start()

[jourdain]

The issue with your code is that you have server side logic (MouseInteractorStyle) which won't work on the WASM side.
Basically, you should be able to get the rendering and interaction on the client side with WASM, but if you want the server to be aware of some interaction, you will need to trigger those events when they happen on the client side. New API and infrastructure will be needed within LocalView.

I guess the other question would be why is the remote rendering very unresponsive and laggy.

[resammc]

I see. Thank you for clarifying. And yes, it’s surprisingly slow in the remote view.

[jourdain]

Are you using OSMesa and therefore no GPU on the server side?

[resammc]

Sorry for the delayed response. I'm running the code on a laptop with an RTX A1000. I created a new environment and reinstalled everything from scratch, but I see the same behavior.
Does it mean that the above code works just fine on your system? I digged a bit further and it looks like if I add render_window.OffScreenRenderingOn() to the code, it works as expected.
Thank you for looking into this
(if it's not the right place to discuss this, I can also open a discussion)

[jourdain]

Remote rendering should feel fairly responsive like 25-30 fps.

[Fevola]

Hi @jourdain , may I ask how to listen for click events in LocalView?
LocalView does not support using interactor_events like VtkLocalView, which leaves me unsure of how to use events like LeftButtonPress for picking.
I would greatly appreciate your response.

[jourdain]

You might be able to attache listener on the interactor. You can see examples of listeners inside the widgets examples. You will have to figure out which vtk object you want to listen to and which event type. So far I don't have any example doing that.

[Fevola]

Thank you for your reply!
I've got some ideas, but I'm still not sure about the structure of the listeners.
Referring to the following listener, if I need to listen for the LeftButtonPress event to implement cell picking on a vtk_obj, and I have a self.interactor.

# example
self.state.wasm_listeners = {
    self.widget_id: {
        "InteractionEvent": {
            "plane_widget": {
                "normal": (
                    self.widget_id,
                    "WidgetRepresentation",
                    "Normal",
                ),
            }
        }
    }
}

Should I replace self.widget_id with the self.interactor or with the vtk_obj_id obtained through get_wasm_id? Should InteractionEvent be replaced with LeftButtonPress?
In the example, plane_widget is a state, but I don’t have a corresponding state here, so I'm not sure how to fill in the contents.

# mycode
self.state.wasm_listeners = {
    self.interactor:{
        "LeftButtonPress":{
            "?":{
                "?":{
                    self.vtk_obj_id,
                    "origin",
                    "?"
                }
            }
        }
    }
}

I apologize for bothering you with this question again and thank you in advance.

[jourdain]

To really provide a solution to you, I will need to spend time investigating and solving that specific use case. To really put the time needed, I will need some support.