Skip to content

Scene Camera Video

It's very easy to stream the scene camera feed with timestamps for real-time monitoring. Simply use the device.receive_scene_video_frame method.

bgr_pixels, frame_datetime = device.receive_scene_video_frame()
Scene camera
SimpleVideoFrame

SimpleVideoFrame

Bases: NamedTuple

A simplified video frame representation.

This class provides a simplified representation of a video frame with BGR pixel data and timestamp information.

Attributes:

bgr_pixels instance-attribute 

bgr_pixels: BGRBuffer

BGR pixel data as a NumPy array.

timestamp_unix_seconds instance-attribute 

timestamp_unix_seconds: float

Timestamp in seconds since Unix epoch.

Check the whole example code here
stream_scene_camera_video.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import cv2
import numpy as np

# Workaround for https://github.com/opencv/opencv/issues/21952
cv2.imshow("cv/av bug", np.zeros(1))
cv2.destroyAllWindows()

from pupil_labs.realtime_api.simple import discover_one_device  # noqa: E402


def main():
    # Look for devices. Returns as soon as it has found the first device.
    print("Looking for the next best device...")
    device = discover_one_device(max_search_duration_seconds=10)
    if device is None:
        print("No device found.")
        raise SystemExit(-1)

    print(f"Connecting to {device}...")

    try:
        while True:
            bgr_pixels, frame_datetime = device.receive_scene_video_frame()
            draw_time(bgr_pixels, frame_datetime)
            cv2.imshow("Scene Camera - Press ESC to quit", bgr_pixels)
            if cv2.waitKey(1) & 0xFF == 27:
                break
    except KeyboardInterrupt:
        pass
    finally:
        print("Stopping...")
        device.close()  # explicitly stop auto-update


def draw_time(frame, time):
    frame_txt_font_name = cv2.FONT_HERSHEY_SIMPLEX
    frame_txt_font_scale = 1.0
    frame_txt_thickness = 1

    # first line: frame index
    frame_txt = str(time)

    cv2.putText(
        frame,
        frame_txt,
        (20, 50),
        frame_txt_font_name,
        frame_txt_font_scale,
        (255, 255, 255),
        thickness=frame_txt_thickness,
        lineType=cv2.LINE_8,
    )


if __name__ == "__main__":
    main()

Scene Camera Video with Overlayed Gaze

For additional context about where the wearer was gazing, it's useful to overlay gaze measurements onto the scene camera video stream. Since the scene camera and gaze signal can have different sampling rates, we need to be sure they are matched. For that, you can use (device.receive_matched_scene_video_frame_and_gaze). This receives a pair of scene camera video and gaze data already matched.

frame, gaze = device.receive_matched_scene_video_frame_and_gaze()
Scene camera with gaze overlay
MatchedItem

MatchedItem

Bases: NamedTuple

A matched pair of scene video frame and gaze data.

This class represents a scene video frame and gaze data point that occurred at approximately the same time.

Note

The name MatchedItem is maintained for backward compatibility. It represents a matched pair of scene video frame and gaze data.

Attributes:

frame instance-attribute 

frame: SimpleVideoFrame

Scene video frame.

gaze instance-attribute 

gaze: GazeDataType

Corresponding gaze data.

Check the whole example code here
stream_video_with_overlayed_gaze.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
import cv2
import numpy as np

# Workaround for https://github.com/opencv/opencv/issues/21952
cv2.imshow("cv/av bug", np.zeros(1))
cv2.destroyAllWindows()

from pupil_labs.realtime_api.simple import discover_one_device  # noqa: E402
from pupil_labs.realtime_api.streaming.gaze import (  # noqa: E402
    BinoAndDualMonoGazeData,
    EyestateEyelidDualMonoGazeData,
)

LEGEND = {
    "mono_right": (255, 0, 0),  # right eye monocular gaze
    "mono_left": (0, 255, 0),  # left eye monocular gaze
    "bino": (0, 0, 255),  # binocular gaze
}


def main():
    # Look for devices. Returns as soon as it has found the first device.
    print("Looking for the next best device...")
    device = discover_one_device(max_search_duration_seconds=10)
    if device is None:
        print("No device found.")
        raise SystemExit(-1)

    print(f"Connecting to {device}...")

    try:
        while True:
            frame, gaze = device.receive_matched_scene_video_frame_and_gaze()
            cv2.circle(
                frame.bgr_pixels,
                (int(gaze.x), int(gaze.y)),
                radius=40,
                color=(0, 0, 255),
                thickness=5,
            )
            if isinstance(
                gaze, (BinoAndDualMonoGazeData, EyestateEyelidDualMonoGazeData)
            ):
                cv2.circle(
                    frame.bgr_pixels,
                    (int(gaze.mono_right_x), int(gaze.mono_right_y)),
                    radius=40,
                    color=(255, 0, 0),
                    thickness=5,
                )
                cv2.circle(
                    frame.bgr_pixels,
                    (int(gaze.mono_left_x), int(gaze.mono_left_y)),
                    radius=40,
                    color=(0, 255, 0),
                    thickness=5,
                )
                for eye, color in LEGEND.items():
                    cv2.rectangle(
                        frame.bgr_pixels,
                        (10, 10 + 30 * list(LEGEND.keys()).index(eye)),
                        (30, 30 + 30 * list(LEGEND.keys()).index(eye)),
                        color,
                        -1,
                    )
                    cv2.putText(
                        frame.bgr_pixels,
                        eye,
                        (40, 30 + 30 * list(LEGEND.keys()).index(eye)),
                        cv2.FONT_HERSHEY_SIMPLEX,
                        0.5,
                        color,
                        1,
                        cv2.LINE_AA,
                    )

            cv2.imshow("Scene camera with gaze overlay", frame.bgr_pixels)
            if cv2.waitKey(1) & 0xFF == 27:
                break
    except KeyboardInterrupt:
        pass
    finally:
        print("Stopping...")
        device.close()  # explicitly stop auto-update


if __name__ == "__main__":
    main()

Scene Camera Video with Overlayed Eyes Video and Gaze Circle

Neon

It might also be useful to overlay the eye camera video frames, e.g. if you want to manually inspect the eye data or blinking behaviour. This can be achieved using the device.receive_matched_scene_and_eyes_video_frames_and_gaze method.

matched = device.receive_matched_scene_and_eyes_video_frames_and_gaze()
Scene camera with gaze and eyes overlay
MatchedGazeEyesSceneItem

MatchedGazeEyesSceneItem

Bases: NamedTuple

A matched triplet of scene video frame, eye video frame, and gaze data.

This class represents scene and eye video frames along with gaze data that occurred at approximately the same time.

Attributes:

eyes instance-attribute 

eyes: SimpleVideoFrame

Eye camera video frame.

gaze instance-attribute 

gaze: GazeDataType

Corresponding gaze data.

scene instance-attribute 

scene: SimpleVideoFrame

Scene video frame.

Check the whole example code here
stream_scene_eyes_and_gaze.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import cv2
import numpy as np

# Workaround for https://github.com/opencv/opencv/issues/21952
cv2.imshow("cv/av bug", np.zeros(1))
cv2.destroyAllWindows()

from pupil_labs.realtime_api.simple import discover_one_device  # noqa: E402


def main():
    # Look for devices. Returns as soon as it has found the first device.
    print("Looking for the next best device...")
    device = discover_one_device(max_search_duration_seconds=10)
    if device is None:
        print("No device found.")
        raise SystemExit(-1)

    print(f"Connecting to {device}...")

    try:
        while True:
            matched = device.receive_matched_scene_and_eyes_video_frames_and_gaze()
            if not matched:
                print(
                    "Not able to find a match! Note: Pupil Invisible does not support "
                    "streaming eyes video"
                )
                continue

            cv2.circle(
                matched.scene.bgr_pixels,
                (int(matched.gaze.x), int(matched.gaze.y)),
                radius=80,
                color=(0, 0, 255),
                thickness=15,
            )

            # Render eyes video into the scene video
            height, width, _ = matched.eyes.bgr_pixels.shape
            matched.scene.bgr_pixels[:height, :width, :] = matched.eyes.bgr_pixels

            cv2.imshow(
                "Scene camera with eyes and gaze overlay", matched.scene.bgr_pixels
            )
            if cv2.waitKey(1) & 0xFF == 27:
                break
    except KeyboardInterrupt:
        pass
    finally:
        print("Stopping...")
        device.close()  # explicitly stop auto-update


if __name__ == "__main__":
    main()

Camera Controls

Neon +1.9.0

You can get and set scene camera parameters, such as auto exposure mode, manual exposure time, gain, brightness, contrast, and gamma, using device.get_camera_state and device.set_camera_state.

Note

Initiate the video stream (e.g. by calling device.receive_scene_video_frame()) or open the scene video preview in the Companion app before querying or setting camera controls to ensure the camera is active.

# Retrieve current camera state
camera_state = device.get_camera_state()

# Update camera controls
device.set_camera_state(
    ae_mode="manual",
    man_exp=500,
    gain=64,
    brightness=0,
    contrast=32,
    gamma=300,
    validate_with_state=camera_state,
)
CameraState

CameraState dataclass 

CameraState(ae_mode: ControlStateEnum, man_exp: ControlStateInteger, gain: ControlStateInteger, brightness: ControlStateInteger, contrast: ControlStateInteger, gamma: ControlStateInteger)
ControlStateEnum

ControlStateEnum dataclass 

ControlStateEnum(current_value: str, allowed_values: Sequence[str])
ControlStateInteger

ControlStateInteger dataclass 

ControlStateInteger(current_value: int, value_min: int, value_max: int)
Check the whole example code here
camera_controls.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
import time

import cv2
from rich import print  # noqa: A004

from pupil_labs.realtime_api.simple import discover_one_device


def set_state_1(device, camera_state):
    print("Updating camera to state 1")
    device.set_camera_state(
        ae_mode="manual",
        man_exp=500,
        gain=64,
        brightness=0,
        contrast=32,
        gamma=300,
        validate_with_state=camera_state,
    )


def set_state_2(device, camera_state):
    print("Updating camera to state 2")
    device.set_camera_state(
        ae_mode="manual",
        man_exp=1000,
        gain=75,
        brightness=-20,
        contrast=50,
        gamma=200,
        validate_with_state=camera_state,
    )


def main():
    print("Looking for the next best device...")
    device = discover_one_device(max_search_duration_seconds=10)
    if device is None:
        print("No device found.")
        raise SystemExit(-1)

    # Initiate video stream before querying camera state to ensure camera is active
    # You could instead simply open the scene video preview in the companion app
    device.receive_scene_video_frame()

    print("Retrieving camera state...")
    camera_state = device.get_camera_state()
    print(f"Current camera state: {camera_state}")

    set_state_1(device, camera_state)
    last_state = 1
    last_tick_time = time.time()
    while True:
        bgr_pixels, frame_datetime = device.receive_scene_video_frame()
        draw_time(bgr_pixels, frame_datetime)
        cv2.imshow("Scene Camera - Press ESC to quit", bgr_pixels)

        if time.time() - last_tick_time > 4:
            if last_state == 1:
                set_state_2(device, camera_state)
                last_state = 2
            else:
                set_state_1(device, camera_state)
                last_state = 1
            last_tick_time = time.time()

        if cv2.waitKey(1) & 0xFF == 27:
            break

    device.close()


def draw_time(frame, timestamp):
    frame_txt_font_name = cv2.FONT_HERSHEY_SIMPLEX
    frame_txt_font_scale = 1.0
    frame_txt_thickness = 1

    # first line: frame index
    frame_txt = str(timestamp)

    cv2.putText(
        frame,
        frame_txt,
        (20, 50),
        frame_txt_font_name,
        frame_txt_font_scale,
        (255, 255, 255),
        thickness=frame_txt_thickness,
        lineType=cv2.LINE_8,
    )


if __name__ == "__main__":
    main()