Skip to content

Commit

Permalink
Implement dynamic framerate
Browse files Browse the repository at this point in the history
Reduce tracking frequency to head fps
  • Loading branch information
zmerp committed Jun 1, 2023
1 parent bb559d1 commit 069b877
Show file tree
Hide file tree
Showing 12 changed files with 97 additions and 36 deletions.
7 changes: 6 additions & 1 deletion alvr/client_core/src/c_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -398,9 +398,14 @@ pub extern "C" fn alvr_get_tracker_prediction_offset_ns() -> u64 {
}

#[no_mangle]
pub extern "C" fn alvr_report_submit(target_timestamp_ns: u64, vsync_queue_ns: u64) {
pub extern "C" fn alvr_report_submit(
target_timestamp_ns: u64,
frame_interval_ns: u64,
vsync_queue_ns: u64,
) {
crate::report_submit(
Duration::from_nanos(target_timestamp_ns),
Duration::from_nanos(frame_interval_ns),
Duration::from_nanos(vsync_queue_ns),
);
}
Expand Down
4 changes: 2 additions & 2 deletions alvr/client_core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,9 @@ pub fn get_tracker_prediction_offset() -> Duration {
}
}

pub fn report_submit(target_timestamp: Duration, vsync_queue: Duration) {
pub fn report_submit(target_timestamp: Duration, frame_interval: Duration, vsync_queue: Duration) {
if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
stats.report_submit(target_timestamp, vsync_queue);
stats.report_submit(target_timestamp, frame_interval, vsync_queue);

if let Some(sender) = &*STATISTICS_SENDER.lock() {
if let Some(stats) = stats.summary(target_timestamp) {
Expand Down
13 changes: 11 additions & 2 deletions alvr/client_core/src/statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,15 @@ impl StatisticsManager {
}

// vsync_queue is the latency between this call and the vsync. it cannot be measured by ALVR and
// should be reported by the VR runtime
pub fn report_submit(&mut self, target_timestamp: Duration, vsync_queue: Duration) {
// should be reported by the VR runtime.
// predicted_frame_interval is the frame interval returned by the runtime. this is more stable
// any any interval mearued by us.
pub fn report_submit(
&mut self,
target_timestamp: Duration,
predicted_frame_interval: Duration,
vsync_queue: Duration,
) {
let now = Instant::now();

if let Some(frame) = self
Expand All @@ -118,6 +125,8 @@ impl StatisticsManager {
let vsync = now + vsync_queue;
frame.client_stats.frame_interval = vsync.saturating_duration_since(self.prev_vsync);
self.prev_vsync = vsync;

frame.client_stats.predicted_frame_interval = predicted_frame_interval
}
}

Expand Down
3 changes: 2 additions & 1 deletion alvr/client_mock/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ fn tracking_thread(streaming: Arc<RelaxedAtomic>, fps: f32, input: Arc<RwLock<Wi

drop(input_lock);

loop_deadline += Duration::from_secs_f32(1.0 / fps / 3.0);
loop_deadline += Duration::from_secs_f32(1.0 / fps);
thread::sleep(loop_deadline.saturating_duration_since(Instant::now()))
}
}
Expand Down Expand Up @@ -271,6 +271,7 @@ fn client_thread(

alvr_client_core::report_submit(
window_output.current_frame_timestamp,
Duration::from_secs_f32(1.0 / window_output.fps),
Duration::from_millis(input_lock.emulated_vsync_ms),
);

Expand Down
22 changes: 19 additions & 3 deletions alvr/client_openxr/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ pub fn entry_point() {
let mut views_history = VecDeque::new();

let (history_view_sender, history_view_receiver) = mpsc::channel();
let mut frame_interval_sender = None;
let mut reference_space_sender = None::<mpsc::Sender<_>>;

let default_view = xr::View {
Expand Down Expand Up @@ -692,10 +693,13 @@ pub fn entry_point() {

let (sender, reference_space_receiver) = mpsc::channel();
reference_space_sender = Some(sender);
let (sender, frame_interval_receiver) = mpsc::channel();
frame_interval_sender = Some(sender);

streaming_input_thread = Some(thread::spawn(move || {
let mut deadline = Instant::now();
let frame_interval = Duration::from_secs_f32(1.0 / refresh_rate_hint);
let mut frame_interval =
Duration::from_secs_f32(1.0 / refresh_rate_hint);

while is_streaming.value() {
update_streaming_input(&mut context);
Expand All @@ -704,7 +708,11 @@ pub fn entry_point() {
context.reference_space = reference_space;
}

deadline += frame_interval / 3;
if let Ok(interval) = frame_interval_receiver.try_recv() {
frame_interval = interval;
}

deadline += frame_interval;
thread::sleep(deadline.saturating_duration_since(Instant::now()));
}
}));
Expand Down Expand Up @@ -832,6 +840,10 @@ pub fn entry_point() {
let vsync_time =
Duration::from_nanos(frame_state.predicted_display_time.as_nanos() as _);

if let Some(sender) = &frame_interval_sender {
sender.send(frame_interval).ok();
}

xr_frame_stream.begin().unwrap();

if !frame_state.should_render {
Expand Down Expand Up @@ -901,7 +913,11 @@ pub fn entry_point() {

if !hardware_buffer.is_null() {
if let Some(now) = xr_runtime_now(&xr_instance) {
alvr_client_core::report_submit(timestamp, vsync_time.saturating_sub(now));
alvr_client_core::report_submit(
timestamp,
frame_interval,
vsync_time.saturating_sub(now),
);
}
}

Expand Down
1 change: 1 addition & 0 deletions alvr/packets/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ pub struct ClientStatistics {
pub rendering: Duration,
pub vsync_queue: Duration,
pub total_pipeline_latency: Duration,
pub predicted_frame_interval: Duration,
}

#[derive(Serialize, Deserialize, Clone, Debug)]
Expand Down
2 changes: 0 additions & 2 deletions alvr/server/cpp/alvr_server/alvr_server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,6 @@ void DeinitializeStreaming() {
}
}

void SendVSync() { vr::VRServerDriverHost()->VsyncEvent(0.0); }

void RequestIDR() {
if (g_driver_provider.hmd && g_driver_provider.hmd->m_encoder) {
g_driver_provider.hmd->m_encoder->InsertIDR();
Expand Down
1 change: 0 additions & 1 deletion alvr/server/cpp/alvr_server/bindings.h
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,6 @@ extern "C" void (*WaitForVSync)();
extern "C" void *CppEntryPoint(const char *pInterfaceName, int *pReturnCode);
extern "C" void InitializeStreaming();
extern "C" void DeinitializeStreaming();
extern "C" void SendVSync();
extern "C" void RequestIDR();
extern "C" void SetTracking(unsigned long long targetTimestampNs,
float controllerPoseTimeOffsetS,
Expand Down
25 changes: 17 additions & 8 deletions alvr/server/src/connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use crate::{
bitrate::BitrateManager,
buttons::BUTTON_PATH_FROM_ID,
face_tracking::FaceTrackingSink,
haptics,
haptics, openvr_props,
sockets::WelcomeSocket,
statistics::StatisticsManager,
tracking::{self, TrackingManager},
Expand All @@ -24,7 +24,10 @@ use alvr_packets::{
ButtonValue, ClientConnectionResult, ClientControlPacket, ClientListAction, ClientStatistics,
ServerControlPacket, StreamConfigPacket, Tracking, AUDIO, HAPTICS, STATISTICS, TRACKING, VIDEO,
};
use alvr_session::{CodecType, ControllersEmulationMode, FrameSize, OpenvrConfig};
use alvr_session::{
CodecType, ControllersEmulationMode, FrameSize, OpenvrConfig, OpenvrPropValue,
OpenvrPropertyKey,
};
use alvr_sockets::{
spawn_cancelable, ControlSocketReceiver, ControlSocketSender, PeerType, ProtoControlSocket,
StreamSocketBuilder, KEEPALIVE_INTERVAL,
Expand Down Expand Up @@ -637,8 +640,8 @@ async fn connection_pipeline(
crate::SetOpenvrProperty(
*alvr_common::HEAD_ID,
crate::openvr_props::to_ffi_openvr_prop(
alvr_session::OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
alvr_session::OpenvrPropValue::String(id),
OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
OpenvrPropValue::String(id),
),
)
}
Expand All @@ -661,8 +664,8 @@ async fn connection_pipeline(
crate::SetOpenvrProperty(
*alvr_common::HEAD_ID,
crate::openvr_props::to_ffi_openvr_prop(
alvr_session::OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
alvr_session::OpenvrPropValue::String(id),
OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
OpenvrPropValue::String(id),
),
)
}
Expand All @@ -686,8 +689,8 @@ async fn connection_pipeline(
crate::SetOpenvrProperty(
*alvr_common::HEAD_ID,
crate::openvr_props::to_ffi_openvr_prop(
alvr_session::OpenvrPropertyKey::AudioDefaultRecordingDeviceId,
alvr_session::OpenvrPropValue::String(id),
OpenvrPropertyKey::AudioDefaultRecordingDeviceId,
OpenvrPropValue::String(id),
),
)
}
Expand Down Expand Up @@ -925,6 +928,12 @@ async fn connection_pipeline(
network_latency,
decoder_latency,
);

openvr_props::set_prop(
*HEAD_ID,
OpenvrPropertyKey::DisplayFrequency,
OpenvrPropValue::Float(1.0 / stats.frame_interval_average().as_secs_f32()),
);
}
}
}
Expand Down
9 changes: 2 additions & 7 deletions alvr/server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -468,13 +468,8 @@ pub unsafe extern "C" fn HmdDriverFactory(
}

extern "C" fn wait_for_vsync() {
let wait_duration = STATISTICS_MANAGER
.lock()
.as_mut()
.map(|stats| stats.duration_until_next_vsync());

if let Some(duration) = wait_duration {
thread::sleep(duration);
if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
thread::sleep(stats.duration_until_next_vsync());
}
}

Expand Down
12 changes: 8 additions & 4 deletions alvr/server/src/openvr_props.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// todo: fill out more properties for headset and controllers
// todo: add more emulation modes

use crate::{FfiOpenvrProperty, FfiOpenvrPropertyValue, SERVER_DATA_MANAGER};
use crate::{openvr_props, FfiOpenvrProperty, FfiOpenvrPropertyValue, SERVER_DATA_MANAGER};
use alvr_common::{prelude::*, settings_schema::Switch, HEAD_ID, LEFT_HAND_ID, RIGHT_HAND_ID};
use alvr_session::{
ControllersEmulationMode, HeadsetEmulationMode, OpenvrPropValue,
Expand All @@ -13,6 +13,12 @@ use std::{
ptr,
};

pub fn set_prop(device_id: u64, key: OpenvrPropertyKey, value: OpenvrPropValue) {
unsafe {
crate::SetOpenvrProperty(device_id, to_ffi_openvr_prop(key, value));
}
}

pub fn to_ffi_openvr_prop(key: OpenvrPropertyKey, value: OpenvrPropValue) -> FfiOpenvrProperty {
let type_ = match value {
OpenvrPropValue::Bool(_) => crate::FfiOpenvrPropertyType_Bool,
Expand Down Expand Up @@ -113,9 +119,7 @@ pub extern "C" fn set_device_openvr_props(device_id: u64) {
if device_id == *HEAD_ID {
fn set_prop(key: OpenvrPropertyKey, value: OpenvrPropValue) {
info!("Setting head OpenVR prop: {key:?} => {value:?}");
unsafe {
crate::SetOpenvrProperty(*HEAD_ID, to_ffi_openvr_prop(key, value));
}
openvr_props::set_prop(*HEAD_ID, key, value);
}
fn set_string(key: OpenvrPropertyKey, value: &str) {
set_prop(key, OpenvrPropValue::String(value.into()));
Expand Down
34 changes: 29 additions & 5 deletions alvr/server/src/statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const FULL_REPORT_INTERVAL: Duration = Duration::from_millis(500);
pub struct HistoryFrame {
target_timestamp: Duration,
tracking_received: Instant,
// tracking_submitted: Instant,
frame_present: Instant,
frame_composed: Instant,
frame_encoded: Instant,
Expand All @@ -23,6 +24,7 @@ impl Default for HistoryFrame {
Self {
target_timestamp: Duration::ZERO,
tracking_received: now,
// tracking_submitted: now,
frame_present: now,
frame_composed: now,
frame_encoded: now,
Expand All @@ -46,8 +48,8 @@ pub struct StatisticsManager {
battery_gauges: HashMap<u64, f32>,
steamvr_pipeline_latency: Duration,
total_pipeline_latency_average: SlidingWindowAverage<Duration>,
predicted_frame_interval_average: SlidingWindowAverage<Duration>,
last_vsync_time: Instant,
frame_interval: Duration,
}

impl StatisticsManager {
Expand Down Expand Up @@ -78,7 +80,10 @@ impl StatisticsManager {
max_history_size,
),
last_vsync_time: Instant::now(),
frame_interval: nominal_server_frame_interval,
predicted_frame_interval_average: SlidingWindowAverage::new(
nominal_server_frame_interval,
max_history_size,
),
}
}

Expand All @@ -100,6 +105,16 @@ impl StatisticsManager {
}
}

// pub fn report_tracking_submitted(&mut self, target_timestamp: Duration) {
// if let Some(frame) = self
// .history_buffer
// .iter_mut()
// .find(|frame| frame.target_timestamp == target_timestamp)
// {
// frame.tracking_submitted = Instant::now();
// }
// }

pub fn report_frame_present(&mut self, target_timestamp: Duration, offset: Duration) {
if let Some(frame) = self
.history_buffer
Expand Down Expand Up @@ -155,6 +170,9 @@ impl StatisticsManager {
// Called every frame. Some statistics are reported once every frame
// Returns network latency
pub fn report_statistics(&mut self, client_stats: ClientStatistics) -> Duration {
self.predicted_frame_interval_average
.submit_sample(client_stats.predicted_frame_interval);

if let Some(frame) = self
.history_buffer
.iter_mut()
Expand Down Expand Up @@ -275,15 +293,21 @@ impl StatisticsManager {
.saturating_sub(self.total_pipeline_latency_average.get_average())
}

pub fn frame_interval_average(&self) -> Duration {
self.predicted_frame_interval_average.get_average()
}

// NB: this call is non-blocking, waiting should be done externally
pub fn duration_until_next_vsync(&mut self) -> Duration {
let now = Instant::now();

let frame_interval = self.predicted_frame_interval_average.get_average();

// update the last vsync if it's too old
while self.last_vsync_time + self.frame_interval < now {
self.last_vsync_time += self.frame_interval;
while self.last_vsync_time + frame_interval < now {
self.last_vsync_time += frame_interval;
}

(self.last_vsync_time + self.frame_interval).saturating_duration_since(now)
(self.last_vsync_time + frame_interval).saturating_duration_since(now)
}
}

0 comments on commit 069b877

Please sign in to comment.