6 Commits
33 changed files with 1198 additions and 105 deletions

No files matched your search

Generated
+1 -1
View File
@@ -422,7 +422,7 @@ dependencies = [
[[package]]
name = "framemate-agent"
version = "0.2.0"
version = "0.3.0"
dependencies = [
"anyhow",
"axum",
+1 -1
View File
@@ -5,7 +5,7 @@ resolver = "3"
members = ["crates/agent"]
[workspace.package]
version = "0.2.0"
version = "0.3.0"
edition = "2024"
license = "GPL-3.0-or-later"
+6 -1
View File
@@ -42,6 +42,10 @@ the load low.
- **Downloads** – the Frame's download queue with progress, speed and time left, plus recently
finished updates. Downloads of your other PCs (Steam Remote Downloads) are left out.
- **Mirroring** – see above.
- **Phone notifications in VR** – your phone's notifications show up in the headset as normal
Steam toasts. It's opt-in: off by default, you turn it on with the bell in the app's top bar.
Apps you don't want in there can be switched off in the settings. Turn it off again just as
quickly, e.g. when someone else is wearing the headset. Nothing gets stored, it only goes from your phone to your Frame.
- **System** – performance profile, CPU/GPU settings, network throughput, SteamOS and Steam
client versions, battery temperature and health.
- **Encrypted** – the phone talks to the Frame over TLS, pinned to the Frame's own key from the
@@ -157,6 +161,7 @@ deno task tauri dev # desktop window for UI work
| `GET /api/state` | full state as JSON |
| `GET /api/ws` | the same, pushed on every change |
| `GET /api/stream/ws` | mirroring: `{codec}` header, fMP4 init segment, one fragment per frame |
| `POST /api/notify` | toast in the headset: `{app, title, body, icon}` → `{"result": "shown"}` or `"headset_asleep"` |
| `GET /healthz` | liveness, the only route without auth |
Reachable off the Frame on `https://<frame>:7381`. The agent signs its own certificate and the
@@ -196,7 +201,7 @@ FrameMate name, logo and application identifiers (forks must use their own).
- [ ] Feed on the App notifying you of newly frame verified games (in your library)
- [ ] Control Downloads (pause, resume, reorder)
- [ ] View and Transfer Screenshots
- [ ] Mobile Notifications in VR
- [x] Mobile Notifications in VR
- [ ] Turn off controller, headset, etc.
## AI usage
+24
View File
@@ -78,6 +78,30 @@ env XDG_RUNTIME_DIR=/run/user/$(id -u) DBUS_SESSION_BUS_ADDRESS=unix:path=/run/u
Installing over SSH doesn't have this problem.
## Notifications
### "Controlled by restricted setting"
Android doesn't let sideloaded apps (anything not installed from a store) get notification
access right away, so the switch for FrameMate is greyed out. You have to allow it once:
1. Try to turn on notification access for FrameMate once, so you get the "restricted setting"
message. (The option below only shows up after that.)
2. Open the app info: Settings > Apps > FrameMate, or the **Open app info** button in
FrameMate's settings.
3. Tap the ⋮ menu in the top right and choose **Allow restricted settings**, confirm with your
PIN or fingerprint.
4. Go back to notification access and turn it on for FrameMate, now it works.
### Notifications don't show up in the headset
- Is the bell in the top bar on (not crossed out)?
- Is the app you're expecting switched off under Settings > Notifications?
- Are you wearing the headset? Steam doesn't show toasts while it's asleep.
- Try **Send a test notification** in the settings. If that works, the connection is fine and
it's about the app or its notifications. If not, the line below the button tells you what went wrong.
- Silent and ongoing notifications (music players, downloads, ...) are skipped on purpose.
## Other issues
You encountered a different issue or think you found a bug, please let me know!
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "framemate-app",
"version": "0.2.0",
"version": "0.3.0",
"description": "",
"type": "module",
"scripts": {
+2 -1
View File
@@ -891,8 +891,9 @@ dependencies = [
[[package]]
name = "framemate-app"
version = "0.2.0"
version = "0.3.0"
dependencies = [
"jni",
"rcgen",
"ring",
"rustls",
+5 -1
View File
@@ -4,7 +4,7 @@
[package]
name = "framemate-app"
description = "FrameMate companion app"
version = "0.2.0"
version = "0.3.0"
license = "GPL-3.0-or-later"
edition = "2024"
@@ -30,6 +30,10 @@ webpki = { package = "rustls-webpki", version = "0.103", default-features = fals
[target.'cfg(any(target_os = "android", target_os = "ios"))'.dependencies]
tauri-plugin-barcode-scanner = "2"
# JNI entry point for the notification listener (forward.rs); already in the tree via Tauri.
[target.'cfg(target_os = "android")'.dependencies]
jni = "0.21"
[dev-dependencies]
rcgen = { version = "0.14.10", default-features = false, features = ["crypto", "ring"] }
tokio = { version = "1", features = ["time"] }
+5
View File
@@ -16,6 +16,11 @@
@android.webkit.JavascriptInterface <methods>;
}
# JNI into the Rust library (src-tauri/src/forward.rs looks the method up by name).
-keep class dev.framemate.app.NotificationBridge {
native <methods>;
}
# ML Kit (QR pairing via tauri-plugin-barcode-scanner) creates its component registrars by
# reflection from manifest meta-data. Its own rule keeps only their names, and R8 full mode
# then drops the constructor and getComponents(): no SharedPrefManager etc. get registered,
@@ -25,6 +25,23 @@
</intent-filter>
</activity>
<!-- Phone notifications as toasts in the headset (FrameNotificationListener.kt). The
user grants it under "Notification access"; the permission keeps other apps from
binding to it. -->
<service
android:name=".FrameNotificationListener"
android:label="@string/app_name"
android:permission="android.permission.BIND_NOTIFICATION_LISTENER_SERVICE"
android:exported="true">
<intent-filter>
<action android:name="android.service.notification.NotificationListenerService" />
</intent-filter>
<!-- Only notifications that alert or are conversations; ongoing/silent ones never reach us. -->
<meta-data
android:name="android.service.notification.default_filter_types"
android:value="conversations|alerting" />
</service>
<provider
android:name="androidx.core.content.FileProvider"
android:authorities="${applicationId}.fileprovider"
@@ -0,0 +1,122 @@
package dev.framemate.app
import android.app.Notification
import android.content.Context
import android.content.pm.ApplicationInfo
import android.graphics.Bitmap
import android.service.notification.NotificationListenerService
import android.os.SystemClock
import android.service.notification.StatusBarNotification
import android.util.Base64
import androidx.core.graphics.drawable.toBitmap
import androidx.core.os.BundleCompat
import org.json.JSONObject
import java.io.ByteArrayOutputStream
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit
private const val ICON_PX = 96
/** Older than this when its turn comes (Frame asleep or away, sends timing out): dropped. */
private const val MAX_AGE_MS = 30_000L
/** Waiting sends; beyond that the oldest is dropped. */
private const val MAX_QUEUED = 10
/** Notification flags that mean "status, not news": never shown in the headset. */
private const val SKIP_FLAGS =
Notification.FLAG_ONGOING_EVENT or Notification.FLAG_FOREGROUND_SERVICE or Notification.FLAG_GROUP_SUMMARY
/**
* Forwards notifications of the apps the user picked to the Frame as toasts.
*
* Granted by the user under "Notification access" (sideloaded apps first need "Allow restricted
* settings" in the app info menu since Android 13). Content goes to the Frame only, over the
* pinned TLS connection, and is never logged or stored.
*/
class FrameNotificationListener : NotificationListenerService() {
/**
* One at a time and in order, off the main thread: sending blocks on the network (up to ~15 s
* per notification while the Frame is unreachable), so the queue is short and drops the oldest.
*/
private val sender = ThreadPoolExecutor(
1, 1, 0, TimeUnit.MILLISECONDS, LinkedBlockingQueue(MAX_QUEUED), ThreadPoolExecutor.DiscardOldestPolicy(),
)
/**
* Last content per notification key: apps re-post the same notification for progress etc. A new
* message with the same text (e.g. a second "ok") has a new `when`, so it still goes through.
*/
private val lastText = LinkedHashMap<String, Int>()
private val icons = HashMap<String, String?>()
override fun onNotificationPosted(sbn: StatusBarNotification) {
if (sbn.packageName == packageName) return
val notification = sbn.notification ?: return
if (notification.flags and SKIP_FLAGS != 0 || !sbn.isClearable) return
val prefs = NotificationPrefs(this)
val app = appInfo(sbn)
val label = app?.loadLabel(packageManager)?.toString() ?: sbn.packageName
prefs.rememberApp(sbn.packageName, label)
if (!prefs.enabled || !prefs.isAppEnabled(sbn.packageName)) return
val extras = notification.extras
val title = extras.getCharSequence(Notification.EXTRA_TITLE)?.toString().orEmpty()
val body = (extras.getCharSequence(Notification.EXTRA_BIG_TEXT) ?: extras.getCharSequence(Notification.EXTRA_TEXT))
?.toString().orEmpty()
if (title.isBlank() && body.isBlank()) return
// ONLY_ALERT_ONCE: the app itself says re-posts aren't news, even if it bumps `when`.
val time = if (notification.flags and Notification.FLAG_ONLY_ALERT_ONCE != 0) 0L else notification.`when`
val fingerprint = (title + "\u0000" + body + "\u0000" + time).hashCode()
synchronized(lastText) {
if (lastText[sbn.key] == fingerprint) return
lastText[sbn.key] = fingerprint
while (lastText.size > 200) lastText.remove(lastText.keys.first())
}
val json = JSONObject()
.put("app", label)
.put("title", title)
.put("body", body)
.put("icon", icon(sbn.packageName, app) ?: JSONObject.NULL)
.toString()
val posted = SystemClock.elapsedRealtime()
sender.execute {
// A toast for something that happened minutes ago would only confuse.
if (SystemClock.elapsedRealtime() - posted > MAX_AGE_MS) return@execute
prefs.recordResult(NotificationBridge.send(dataDir.absolutePath, json))
}
}
override fun onDestroy() {
sender.shutdown()
super.onDestroy()
}
/**
* The posting app, from the notification itself (`android.appInfo`, what the system uses for
* the notification header). `getApplicationInfo(pkg)` fails for most apps since Android 11's
* package visibility rules, which silently dropped their notifications.
*/
private fun appInfo(sbn: StatusBarNotification): ApplicationInfo? =
BundleCompat.getParcelable(sbn.notification.extras, "android.appInfo", ApplicationInfo::class.java)
?: runCatching { packageManager.getApplicationInfo(sbn.packageName, 0) }.getOrNull()
/** The app's launcher icon as a small PNG data URI; the agent accepts up to 64 KB. */
private fun icon(pkg: String, app: ApplicationInfo?): String? = icons.getOrPut(pkg) {
runCatching {
val bitmap = app!!.loadIcon(packageManager).toBitmap(ICON_PX, ICON_PX, Bitmap.Config.ARGB_8888)
val png = ByteArrayOutputStream().also { bitmap.compress(Bitmap.CompressFormat.PNG, 100, it) }.toByteArray()
"data:image/png;base64," + Base64.encodeToString(png, Base64.NO_WRAP)
}.getOrNull()
}
companion object {
/** Sends a test toast the same way a real notification would go. Blocking. */
fun sendTest(context: Context): String {
// The bell is the master switch: off means nothing reaches the headset, tests included.
if (!NotificationPrefs(context).enabled) return "disabled".also { NotificationPrefs(context).recordResult(it) }
val json = JSONObject().put("app", "FrameMate").put("title", "FrameMate")
.put("body", "Notifications from your phone will show up like this.").toString()
return NotificationBridge.send(context.dataDir.absolutePath, json).also { NotificationPrefs(context).recordResult(it) }
}
}
}
@@ -1,5 +1,6 @@
package dev.framemate.app
import android.content.ComponentName
import android.content.Intent
import android.content.pm.ActivityInfo
import android.content.pm.PackageManager
@@ -64,6 +65,38 @@ class MainActivity : TauriActivity() {
@JavascriptInterface
fun localNetworkAllowed() = this@MainActivity.localNetworkAllowed()
/** Whether the user granted "Notification access" to FrameNotificationListener. */
@JavascriptInterface
fun notificationAccessGranted(): Boolean {
val enabled = Settings.Secure.getString(contentResolver, "enabled_notification_listeners").orEmpty()
val ours = ComponentName(this@MainActivity, FrameNotificationListener::class.java)
return enabled.split(':').any { ComponentName.unflattenFromString(it) == ours }
}
@JavascriptInterface
fun openNotificationAccessSettings() = runOnUiThread {
startActivity(Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS))
}
/** `{enabled, apps: [{pkg, label, enabled}], last: {result, time} | null}` */
@JavascriptInterface
fun notificationSettings(): String = NotificationPrefs(this@MainActivity).toJson()
@JavascriptInterface
fun setNotificationsEnabled(enabled: Boolean) {
NotificationPrefs(this@MainActivity).enabled = enabled
}
@JavascriptInterface
fun setNotificationAppEnabled(pkg: String, enabled: Boolean) =
NotificationPrefs(this@MainActivity).setAppEnabled(pkg, enabled)
/** Returns at once; the result lands in `notificationSettings().last`. */
@JavascriptInterface
fun sendTestNotification() {
Thread { FrameNotificationListener.sendTest(this@MainActivity) }.start()
}
/** After two denials Android stops prompting; only the app's settings page is left. */
@JavascriptInterface
fun openAppSettings() = runOnUiThread {
@@ -0,0 +1,16 @@
package dev.framemate.app
/** JNI into the app's Rust library (src-tauri/src/forward.rs). */
object NotificationBridge {
init {
// Already loaded when the app's window runs; the listener service can start without it.
System.loadLibrary("framemate_app_lib")
}
/**
* Sends one toast to the paired Frame; blocks for up to ~15 s, never call it on the main
* thread. Returns `shown`, `headset_asleep`, `rate_limited`, `not_paired` or `error: …`.
*/
@JvmStatic
external fun send(dataDir: String, json: String): String
}
@@ -0,0 +1,56 @@
package dev.framemate.app
import android.content.Context
import org.json.JSONArray
import org.json.JSONObject
/**
* Forwarding settings, shared by the listener service and the WebView bridge. Kept in
* SharedPreferences rather than the WebView's storage: the service runs without the WebView.
* Nothing here stores notification content, only app names and the last send status.
*/
class NotificationPrefs(context: Context) {
private val prefs = context.getSharedPreferences("notifications", Context.MODE_PRIVATE)
var enabled: Boolean
get() = prefs.getBoolean("enabled", false)
set(value) = prefs.edit().putBoolean("enabled", value).apply()
/** Every app is forwarded unless switched off here; the bell (`enabled`) is the opt-in. */
fun isAppEnabled(pkg: String) = pkg !in disabledApps()
fun setAppEnabled(pkg: String, value: Boolean) {
val disabled = disabledApps().toMutableSet()
if (value) disabled.remove(pkg) else disabled.add(pkg)
prefs.edit().putStringSet("disabled_apps", disabled).apply()
}
private fun disabledApps(): Set<String> = prefs.getStringSet("disabled_apps", emptySet()) ?: emptySet()
/** Apps that posted a notification since forwarding was set up, so the user can pick them. */
fun rememberApp(pkg: String, label: String) {
val seen = seenApps()
if (seen.optString(pkg) == label) return
seen.put(pkg, label)
prefs.edit().putString("seen", seen.toString()).apply()
}
private fun seenApps() = runCatching { JSONObject(prefs.getString("seen", "{}")!!) }.getOrElse { JSONObject() }
fun recordResult(result: String) {
prefs.edit().putString("last_result", result).putLong("last_time", System.currentTimeMillis()).apply()
}
/** For the settings page: `{enabled, apps: [{pkg, label, enabled}], last: {result, time}}`. */
fun toJson(): String {
val seen = seenApps()
val apps = JSONArray()
seen.keys().asSequence()
.sortedBy { seen.optString(it).lowercase() }
.forEach { pkg -> apps.put(JSONObject().put("pkg", pkg).put("label", seen.optString(pkg)).put("enabled", isAppEnabled(pkg))) }
val last = prefs.getString("last_result", null)?.let {
JSONObject().put("result", it).put("time", prefs.getLong("last_time", 0))
}
return JSONObject().put("enabled", enabled).put("apps", apps).put("last", last ?: JSONObject.NULL).toString()
}
}
+93
View File
@@ -0,0 +1,93 @@
//! Phone notifications → toast in the headset (`POST /api/notify` on the agent).
//!
//! Called by the Kotlin `NotificationListenerService` through JNI. Android can start that service
//! without the app's window, so Tauri, the WebView and the loopback proxy may not exist: this
//! reads the pairing straight from disk and talks pinned TLS itself, reusing proxy.rs.
use std::path::Path;
use std::sync::{Mutex, OnceLock};
use crate::pairing::{self, Pairing};
use crate::proxy::Target;
/// The last target, kept so the address that worked (mDNS name or IP) is tried first next time.
static TARGET: Mutex<Option<(Pairing, std::sync::Arc<Target>)>> = Mutex::new(None);
fn runtime() -> &'static tokio::runtime::Runtime {
static RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
RUNTIME.get_or_init(|| {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.expect("tokio runtime")
})
}
/// Sends one toast. `config_dir` holds `pairing.json`, returns a short status
pub fn send(config_dir: &Path, json: &str) -> String {
let Some(pairing) = pairing::load(&config_dir.join("pairing.json")) else {
return "not_paired".into();
};
let target = {
let mut cached = TARGET.lock().unwrap_or_else(|e| e.into_inner());
match cached.as_ref() {
Some((p, target)) if *p == pairing => target.clone(),
_ => match Target::new(pairing.clone()) {
Ok(target) => {
let target = std::sync::Arc::new(target);
*cached = Some((pairing, target.clone()));
target
}
Err(e) => return format!("error: {e}"),
},
}
};
match runtime().block_on(target.post_json("/api/notify", json)) {
Ok((200, body)) => serde_json::from_str::<serde_json::Value>(&body)
.ok()
.and_then(|v| v["result"].as_str().map(str::to_owned))
.unwrap_or_else(|| "error: unexpected answer from the Frame".into()),
Ok((401, _)) => "error: the Frame rejected the token, pair again".into(),
Ok((429, _)) => "rate_limited".into(),
// Agents before v0.3.0 have no /api/notify.
Ok((404, _)) => "error: the FrameMate agent on the Frame is too old for notifications, update it".into(),
Ok((status, body)) => format!("error: {status} {body}"),
Err(e) => format!("error: {e}"),
}
}
#[cfg(target_os = "android")]
mod jni_glue {
use jni::JNIEnv;
use jni::objects::{JClass, JString};
use jni::sys::jstring;
/// `dev.framemate.app.NotificationBridge.send(dataDir, json)`; blocking
#[unsafe(no_mangle)]
pub extern "system" fn Java_dev_framemate_app_NotificationBridge_send<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
data_dir: JString<'local>,
json: JString<'local>,
) -> jstring {
let result = match (env.get_string(&data_dir), env.get_string(&json)) {
(Ok(dir), Ok(json)) => {
let (dir, json): (String, String) = (dir.into(), json.into());
super::send(std::path::Path::new(&dir), &json)
}
_ => "error: bad arguments".into(),
};
env.new_string(result).map(|s| s.into_raw()).unwrap_or(std::ptr::null_mut())
}
}
#[cfg(test)]
mod tests {
#[test]
fn reports_a_missing_pairing() {
let dir = std::env::temp_dir().join("framemate-forward-test-empty");
let _ = std::fs::create_dir_all(&dir);
assert_eq!(super::send(&dir, "{}"), "not_paired");
}
}
+1
View File
@@ -1,3 +1,4 @@
pub mod forward;
pub mod pairing;
pub mod proxy;
+45
View File
@@ -16,6 +16,9 @@ use crate::pairing::Pairing;
const HEADER_LIMIT: usize = 8192;
const CONNECT_TIMEOUT: Duration = Duration::from_secs(3);
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2);
/// Whole `post_json` exchange: connect + steam
const POST_TIMEOUT: Duration = Duration::from_secs(15);
const RESPONSE_LIMIT: u64 = 16 * 1024;
const ACCEPT_BACKOFF: Duration = Duration::from_millis(100);
const HEAD_TIMEOUT: Duration = Duration::from_secs(10);
/// Crockford base32
@@ -82,6 +85,40 @@ impl Target {
pub fn token(&self) -> String {
self.pairing.token.clone()
}
pub async fn post_json(&self, path: &str, json: &str) -> Result<(u16, String), String> {
let exchange = async {
let mut tls = connect(&self.connector, &self.pairing, &self.preferred).await.map_err(|e| match e {
Failure::Rejected(e) => format!("this isn't the Frame the app was paired with, pair again ({e})"),
Failure::Unreachable(e) => format!("can't reach the Frame ({e})"),
})?;
let request = format!(
"POST {path} HTTP/1.1\r\nHost: framemate\r\nAuthorization: Bearer {}\r\n\
Content-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{json}",
self.pairing.token,
json.len()
);
tls.write_all(request.as_bytes()).await.map_err(|e| e.to_string())?;
let mut response = Vec::new();
// `Connection: close`: the agent ends the stream after the response.
tls.take(RESPONSE_LIMIT).read_to_end(&mut response).await.map_err(|e| e.to_string())?;
parse_response(&response)
};
tokio::time::timeout(POST_TIMEOUT, exchange)
.await
.unwrap_or_else(|_| Err(format!("no answer within {}s", POST_TIMEOUT.as_secs())))
}
}
fn parse_response(response: &[u8]) -> Result<(u16, String), String> {
let text = String::from_utf8_lossy(response);
let (head, body) = text.split_once("\r\n\r\n").ok_or("incomplete response")?;
let status = head
.split(' ')
.nth(1)
.and_then(|code| code.parse().ok())
.ok_or("malformed status line")?;
Ok((status, body.trim().to_owned()))
}
pub type Current = Arc<Mutex<Option<Target>>>;
@@ -319,6 +356,14 @@ mod tests {
assert!(base32(&[0x5a; 16]).bytes().all(|b| ALPHABET.contains(&b)));
}
#[test]
fn parses_the_agents_response() {
let raw = b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\n\r\n{\"result\":\"shown\"}";
assert_eq!(parse_response(raw), Ok((200, "{\"result\":\"shown\"}".to_owned())));
assert_eq!(parse_response(b"HTTP/1.1 429 Too Many Requests\r\n\r\nslow down\n"), Ok((429, "slow down".to_owned())));
assert!(parse_response(b"HTTP/1.1 200").is_err());
}
#[test]
fn secrets_differ_per_run() {
let (a, b) = (random_secret().unwrap(), random_secret().unwrap());
+1 -1
View File
@@ -1,7 +1,7 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "FrameMate",
"version": "0.2.0",
"version": "0.3.0",
"identifier": "dev.framemate.app",
"build": {
"beforeDevCommand": "deno task dev",
+15
View File
@@ -48,3 +48,18 @@ async fn reaches_the_real_agent_through_the_proxy() {
assert_eq!(status.take(), None);
}
}
/// The notification listener's path (forward.rs): pairing.json on disk -> pinned POST /api/notify
/// Shows a toast if the headset is awake, else reports `headset_asleep`.
#[test]
#[ignore = "needs a running agent and FRAMEMATE_PAIRING"]
fn forwards_a_notification_to_the_real_agent() {
let payload = std::env::var("FRAMEMATE_PAIRING").expect("set FRAMEMATE_PAIRING");
let pairing = Pairing::parse(payload.trim()).expect("payload must parse");
let dir = std::env::temp_dir().join("framemate-forward-live");
framemate_app_lib::pairing::store(&dir.join("pairing.json"), &pairing).unwrap();
let json = r#"{"app":"FrameMate","title":"FrameMate","body":"Test from the forwarding path"}"#;
let result = framemate_app_lib::forward::send(&dir, json);
println!("forward: {result}");
assert!(result == "shown" || result == "headset_asleep", "{result}");
}
+9
View File
@@ -14,6 +14,15 @@ declare global {
/** False on Android 17+ until "Nearby devices" (local network) is granted. */
localNetworkAllowed?(): boolean;
openAppSettings?(): void;
/** Phone notifications. */
notificationAccessGranted?(): boolean;
openNotificationAccessSettings?(): void;
/** JSON, see NotificationSettings in settings/+page.svelte. */
notificationSettings?(): string;
setNotificationsEnabled?(enabled: boolean): void;
setNotificationAppEnabled?(pkg: string, enabled: boolean): void;
/** Returns at once; the outcome shows up in notificationSettings().last. */
sendTestNotification?(): void;
};
}
}
+45 -7
View File
@@ -1,26 +1,50 @@
<script lang="ts">
import type { Snippet } from "svelte";
let { title, aside, children }: { title: string; aside?: Snippet; children: Snippet } = $props();
let {
title,
aside,
children,
collapsible = false,
open = $bindable(!collapsible),
}: { title: string; aside?: Snippet; children: Snippet; collapsible?: boolean; open?: boolean } = $props();
</script>
<section>
<header>
<h2>{title}</h2>
{@render aside?.()}
</header>
{@render children()}
{#if collapsible}
<button class="header" aria-expanded={open} onclick={() => (open = !open)}>
<h2><span class="chevron" class:open></span>{title}</h2>
{@render aside?.()}
</button>
{:else}
<header class="header">
<h2>{title}</h2>
{@render aside?.()}
</header>
{/if}
{#if open}
{@render children()}
{/if}
</section>
<style>
section {
margin: 20px 0 0;
}
header {
.header {
display: flex;
align-items: baseline;
justify-content: space-between;
width: 100%;
padding: 0 var(--gutter) 8px;
}
button.header {
background: none;
border: none;
color: inherit;
font: inherit;
text-align: left;
cursor: pointer;
}
h2 {
margin: 0;
font-size: 13px;
@@ -29,4 +53,18 @@
text-transform: uppercase;
color: var(--text);
}
/* Small right-pointing chevron, turned down when open. */
.chevron {
display: inline-block;
width: 6px;
height: 6px;
margin: 0 8px 1px 0;
border-right: 2px solid currentColor;
border-bottom: 2px solid currentColor;
transform: rotate(-45deg);
transition: transform 0.15s;
}
.chevron.open {
transform: rotate(45deg);
}
</style>
+2
View File
@@ -12,6 +12,8 @@ glyph), width/height removed, `fill="currentColor"` added.
| File | Original icon | Weight | Upstream URL |
|---|---|---|---|
| back.svg | `arrow_back` | 700 | https://cdn.jsdelivr.net/npm/@material-symbols/svg-700@0.47.6/sharp/arrow_back.svg |
| bell.svg | `notifications` (fill) | 400 | https://cdn.jsdelivr.net/npm/@material-symbols/svg-400@0.47.6/sharp/notifications-fill.svg |
| bell-off.svg | `notifications_off` (fill) | 400 | https://cdn.jsdelivr.net/npm/@material-symbols/svg-400@0.47.6/sharp/notifications_off-fill.svg |
| bell-badge.svg | `notifications_unread` (fill) | 400 | https://cdn.jsdelivr.net/npm/@material-symbols/svg-400@0.47.6/sharp/notifications_unread-fill.svg |
| cast.svg | `cast` | 700 | https://cdn.jsdelivr.net/npm/@material-symbols/svg-700@0.47.6/sharp/cast.svg |
| download.svg | `download` | 700 | https://cdn.jsdelivr.net/npm/@material-symbols/svg-700@0.47.6/sharp/download.svg |
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="36 -924 888 888" fill="currentColor"><path d="M160-200v-60h80v-304q0-33 8-65t25-61l43 126L75-805l42-42 726 727-42 42-122-122H160Zm560-129L310-738q23-23 50.5-38t59.5-22v-82h120v82q81 17 130.5 83.5T720-564v235ZM480-80q-33 0-56.5-23.5T400-160h160q0 33-23.5 56.5T480-80Z"/></svg>

After

Width:  |  Height:  |  Size: 324 B

+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="36 -924 888 888" fill="currentColor"><path d="M160-200v-60h80v-304q0-84 49.5-150.5T420-798v-82h120v82q81 17 130.5 83.5T720-564v304h80v60H160ZM480-80q-33 0-56.5-23.5T400-160h160q0 33-23.5 56.5T480-80Z"/></svg>

After

Width:  |  Height:  |  Size: 257 B

+78
View File
@@ -0,0 +1,78 @@
// Phone notifications as toasts in the headset (Android only). The settings live on the Kotlin
// side (NotificationPrefs.kt) because the listener service runs without the WebView; this
// mirrors them for the top bar's bell and the Settings page.
export interface NotificationSettings {
/** The bell: off by default, and quick to switch off when someone else wears the headset. */
enabled: boolean;
/** Apps that posted a notification so far; all forwarded unless switched off. */
apps: { pkg: string; label: string; enabled: boolean }[];
/** Outcome of the last send: shown, headset_asleep, rate_limited, not_paired, error: … */
last: { result: string; time: number } | null;
}
const bridge = () => window.FrameMateAndroid;
class Notifications {
readonly supported = typeof window !== "undefined" && !!window.FrameMateAndroid?.notificationSettings;
settings = $state<NotificationSettings | null>(null);
access = $state(false);
testing = $state(false);
constructor() {
// Known before the first render: Settings opens its section depending on `access`.
this.refresh();
}
get enabled() {
return this.settings?.enabled ?? false;
}
refresh() {
if (!this.supported) return;
try {
this.settings = JSON.parse(bridge()!.notificationSettings!());
this.access = bridge()!.notificationAccessGranted?.() ?? false;
} catch {}
}
setEnabled(enabled: boolean) {
bridge()?.setNotificationsEnabled?.(enabled);
this.refresh();
}
setApp(pkg: string, enabled: boolean) {
bridge()?.setNotificationAppEnabled?.(pkg, enabled);
this.refresh();
}
openAccessSettings() {
bridge()?.openNotificationAccessSettings?.();
}
async sendTest() {
const before = this.settings?.last?.time ?? 0;
this.testing = true;
bridge()?.sendTestNotification?.();
// The send runs on a Kotlin thread (up to ~15 s); poll for its result.
for (let i = 0; i < 40 && (this.settings?.last?.time ?? 0) <= before; i++) {
await new Promise(resolve => setTimeout(resolve, 500));
this.refresh();
}
this.testing = false;
}
}
export const notifications = new Notifications();
export function describeResult(result: string) {
return (
{
shown: "Shown in the headset.",
headset_asleep: "Delivered, but the headset is asleep, so Steam didn't show it.",
rate_limited: "Too many at once. Try again in a few seconds.",
not_paired: "Pair the app with your Frame first.",
disabled: "Notifications are off. Turn them on with the bell in the top bar.",
}[result] ?? result.replace(/^error: /, "Failed: ")
);
}
+41 -1
View File
@@ -7,6 +7,7 @@
import Battery from "$lib/components/Battery.svelte";
import Icon from "$lib/components/Icon.svelte";
import UpdateNotice from "$lib/components/UpdateNotice.svelte";
import { notifications } from "$lib/notifications.svelte";
import { updates } from "$lib/updates.svelte";
let { children }: { children: Snippet } = $props();
@@ -26,7 +27,23 @@
new Date(agent.receivedAt).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }),
);
// The bell: notification sharing is opt-in and easy to switch off again (someone else wearing
// the headset). Switching it on without notification access leads to Android's access page
// (and to Settings, which explains the "restricted settings" step for sideloaded apps).
function toggleNotifications() {
if (notifications.enabled) {
notifications.setEnabled(false);
return;
}
notifications.setEnabled(true);
if (!notifications.access) {
goto("/settings");
notifications.openAccessSettings();
}
}
onMount(() => {
notifications.refresh();
// Pairing lives on the Rust side now, so the connection details arrive asynchronously.
agent.start().then(() => {
if (!agent.configured) goto("/settings");
@@ -35,7 +52,9 @@
// Coming back from the background: the socket may be dead without knowing it, so always
// fetch a fresh snapshot instead of trusting the old one.
const onVisible = () => {
if (document.visibilityState === "visible") agent.refresh();
if (document.visibilityState !== "visible") return;
agent.refresh();
notifications.refresh(); // e.g. back from Android's notification access page
};
document.addEventListener("visibilitychange", onVisible);
window.addEventListener("online", onVisible);
@@ -61,6 +80,17 @@
<span class="stale-note">{agent.status === "connecting" ? "Updating…" : lastUpdate}</span>
{/if}
<span class="status {agent.status}" title={agent.status}></span>
{#if notifications.supported}
<button
class="bell"
class:on={notifications.enabled}
onclick={toggleNotifications}
aria-pressed={notifications.enabled}
aria-label={notifications.enabled ? "Notifications in the headset: on" : "Notifications in the headset: off"}
>
<Icon name={notifications.enabled ? "bell" : "bell-off"} size={20} />
</button>
{/if}
<a href="/help" class="help" class:active={true} aria-label="Help" >
<Icon name="help" size={20} />
</a>
@@ -127,6 +157,16 @@
.status.unauthorized {
background: var(--red);
}
.bell {
display: flex;
padding: 0;
color: var(--muted);
background: none;
border: none;
}
.bell.on {
color: var(--text);
}
/* Flex: on a text line, line-height pushes the icon off-center. */
.help {
display: flex;
+19
View File
@@ -43,6 +43,25 @@
</p>
</Section>
<Section title="Notifications in VR">
<p>
FrameMate can show your phone's notifications in the headset, as regular Steam toasts. It's off
by default: tap the bell in the top bar to turn it on, the crossed out bell means it's off.
The first time, Android asks you to give FrameMate notification access. After that every app's
notifications get forwarded, you can switch off the ones you don't want in Settings (apps show
up there once they've posted a notification).
</p>
<p>
Your notifications only go to your Frame, encrypted, and are never stored. If someone else is
using the headset, just tap the bell again and nothing gets forwarded.
</p>
<p>
Android greys out the notification access switch for sideloaded apps. Open the app info, tap ⋮
and choose <b>Allow restricted settings</b>, then try again. More in the
<a href="https://github.com/nailuj05/framemate/blob/main/TROUBLESHOOTING.md#notifications" onclick={external}>troubleshooting guide</a>.
</p>
</Section>
<Section title="Issues">
<p>
If you encounter any issues with the app or agent, please submit a GitHub Issue so I can get it fixed. <br>
+94
View File
@@ -3,6 +3,7 @@
import Row from "$lib/components/Row.svelte";
import Section from "$lib/components/Section.svelte";
import { openExternal } from "$lib/external";
import { describeResult, notifications } from "$lib/notifications.svelte";
import { APP_VERSION, updates } from "$lib/updates.svelte";
let code = $state("");
@@ -16,6 +17,13 @@
return typeof message === "string" ? message : JSON.stringify(error);
}
let appsOpen = $state(false);
const appsSummary = $derived.by(() => {
const apps = notifications.settings?.apps ?? [];
const off = apps.filter(app => !app.enabled).length;
return apps.length ? `${apps.length}${off ? ` · ${off} off` : ""}` : "none yet";
});
const statusText = $derived(
{
unconfigured: "Not paired yet",
@@ -96,6 +104,54 @@
</form>
</Section>
{#if notifications.supported && notifications.settings}
<!-- Open while access is missing: the bell sends people here for the "restricted settings" step. -->
<Section title="Notifications" collapsible open={!notifications.access}>
{#snippet aside()}<span class="aside">{notifications.enabled ? "On" : "Off"}</span>{/snippet}
<Row label="Show in the headset" value={notifications.enabled ? "On" : "Off (bell in the top bar)"} />
{#if !notifications.access}
<div class="actions">
<p class="hint">FrameMate needs notification access to read your notifications. They only go to your Frame,
encrypted, and are never stored.</p>
<button class="button" onclick={() => notifications.openAccessSettings()}>Grant notification access</button>
<p class="hint">If Android greys FrameMate out there, open the app info, tap ⋮ and choose
<b>Allow restricted settings</b>, then try again.</p>
<button class="button secondary" onclick={() => window.FrameMateAndroid?.openAppSettings?.()}>Open app info</button>
</div>
{:else}
<button class="row expander" aria-expanded={appsOpen} onclick={() => (appsOpen = !appsOpen)}>
<span><span class="chevron" class:open={appsOpen}></span>Apps</span>
<span class="aside">{appsSummary}</span>
</button>
{#if appsOpen}
{#each notifications.settings.apps as app (app.pkg)}
<label class="row toggle app">
<span>{app.label}</span>
<input
type="checkbox"
role="switch"
checked={app.enabled}
onchange={event => notifications.setApp(app.pkg, event.currentTarget.checked)}
/>
</label>
{/each}
{/if}
<div class="actions">
<button
class="button secondary"
onclick={() => notifications.sendTest()}
disabled={notifications.testing || !notifications.enabled}
>
{notifications.testing ? "Sending…" : "Send a test notification"}
</button>
{#if notifications.settings.last && !notifications.testing}
<p class="hint">Last: {describeResult(notifications.settings.last.result)}</p>
{/if}
</div>
{/if}
</Section>
{/if}
<Section title="Updates">
<Row label="App version" value={APP_VERSION} />
<Row label="Agent version" value={agent.state?.agent.version ?? "not connected"} />
@@ -163,6 +219,44 @@
outline: none;
border-color: var(--accent);
}
.aside {
font-size: 13px;
color: var(--muted);
}
/* A Row that opens the app list below it. */
.expander {
display: flex;
justify-content: space-between;
align-items: center;
gap: 16px;
width: 100%;
min-height: 48px;
padding: 0 var(--gutter);
background: var(--surface);
border: none;
border-bottom: 1px solid var(--divider);
color: var(--text);
font: inherit;
text-align: left;
cursor: pointer;
}
.chevron {
display: inline-block;
width: 6px;
height: 6px;
margin: 0 10px 2px 2px;
border-right: 2px solid currentColor;
border-bottom: 2px solid currentColor;
transform: rotate(-45deg);
transition: transform 0.15s;
}
.chevron.open {
transform: rotate(45deg);
}
/* Indented under the "Apps" row. */
.toggle.app {
padding-left: calc(var(--gutter) + 18px);
}
.hint {
margin: 0;
font-size: 13px;
+73 -19
View File
@@ -1,8 +1,8 @@
//! Steam CEF DevTools client (127.0.0.1:8080): attaches to `SharedJSContext`, injects `shim.js`
//! on every `executionContextCreated` (UI reloads) and receives `{topic, data}` through the
//! `__framemateEmit` binding. Reconnects with backoff when Steam restarts.
//! Steam CEF DevTools client (127.0.0.1:8080): attaches to `SharedJSContext`, injects `shim.js` on every `executionContextCreated` (UI reload)
//! Receives `{topic, data}` through `__framemateEmit`. Reconnects when Steam restarts. Carries `/api/notify` requests to shim's hook (notify.rs).
use std::sync::Arc;
use std::collections::HashMap;
use std::sync::{Arc, LazyLock};
use std::time::{Duration, Instant};
use anyhow::{Context, bail};
@@ -10,19 +10,25 @@ use futures_util::{SinkExt, StreamExt};
use serde_json::{Value, json};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
use tokio::sync::{mpsc, oneshot};
use tokio_tungstenite::tungstenite::Message;
use crate::devices::DeviceMemory;
use crate::hub::{Hub, now_ms};
use crate::notify::{self, Outcome};
const TARGET_TITLE: &str = "SharedJSContext";
const BINDING: &str = "__framemateEmit";
const SHIM: &str = include_str!("shim.js");
/// `shim.js` with the fallback toast icon (the FrameMate logo) filled in.
static SHIM: LazyLock<String> = LazyLock::new(|| {
let logo = include_str!("../../../assets/framemate-black.svg");
include_str!("shim.js").replace("\"__FRAMEMATE_LOGO_SVG__\"", &json!(logo).to_string())
});
const MAX_BACKOFF: Duration = Duration::from_secs(30);
/// The shim polls every few seconds; after this much silence, ping once, then reconnect.
/// after this much silence, ping once, then reconnect
const IDLE_TIMEOUT: Duration = Duration::from_secs(30);
pub async fn run(hub: Arc<Hub>, cdp_url: String) {
pub async fn run(hub: Arc<Hub>, cdp_url: String, mut toasts: mpsc::Receiver<notify::Request>) {
let authority = cdp_url
.trim_start_matches("http://")
.trim_end_matches('/')
@@ -31,7 +37,7 @@ pub async fn run(hub: Arc<Hub>, cdp_url: String) {
let mut devices = DeviceMemory::load();
loop {
let started = Instant::now();
let error = match session(&hub, &authority, &mut devices).await {
let error = match session(&hub, &authority, &mut devices, &mut toasts).await {
Ok(()) => "connection closed".to_owned(),
Err(e) => format!("{e:#}"),
};
@@ -48,7 +54,12 @@ pub async fn run(hub: Arc<Hub>, cdp_url: String) {
}
}
async fn session(hub: &Hub, authority: &str, devices: &mut DeviceMemory) -> anyhow::Result<()> {
async fn session(
hub: &Hub,
authority: &str,
devices: &mut DeviceMemory,
toasts: &mut mpsc::Receiver<notify::Request>,
) -> anyhow::Result<()> {
let ws_url = find_target(authority).await?;
let (ws, _) = tokio_tungstenite::connect_async(&ws_url)
.await
@@ -57,12 +68,15 @@ async fn session(hub: &Hub, authority: &str, devices: &mut DeviceMemory) -> anyh
let mut next_id = 0u64;
let mut call = |method: &str, params: Value| {
next_id += 1;
Message::Text(json!({"id": next_id, "method": method, "params": params}).to_string().into())
let msg = json!({"id": next_id, "method": method, "params": params});
(next_id, Message::Text(msg.to_string().into()))
};
// Toast requests waiting for Steam's answer
let mut pending: HashMap<u64, oneshot::Sender<Result<Outcome, String>>> = HashMap::new();
tx.send(call("Runtime.addBinding", json!({"name": BINDING}))).await?;
tx.send(call("Runtime.addBinding", json!({"name": BINDING})).1).await?;
// Emits executionContextCreated for the existing context, which triggers injection.
tx.send(call("Runtime.enable", json!({}))).await?;
tx.send(call("Runtime.enable", json!({})).1).await?;
tracing::info!("steam CDP: attached to {TARGET_TITLE}");
hub.update(|s| {
s.steam.connected = true;
@@ -70,18 +84,41 @@ async fn session(hub: &Hub, authority: &str, devices: &mut DeviceMemory) -> anyh
});
let mut pinged = false;
// Only traffic from Steam moves this, not toast requests: otherwise a steady stream of
// notifications would keep a hung connection alive.
let mut idle_until = tokio::time::Instant::now() + IDLE_TIMEOUT;
loop {
let msg = match tokio::time::timeout(IDLE_TIMEOUT, rx.next()).await {
let next = tokio::select! {
next = tokio::time::timeout_at(idle_until, rx.next()) => next,
Some(request) = toasts.recv() => {
// Queued while Steam was away; showing it now would be confusingly late.
if Instant::now() >= request.deadline {
continue;
}
let expression = format!(
"(() => {{ const notify = window.__framemate?.notify; return notify ? notify({}) \
: {{ unavailable: \"the toast hook isn't installed (state.steam.topic_errors.toasts)\" }}; }})()",
request.args
);
let (id, msg) = call("Runtime.evaluate", json!({"expression": expression, "returnByValue": true}));
tx.send(msg).await?;
pending.insert(id, request.reply);
continue;
}
};
let msg = match next {
Ok(Some(msg)) => msg,
Ok(None) => break,
Err(_) if pinged => bail!("Steam stopped responding"),
Err(_) => {
pinged = true;
tx.send(call("Runtime.evaluate", json!({"expression": "1"}))).await?;
idle_until = tokio::time::Instant::now() + IDLE_TIMEOUT;
tx.send(call("Runtime.evaluate", json!({"expression": "1"})).1).await?;
continue;
}
};
pinged = false;
idle_until = tokio::time::Instant::now() + IDLE_TIMEOUT;
let text = match msg? {
Message::Text(text) => text,
Message::Close(_) => break,
@@ -94,8 +131,8 @@ async fn session(hub: &Hub, authority: &str, devices: &mut DeviceMemory) -> anyh
let context = &params["context"];
if context["auxData"]["isDefault"].as_bool() == Some(true) {
tracing::info!("steam CDP: injecting shim into context {}", context["id"]);
let params = json!({"expression": SHIM, "contextId": context["id"]});
tx.send(call("Runtime.evaluate", params)).await?;
let params = json!({"expression": SHIM.as_str(), "contextId": context["id"]});
tx.send(call("Runtime.evaluate", params).1).await?;
}
}
Some("Runtime.executionContextsCleared") => {
@@ -108,7 +145,16 @@ async fn session(hub: &Hub, authority: &str, devices: &mut DeviceMemory) -> anyh
}
Some(_) => {}
None => {
if let Some(exception) = msg["result"].get("exceptionDetails") {
if let Some(reply) = msg["id"].as_u64().and_then(|id| pending.remove(&id)) {
// Only the outcome is logged, never the content.
let result = if msg["result"].get("exceptionDetails").is_some() || msg.get("error").is_some() {
Err("the toast hook threw inside Steam".to_owned())
} else {
notify::outcome(&msg["result"]["result"]["value"])
};
tracing::debug!("toast: {result:?}");
let _ = reply.send(result);
} else if let Some(exception) = msg["result"].get("exceptionDetails") {
tracing::warn!("steam CDP: shim threw: {exception}");
} else if let Some(error) = msg.get("error") {
tracing::warn!("steam CDP: call {} failed: {error}", msg["id"]);
@@ -162,8 +208,7 @@ async fn find_target(authority: &str) -> anyhow::Result<String> {
Ok(format!("ws://{authority}{path}"))
}
/// Minimal HTTP/1.1 GET (no HTTP client dependency). CEF ignores HTTP/1.0 requests and
/// `Connection: close`, so the body is read by Content-Length.
/// Minimal HTTP/1.1 GET (no HTTP client dependency). CEF ignores HTTP/1.0 requests
pub(crate) async fn http_get(authority: &str, path: &str) -> anyhow::Result<Vec<u8>> {
tokio::time::timeout(Duration::from_secs(5), async {
let mut stream = TcpStream::connect(authority)
@@ -209,3 +254,12 @@ pub(crate) async fn http_get(authority: &str, path: &str) -> anyhow::Result<Vec<
.await
.context("timeout")?
}
#[cfg(test)]
mod tests {
#[test]
fn shim_gets_the_logo() {
assert!(!super::SHIM.contains("__FRAMEMATE_LOGO_SVG__"));
assert!(super::SHIM.contains("<svg"));
}
}
+4 -2
View File
@@ -6,6 +6,7 @@ mod devices;
mod encoder;
mod fmp4;
mod hub;
mod notify;
mod pair;
mod power;
mod server;
@@ -64,9 +65,10 @@ async fn main() -> anyhow::Result<()> {
let hub = hub::Hub::new(config::hostname());
let stream = stream::LiveStream::new(hub.clone(), config.stream.clone());
tokio::spawn(cdp::run(hub.clone(), config.cdp_url.clone()));
let (notifier, toasts) = notify::Notifier::new();
tokio::spawn(cdp::run(hub.clone(), config.cdp_url.clone(), toasts));
tokio::spawn(power::run(hub.clone(), config.power_supply_dir.clone()));
tokio::spawn(steamos::run(hub.clone()));
server::serve(hub, stream, &config).await
server::serve(hub, stream, std::sync::Arc::new(notifier), &config).await
}
+165
View File
@@ -0,0 +1,165 @@
//! Toasts in the headset: `/api/notify` hands requests to the CDP session (cdp.rs)
//! calls `window.__framemate.notify` inside Steam (shim.js) reports back what Steam did
use std::sync::Mutex;
use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use tokio::sync::{mpsc, oneshot};
/// How long a request waits for Steam
pub const DEADLINE: Duration = Duration::from_secs(4);
/// Token bucket: a burst of this many toasts, then one every `REFILL`.
const BURST: f64 = 5.0;
const REFILL: Duration = Duration::from_secs(2);
const QUEUE: usize = 16;
/// Lengths the shim also enforces
pub const MAX_TITLE: usize = 120;
pub const MAX_BODY: usize = 500;
/// A 96 px app icon as a base64 PNG is ~15 KB
pub const MAX_ICON: usize = 64 * 1024;
#[derive(Deserialize)]
pub struct Toast {
/// Name of the app the notification came from; the title when the notification has none.
#[serde(default)]
pub app: String,
#[serde(default)]
pub title: String,
#[serde(default)]
pub body: String,
/// `data:image/png;base64,...`; anything else is dropped by the shim.
#[serde(default)]
pub icon: Option<String>,
}
#[derive(Serialize, Debug, PartialEq)]
#[serde(rename_all = "snake_case", tag = "result")]
pub enum Outcome {
Shown,
// No notif shown
HeadsetAsleep,
}
pub struct Request {
/// JSON argument for `__framemate.notify`.
pub args: serde_json::Value,
pub deadline: Instant,
pub reply: oneshot::Sender<Result<Outcome, String>>,
}
pub enum Error {
Invalid(&'static str),
RateLimited,
/// Steam (or the toast hook) isn't available
Unavailable(String),
}
pub struct Notifier {
tx: mpsc::Sender<Request>,
bucket: Mutex<(f64, Instant)>,
}
impl Notifier {
/// The receiver goes to the CDP task -> owns the Steam connection
pub fn new() -> (Self, mpsc::Receiver<Request>) {
let (tx, rx) = mpsc::channel(QUEUE);
(Self { tx, bucket: Mutex::new((BURST, Instant::now())) }, rx)
}
pub async fn send(&self, toast: Toast) -> Result<Outcome, Error> {
let args = arguments(toast)?;
if !self.take_token() {
return Err(Error::RateLimited);
}
let (reply, answer) = oneshot::channel();
let deadline = Instant::now() + DEADLINE;
self.tx
.try_send(Request { args, deadline, reply })
.map_err(|_| Error::Unavailable("too many toasts waiting for Steam".into()))?;
match tokio::time::timeout_at(deadline.into(), answer).await {
Ok(Ok(result)) => result.map_err(Error::Unavailable),
Ok(Err(_)) => Err(Error::Unavailable("the connection to Steam dropped".into())),
Err(_) => Err(Error::Unavailable("Steam didn't answer in time".into())),
}
}
fn take_token(&self) -> bool {
let mut guard = self.bucket.lock().unwrap_or_else(|e| e.into_inner());
let (tokens, last) = &mut *guard;
let now = Instant::now();
*tokens = (*tokens + now.duration_since(*last).as_secs_f64() / REFILL.as_secs_f64()).min(BURST);
*last = now;
if *tokens < 1.0 {
return false;
}
*tokens -= 1.0;
true
}
}
fn arguments(toast: Toast) -> Result<serde_json::Value, Error> {
let cut = |s: &str, max: usize| s.trim().chars().take(max).collect::<String>();
let (app, title, body) = (cut(&toast.app, MAX_TITLE), cut(&toast.title, MAX_TITLE), cut(&toast.body, MAX_BODY));
let title = if title.is_empty() { app } else { title };
if title.is_empty() && body.is_empty() {
return Err(Error::Invalid("a toast needs a title or a body"));
}
let icon = toast.icon.filter(|icon| icon.len() <= MAX_ICON && icon.starts_with("data:image/"));
Ok(serde_json::json!({ "title": title, "body": body, "icon": icon }))
}
/// What the shim returned: `{id, suppressed}` or `{unavailable: reason}`.
pub fn outcome(value: &serde_json::Value) -> Result<Outcome, String> {
if let Some(reason) = value["unavailable"].as_str() {
return Err(reason.to_owned());
}
match value["suppressed"].as_bool() {
Some(true) => Ok(Outcome::HeadsetAsleep),
Some(false) => Ok(Outcome::Shown),
None => Err("unexpected answer from the toast hook".into()),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn toast(app: &str, title: &str, body: &str) -> Toast {
Toast { app: app.into(), title: title.into(), body: body.into(), icon: None }
}
#[test]
fn falls_back_to_the_app_name_and_cuts_long_fields() {
let args = arguments(toast("Signal", "", "hi")).ok().unwrap();
assert_eq!(args["title"], "Signal");
let args = arguments(toast("Signal", "Alex", &"x".repeat(2000))).ok().unwrap();
assert_eq!(args["title"], "Alex");
assert_eq!(args["body"].as_str().unwrap().chars().count(), MAX_BODY);
assert!(matches!(arguments(toast(" ", "", " ")), Err(Error::Invalid(_))));
}
#[test]
fn drops_icons_that_are_not_inline_images() {
let mut t = toast("", "t", "");
t.icon = Some("https://example.com/x.png".into());
assert!(arguments(t).ok().unwrap()["icon"].is_null());
let mut t = toast("", "t", "");
t.icon = Some(format!("data:image/png;base64,{}", "A".repeat(MAX_ICON)));
assert!(arguments(t).ok().unwrap()["icon"].is_null());
}
#[test]
fn reads_the_shims_answer() {
assert_eq!(outcome(&serde_json::json!({"id": 1, "suppressed": false})), Ok(Outcome::Shown));
assert_eq!(outcome(&serde_json::json!({"id": 1, "suppressed": true})), Ok(Outcome::HeadsetAsleep));
assert!(outcome(&serde_json::json!({"unavailable": "no hook"})).is_err());
}
#[test]
fn rate_limits_after_a_burst() {
let (notifier, _rx) = Notifier::new();
let allowed = (0..10).filter(|_| notifier.take_token()).count();
assert_eq!(allowed, BURST as usize);
}
}
+41 -23
View File
@@ -5,6 +5,8 @@
//! - `GET /api/ws` full state as JSON on connect and after every change (throttled)
//! - `GET /api/stream/ws` headset view: JSON `{codec}`, fMP4 init segment, then one
//! moof+mdat per frame (see stream.rs, fmp4.rs)
//! - `POST /api/notify` toast in the headset: JSON `{app, title, body, icon}` -> `{result}`
//! (`shown` | `headset_asleep`); 400 invalid, 429 rate limited, 503 Steam unavailable
//! - `GET /healthz` liveness, the only route without auth
//!
//! `/api/*` requires the token via `?token=` or `Authorization: Bearer`.
@@ -18,10 +20,10 @@ use anyhow::Context;
use axum::Router;
use axum::extract::connect_info::Connected;
use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade};
use axum::extract::{Query, State};
use axum::extract::{DefaultBodyLimit, Query, State};
use axum::http::{HeaderMap, StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use axum::routing::{get, post};
use axum::serve::IncomingStream;
use socket2::{Domain, Protocol, Socket, Type};
use tokio_rustls::TlsAcceptor;
@@ -29,6 +31,7 @@ use tokio_rustls::TlsAcceptor;
use crate::config::Config;
use crate::fmp4;
use crate::hub::Hub;
use crate::notify::{self, Notifier};
use crate::stream::LiveStream;
use crate::tls::Identity;
@@ -43,35 +46,37 @@ struct AppState {
hub: Arc<Hub>,
token: Arc<str>,
stream: Arc<LiveStream>,
notifier: Arc<Notifier>,
}
pub async fn serve(hub: Arc<Hub>, stream: Arc<LiveStream>, config: &Config) -> anyhow::Result<()> {
pub async fn serve(hub: Arc<Hub>, stream: Arc<LiveStream>, notifier: Arc<Notifier>, config: &Config) -> anyhow::Result<()> {
let app = Router::new()
.route("/api/state", get(state))
.route("/api/ws", get(ws))
.route("/api/stream/ws", get(stream_ws))
// Icon plus text; JSON escaping can double the raw size
.route("/api/notify", post(notify).layer(DefaultBodyLimit::max(2 * notify::MAX_ICON + 16 * 1024)))
.route("/healthz", get(|| async { "ok" }))
.layer(axum::middleware::from_fn_with_state(config.allow_remote, crate::access::local_only))
.with_state(AppState {
hub,
token: config.token.as_str().into(),
stream,
notifier,
});
let plain = bind(config.listen)?;
// Fatal rather than degrading to plaintext only: an agent the app silently can't reach is a
// worse support case than one that fails loudly with the address in the message.
// fail loudly
let tls = TlsListener::spawn(
bind(config.listen_tls)?,
TlsAcceptor::from(Identity::load_or_create()?.server_config()?),
)?;
tracing::info!("listening on {} and {} (TLS)", config.listen, config.listen_tls);
// A prompt for whoever is sitting at the terminal; pointless in journald.
// SAFETY: isatty only inspects the descriptor.
if unsafe { libc::isatty(libc::STDOUT_FILENO) } == 1 {
println!("Run `framemate-agent pair` for the code to scan in the app.");
}
// Two signal registrations of the same kind; tokio delivers to all of them.
tokio::try_join!(
axum::serve(plain, app.clone().into_make_service_with_connect_info::<Peer>())
.with_graceful_shutdown(shutdown_signal()),
@@ -81,11 +86,6 @@ pub async fn serve(hub: Arc<Hub>, stream: Arc<LiveStream>, config: &Config) -> a
Ok(())
}
/// Terminates TLS so `axum::serve` keeps handling graceful shutdown, `ConnectInfo` and the
///
/// Handshakes deliberately do *not* happen in `accept`: awaiting one there is serial, so a
/// client that connects and then sends nothing would block every later connection and take the
/// whole TLS port down. They run on their own tasks and queue up here instead.
struct TlsListener {
local: SocketAddr,
ready: tokio::sync::mpsc::Receiver<(tokio_rustls::server::TlsStream<tokio::net::TcpStream>, SocketAddr)>,
@@ -97,12 +97,10 @@ impl TlsListener {
let (tx, ready) = tokio::sync::mpsc::channel(HANDSHAKE_QUEUE);
tokio::spawn(async move {
loop {
// Delegating keeps axum's own policy for accept errors (it backs off on EMFILE).
// delegating keep axums own policy for accept errors
let (stream, peer) = axum::serve::Listener::accept(&mut tcp).await;
let (acceptor, tx) = (acceptor.clone(), tx.clone());
// One task each, with no cap on how many run at once: capping them would mean
// queueing, and half-open connections would starve real ones all over again.
// What bounds this is the timeout above and the process's file descriptor limit.
// one task each
tokio::spawn(async move {
match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await {
Ok(Ok(tls)) => {
@@ -126,8 +124,7 @@ impl axum::serve::Listener for TlsListener {
async fn accept(&mut self) -> (Self::Io, Self::Addr) {
match self.ready.recv().await {
Some(ready) => ready,
// The accept task runs for the life of the process; only reachable if it panicked,
// and `accept` has no way to report that, so stop handing out connections.
// accept task runs for the life of the process
None => std::future::pending().await,
}
}
@@ -137,9 +134,6 @@ impl axum::serve::Listener for TlsListener {
}
}
/// The peer address access.rs checks. Crate-local because axum ships `Connected` only for its
/// own `TcpListener`, and the orphan rule rejects an impl for `SocketAddr`: `TlsListener`
/// appears only as a nested parameter, which doesn't make the impl local.
#[derive(Clone, Copy)]
pub struct Peer(pub SocketAddr);
@@ -166,7 +160,7 @@ fn bind(addr: SocketAddr) -> anyhow::Result<tokio::net::TcpListener> {
}
}
/// Dual-stack for an IPv6 wildcard (`frame.local` often resolves to IPv6). `IPV6_V6ONLY` has to be cleared before `bind`, which `TcpListener::bind` can't do.
// Dual-stack for an IPv6 wildcard (`frame.local` often resolves to IPv6). `IPV6_V6ONLY` has to be cleared before `bind`, which `TcpListener::bind` can't do.
fn listen(addr: SocketAddr) -> anyhow::Result<tokio::net::TcpListener> {
let socket = Socket::new(Domain::for_address(addr), Type::STREAM, Some(Protocol::TCP))
.context("creating the listening socket")?;
@@ -204,6 +198,30 @@ async fn state(
(cors, [(header::CONTENT_TYPE, "application/json")], app.hub.snapshot_json()).into_response()
}
async fn notify(
State(app): State<AppState>,
headers: HeaderMap,
Query(query): Query<HashMap<String, String>>,
body: axum::body::Bytes,
) -> Response {
if !authorized(&app, &headers, &query) {
return StatusCode::UNAUTHORIZED.into_response();
}
// Parsed here rather than with the Json extractor, so a rejection never echoes content.
let Ok(toast) = serde_json::from_slice::<notify::Toast>(&body) else {
return (StatusCode::BAD_REQUEST, "expected JSON {app, title, body, icon}\n").into_response();
};
match app.notifier.send(toast).await {
Ok(outcome) => axum::Json(outcome).into_response(),
Err(notify::Error::Invalid(reason)) => (StatusCode::BAD_REQUEST, format!("{reason}\n")).into_response(),
Err(notify::Error::RateLimited) => (StatusCode::TOO_MANY_REQUESTS, "slow down\n").into_response(),
Err(notify::Error::Unavailable(reason)) => {
tracing::info!("toast not shown: {reason}");
(StatusCode::SERVICE_UNAVAILABLE, format!("{reason}\n")).into_response()
}
}
}
async fn ws(
State(app): State<AppState>,
headers: HeaderMap,
+122 -1
View File
@@ -5,13 +5,14 @@
const W = window;
try { W.__framemate?.dispose?.(); } catch {}
const handles = [], timers = [];
const handles = [], timers = [], cleanups = [];
let disposed = false;
W.__framemate = {
dispose() {
disposed = true;
for (const h of handles) try { h.unregister(); } catch {}
for (const t of timers) clearInterval(t);
for (const c of cleanups) try { c(); } catch {}
},
};
@@ -205,6 +206,115 @@
});
};
// --- notifications in the headset: Steam's own toast queue, rendered by our component ---
// Steam has no toast type for free text (each type has a fixed renderer), so creating the
// element for its renderer is intercepted for notifications that carry `framemate`. Steam
// still does the rest: queue, sound, the SteamVR notification overlay ("SteamVR -
// Notifications"), "disable all toasts" and suppression while the headset is asleep.
// Webpack modules are found by shape, never by id (ids change with every Steam build).
const LOGO = "data:image/svg+xml;charset=utf-8," + encodeURIComponent("__FRAMEMATE_LOGO_SVG__");
// What the module scan finds, kept per JS context:
// requiring every module again on each re-injection (agent reconnects) would be slow
const findToastParts = () => {
let require;
W.webpackChunksteamui.push([[Symbol("framemate")], {}, r => { require = r; }]);
// Steams runtime exposes no module cache (`require.c`), require every id (like decky) also evaluate modules Steam hadnt run yet
const loaded = Object.keys(require?.m ?? {})
.map(id => { try { return require(id); } catch { return null; } })
.filter(e => e && typeof e === "object");
const source = f => { try { return Function.prototype.toString.call(f); } catch { return ""; } };
let renderer = null;
for (const e of loaded) {
for (const key of Object.keys(e)) {
let v;
try { v = e[key]; } catch { continue; }
if (typeof v === "function" && source(v).includes('controller:"notification",method:')) renderer = v;
}
}
const runtimes = loaded.filter(e => typeof e.jsx === "function" && typeof e.jsxs === "function" && "Fragment" in e);
const css = loaded.map(e => (e.ShortTemplate ? e : e.default)).find(e => e?.ShortTemplate && e.ShortLogoDimensions && e.Title && e.Body);
// toasts borrow "download complete" type. must be a real type
const store = Object.getPrototypeOf(W.NotificationStore);
const member = f => /\.OnNotification\([^,]+,[\w$]+\.[\w$]+\.([\w$]+),/.exec(source(f))?.[1];
const typeKey = member(store.TestDownloadComplete), checkKey = member(store.TestFriendMessage);
let type;
for (const e of typeKey && checkKey ? loaded : []) {
for (const key of Object.keys(e)) {
let v;
try { v = e[key]; } catch { continue; }
if (v && typeof v === "object" && Number.isInteger(v[typeKey]) && Number.isInteger(v[checkKey])) type ??= v[typeKey];
}
}
if (!renderer || !runtimes.length || !css || type === undefined) {
throw new Error(`toast hook: renderer ${!!renderer}, jsx runtimes ${runtimes.length}, css ${!!css}, type ${type}`);
}
return { renderer, runtimes, css, type };
};
const toasts = () => {
const { renderer, runtimes, css, type } = (W.__framemateToastParts ??= findToastParts());
const { jsx, jsxs } = runtimes[0]; // originals, taken before patching
const cx = (...names) => names.filter(Boolean).join(" ");
// Same markup and classes as Steam's own short toast template.
const ours = group => group?.notifications?.findLast?.(n => n?.framemate)?.framemate;
const Toast = ({ group, onDismiss }) => {
const n = ours(group);
return jsxs("div", {
className: cx(css.ShortTemplate, n.body && css.TwoLine),
onClick: () => onDismiss?.(),
children: [
jsx("div", {
className: css.ShortLogoDimensions,
children: jsx("img", { src: n.icon || LOGO, style: { width: "100%", height: "100%", objectFit: "contain" } }),
}),
jsxs("div", {
className: css.Content,
children: [
jsx("div", { className: css.Header, children: jsx("div", { className: css.Title, children: n.title }) }),
n.body ? jsx("div", { className: css.Body, children: n.body }) : null,
],
}),
],
});
};
for (const runtime of runtimes) {
for (const name of ["jsx", "jsxs"]) {
const original = runtime[name];
const wrapper = function (type, props) {
if (!disposed && type === renderer && ours(props?.group)) {
return original.call(this, Toast, props, ...[...arguments].slice(2));
}
return original.apply(this, arguments);
};
runtime[name] = wrapper;
// If something wrapped it again on top of us, `disposed` turns ours into a passthrough.
cleanups.push(() => { if (runtime[name] === wrapper) runtime[name] = original; });
}
}
const text = (v, max) => String(v ?? "").slice(0, max);
// Only images: arbitrary URL would let a caller make Steams UI fetch it
const image = v => (typeof v === "string" && /^data:image\/(png|jpeg|webp|svg\+xml)[;,]/.test(v) ? v : null);
// sound follows borrowed types config and users toast sound setting
// returns {id, suppressed}; suppressed = the headset is asleep, so Steam drops the toast
W.__framemate.notify = ({ title, body, icon, duration_ms = 6000 } = {}) => {
const store = W.NotificationStore;
const id = store.m_nNextTestNotificationID++;
const suppressed = !!store.BShouldSupressToasts();
store.ProcessNotification(
// bEnableInReducedUI: the Frame runs Steam in "reduced UI" mode, which drops other toasts
{ showToast: true, playSound: true, sound: 6 /* ToastMisc */, eFeature: 0, bCritical: false, fnTray: null,
bEnableInReducedUI: true,
toastDurationMS: Math.min(Math.max(Number(duration_ms) || 6000, 2000), 15000) },
{ notificationID: id, rtCreated: Math.floor(Date.now() / 1000), eType: type, nToastDurationMS: 0,
eSource: 1, data: {}, bNewIndicator: true,
framemate: { title: text(title, 120), body: text(body, 500), icon: image(icon) } },
0 /* New */);
return { id, suppressed };
};
};
// Steam's stores load after the context is created; wait for them.
const start = () => {
if (disposed) return;
@@ -215,6 +325,17 @@
for (const section of [battery, downloads, apps, vr, stores]) {
try { section(); } catch (e) { fail(section.name, e); }
}
startToasts();
};
// Toasts wait for their own prerequisites, so a Steam change there only costs the toasts.
const startToasts = (tries = 0) => {
if (disposed) return;
if (!W.NotificationStore || !W.webpackChunksteamui) {
if (tries < 30) timers.push(setTimeout(() => startToasts(tries + 1), 2000));
else fail("toasts", new Error("NotificationStore or webpackChunksteamui not found"));
return;
}
try { toasts(); } catch (e) { fail("toasts", e); }
};
start();
return "framemate shim installed";
+59 -45
View File
@@ -20,66 +20,80 @@ die() { printf 'error: %s\n' "$*" >&2; exit 1; }
agent() { flatpak run --user "$APP_ID" "$@"; }
# dont display escaped and control characters
transaction() {
if { true </dev/tty; } 2>/dev/null; then
flatpak "$@" </dev/tty
else
FLATPAK_FANCY_OUTPUT=0 flatpak "$@"
fi
}
uninstall() {
if flatpak info --user "$APP_ID" >/dev/null 2>&1; then
if flatpak list --user --app --columns=application | grep -qx "$APP_ID"; then
agent uninstall-service || say "Couldn't remove the service; continuing."
# No --delete-data: the token and TLS key stay, so reinstalling doesn't force a re-pair.
flatpak uninstall --user -y "$APP_ID"
transaction uninstall --user -y "$APP_ID"
say "Removed FrameMate."
else
say "FrameMate isn't installed for this user; nothing to do."
fi
}
# The agent lives in the user's Flatpak installation and a systemd *user* unit, so as root
# all of it would land in the wrong place.
[ "$(id -u)" -ne 0 ] || die "run this as your normal user, not as root."
command -v flatpak >/dev/null || die "flatpak is not installed."
main() {
[ "$(id -u)" -ne 0 ] || die "run this as your normal user, not as root."
command -v flatpak >/dev/null || die "flatpak is not installed."
case "${1:-install}" in
install) ;;
uninstall) uninstall; exit 0 ;;
*) die "unknown command '$1' (install, uninstall)" ;;
esac
case "${1:-install}" in
install) ;;
uninstall) uninstall; exit 0 ;;
*) die "unknown command '$1' (install, uninstall)" ;;
esac
arch=$(uname -m)
[ "$arch" = aarch64 ] || die "the agent runs on the Steam Frame (aarch64), not on $arch.
arch=$(uname -m)
[ "$arch" = aarch64 ] || die "the agent runs on the Steam Frame (aarch64), not on $arch.
Run this in a terminal on the Frame via SSH, or use the Desktop Mode Konsole."
command -v curl >/dev/null || die "curl is not installed."
command -v curl >/dev/null || die "curl is not installed."
# A fresh Frame may not have the remote the Freedesktop runtime comes from.
flatpak remote-add --user --if-not-exists \
flathub https://dl.flathub.org/repo/flathub.flatpakrepo >/dev/null
# A fresh Frame may not have the remote the Freedesktop runtime comes from.
flatpak remote-add --user --if-not-exists \
flathub https://dl.flathub.org/repo/flathub.flatpakrepo >/dev/null
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT INT TERM
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT INT TERM
say "Downloading the agent..."
curl -fL --progress-bar -o "$tmp/$BUNDLE" "$RELEASE/$BUNDLE" ||
die "download failed. Check the network, or grab $BUNDLE from the releases page by hand."
say "Downloading the agent..."
curl -fL --progress-bar -o "$tmp/$BUNDLE" "$RELEASE/$BUNDLE" ||
die "download failed. Check the network, or grab $BUNDLE from the releases page by hand."
say ""
say "Installing. Pulling Freedesktop Runtime from Flathub the first time (about 270 MB), may take a while."
# --reinstall so re-running this script updates an existing install.
flatpak install --user -y --reinstall "$tmp/$BUNDLE"
say ""
# Register the unit
# Restart agent
# Runs the self check
# In Desktop Mode it cant reach systemd, says so, and still exits 0 with the unit enabled for the next boot.
agent install-service
say ""
agent pair
# install-service exits 0 whether or not the agent actually came up, so ask the agent itself.
# The pairing code above stays valid either way; only connecting has to wait.
if ! curl -fsS --max-time 5 -o /dev/null "http://127.0.0.1:7380/healthz" 2>/dev/null; then
say ""
say "NOTE: the agent isn't running yet, which is normal in Desktop Mode installs."
say "Scan the code above now anyway: The app will remember the fingerprint and connect to your headset once it's back up."
fi
say "Installing. Pulling Freedesktop Runtime from Flathub the first time (about 270 MB), may take a while."
# --reinstall so re-running this script updates an existing install.
transaction install --user -y --reinstall "$tmp/$BUNDLE"
say ""
# Register the unit, restart the agent and run the self check
# In Desktop Mode it can't reach systemd, say so, recommend reboot
service_ok=1
agent install-service || service_ok=0
say ""
agent pair
# Ask the agent itself whether it came up; the pairing code above stays valid either way.
if [ "$service_ok" -eq 0 ]; then
say ""
say "NOTE: the self check above found a problem. Once it's sorted, the app connects with this code."
say "Repeat the check with: flatpak run --user $APP_ID check"
elif ! curl -fsS --max-time 5 -o /dev/null "http://127.0.0.1:7380/healthz" 2>/dev/null; then
say ""
say "NOTE: the agent isn't running yet, which is normal in Desktop Mode installs."
say "Scan the code above now anyway: The app will remember the fingerprint and connect to your headset once it's back up."
say "Reboot your headset or the service to start."
fi
}
main "$@"