Send analytics to the existing PostHog project; make bug reports private

- Analytics go to the maintainer's PostHog US project 343535, tagged
  $lib = frame-control. Every event carries $ip 0.0.0.0, since PostHog
  stores the sender's address otherwise (checked live), including events
  queued by earlier versions.
- Report a problem sends a private problem_report event to PostHog instead
  of a public GitHub issue, with its own random id so a contact address
  can't be linked to analytics. The dialog asks how to reach the person and
  shows a reference. Maintainers read reports on the PostHog dashboard or
  with `python3 ui/frame_report.py inbox`.
- Community sync pages by timestamp in UTC: PostHog refuses OFFSET for
  personal API keys.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

ui/frame_compat_db.py

@@ -267,7 +267,7 @@ def _posthog_query(sql):
     project = os.environ.get('FRAME_CONTROL_POSTHOG_PROJECT') or cfg.get('project')
     if not project:
         raise DBError('No PostHog project id (ui/telemetry.json "project", or FRAME_CONTROL_POSTHOG_PROJECT)')
-    # The query API lives on the app host (eu.posthog.com), not the ingestion host (eu.i.posthog.com).
+    # The query API lives on the app host (us.posthog.com), not the ingestion host (us.i.posthog.com).
     host = cfg['host'].replace('.i.posthog.com', '.posthog.com')
     req = urllib.request.Request(f'{host}/api/projects/{urllib.parse.quote(str(project))}/query/', method='POST',
                                  data=json.dumps({'query': {'kind': 'HogQLQuery', 'query': sql}}).encode(),
@@ -352,15 +352,21 @@ def sync(dry_run=False):
     key()  # the maintainer's copy only
     state = _sync_state()
     since = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time() - SYNC_OVERLAP_DAYS * 86400))
-    events = []
-    for page in range(40):
-        res = _posthog_query("SELECT properties, distinct_id, timestamp FROM events "
-                             f"WHERE event = 'compat_report' AND timestamp >= toDateTime('{since}') "
-                             f"ORDER BY timestamp, uuid LIMIT {SYNC_PAGE} OFFSET {page * SYNC_PAGE}")
+    events, after = [], f"timestamp >= toDateTime('{since}', 'UTC')"
+    for _ in range(40):
+        # Keyset paging: PostHog refuses OFFSET with a personal API key. The cursor is in UTC,
+        # since a local time is ambiguous in the hour clocks go back.
+        res = _posthog_query("SELECT properties, distinct_id, timestamp, toString(uuid), "
+                             "formatDateTime(timestamp, '%Y-%m-%d %H:%i:%S.%f', 'UTC') FROM events "
+                             f"WHERE event = 'compat_report' AND {after} "
+                             f"ORDER BY timestamp, toString(uuid) LIMIT {SYNC_PAGE}")
         rows = res.get('results') or []
-        events += rows
+        events += [row[:3] for row in rows]
         if len(rows) < SYNC_PAGE:
             break
+        last_uuid, last_ts = rows[-1][3], rows[-1][4]
+        after = (f"(timestamp > toDateTime64('{last_ts}', 6, 'UTC') OR "
+                 f"(timestamp = toDateTime64('{last_ts}', 6, 'UTC') AND toString(uuid) > '{last_uuid}'))")
     rows, skipped = community_rows(events, state)
     if dry_run:
         return rows, skipped

ui/frame_report.py

@@ -1,32 +1,26 @@
 """Report a problem from inside Frame Control. Python stdlib only.
 
 The page's Report a problem dialog shows the diagnostics below before anything
-is sent, then this sends the report to the website's feedback API
-(site/functions/api/feedback.js), which files it as a GitHub issue. The user
-needs no GitHub account. Everything collected is scrubbed first
-(frame_telemetry.scrub), since the issue is public.
+is sent, then this sends the report privately to Frame Control's PostHog
+project as a `problem_report` event: only the maintainer can read it, and
+nothing is published. It is sent whatever the analytics settings are, because
+the person sends it deliberately. Diagnostics are scrubbed first
+(frame_telemetry.scrub); the person's own words are sent as written.
 """
-import json
 import os
 import platform
 import sys
 import time
-import urllib.error
-import urllib.request
+import uuid
 
 import frame_host
 import frame_telemetry
 
-FEEDBACK_URL = os.environ.get('FRAME_CONTROL_FEEDBACK_URL', 'https://frame-control.pages.dev/api/feedback')
-ISSUES_URL = 'https://github.com/saphid/frame-control/issues/new'
 KINDS = ('bug', 'idea', 'question', 'other')
-MESSAGE_MAX = 5000  # the feedback API's limit, in JavaScript (UTF-16) units
-TEXT_MAX = 3500     # the person's own text
-DIAG_MAX = 1300     # the diagnostics block, so text + diagnostics always fit MESSAGE_MAX
-LOG_LINES = 40
+TEXT_MAX = 5000     # the person's own text, in JavaScript (UTF-16) units like the page's maxlength
+DIAG_MAX = 8000     # the diagnostics block
+LOG_LINES = 60
 ACTIVITY_LINES = 25
-HEAD = '\n\n---\nDiagnostics from Frame Control (personal details removed):\n```\n'
-TAIL = '\n```'
 
 frame = {}  # the Frame's last known SteamOS build, set by server.status()
 
@@ -96,52 +90,72 @@ def diagnostics(activity=(), include_logs=False, limit=DIAG_MAX):
 
 
 def compose(body):
-    """(title, message) for the feedback API: the person's text, then the diagnostics exactly as
-    the dialog previewed them (passed back, scrubbed again and bounded here)."""
+    """(title, text, diagnostics): the diagnostics exactly as the dialog previewed them (passed
+    back, scrubbed again and bounded here)."""
     title = ' '.join(str(body.get('title') or '').split())
     text = str(body.get('message') or '').strip()
     if len(title) < 5:
         raise ValueError('give it a short title (at least 5 characters)')
     if len(text) < 10:
         raise ValueError('say a little more about what happened (at least 10 characters)')
-    title, text = cut(title, 120), cut(text, TEXT_MAX)
     diag = body.get('diagnostics')
-    if not isinstance(diag, str) or not diag.strip():
-        return title, text
-    diag = cut(frame_telemetry.scrub(diag, 20000), DIAG_MAX)
-    return title, f'{text}{HEAD}{diag}{TAIL}'
+    diag = cut(frame_telemetry.scrub(diag, 40000), DIAG_MAX) if isinstance(diag, str) and diag.strip() else ''
+    return cut(title, 120), cut(text, TEXT_MAX), diag
 
 
 def send(body):
-    """File the report. Returns {"number", "url"} of the new issue; raises ReportError."""
+    """Send the report to PostHog. Returns {"id", "message"}; raises ReportError."""
     kind = body.get('kind') if body.get('kind') in KINDS else 'bug'
-    title, message = compose(body)
-    payload = {'kind': kind, 'title': title, 'message': message, 'website': '',
-               'github': str(body.get('github') or '').strip().lstrip('@')[:40],
-               'version': frame_telemetry.app_version(), 'os': f'{frame_host.NAME} {platform.machine()}',
-               'steamos': str(frame.get('build') or '')[:120],
-               # How long the dialog was open; the API treats anything under 3 s as a script.
-               'elapsed': max(0, int(body.get('elapsed') or 0))}
-    req = urllib.request.Request(FEEDBACK_URL, data=json.dumps(payload).encode(), method='POST',
-                                 headers={'content-type': 'application/json',
-                                          'user-agent': f'FrameControl/{frame_telemetry.app_version()}'})
+    title, text, diag = compose(body)
+    ref = uuid.uuid4().hex[:8].upper()
+    props = {**frame_telemetry.common(), 'kind': kind, 'title': title, 'message': text,
+             'contact': str(body.get('contact') or '').strip()[:120], 'diagnostics': diag,
+             'report_id': ref, 'steamos': str(frame.get('build') or '')[:120], 'level': 'report'}
+    # Its own random id: a report can carry contact details, so it isn't linked to this copy's analytics.
+    event = {'event': 'problem_report', 'distinct_id': str(uuid.uuid4()), 'uuid': str(uuid.uuid4()),
+             'timestamp': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'properties': props}
     try:
-        with urllib.request.urlopen(req, timeout=30) as r:
-            res = json.loads(r.read() or b'{}')
-    except urllib.error.HTTPError as e:
-        try:
-            why = json.loads(e.read() or b'{}').get('error')
-        except ValueError:
-            why = None
-        e.close()
-        raise ReportError(why or f'the feedback service said HTTP {e.code}')
-    except (urllib.error.URLError, OSError, ValueError) as e:
-        raise ReportError(f"couldn't reach the feedback service: {e}")
-    if not res.get('url'):
-        raise ReportError("the feedback service didn't file it; try again in a moment")
-    frame_telemetry.capture('problem_reported', {'kind': kind, 'with_diagnostics': bool(body.get('diagnostics'))})
-    return {'number': res.get('number'), 'url': res['url'], 'message': f"Sent. It's issue #{res.get('number')} on GitHub."}
+        frame_telemetry.post([event], timeout=30)
+    except frame_telemetry.SendError as e:
+        raise ReportError(str(e))
+    try:
+        frame_telemetry.record_sent([event])
+    except OSError:
+        pass  # it was sent; failing to log it here mustn't make the person send it again
+    return {'id': ref, 'message': f'Sent privately to the Frame Control developer (report {ref}).'}
 
 
 class ReportError(RuntimeError):
     pass
+
+
+def inbox(days=30):
+    """The maintainer's recent reports from PostHog, newest first (needs the personal API key
+    frame_compat_db.sync uses)."""
+    import frame_compat_db
+    res = frame_compat_db._posthog_query(
+        "SELECT timestamp, properties.report_id, properties.kind, properties.title, properties.message, "
+        "properties.contact, properties.app_version, properties.os, properties.steamos, properties.diagnostics "
+        f"FROM events WHERE event = 'problem_report' AND timestamp > now() - INTERVAL {int(days)} DAY "
+        "ORDER BY timestamp DESC LIMIT 200")
+    return res.get('results') or []
+
+
+def main():
+    cmd, *args = sys.argv[1:] or ['inbox']
+    if cmd != 'inbox':
+        sys.exit('usage: frame_report.py inbox [days]')
+    for row in inbox(*(args[:1] or [30])):
+        if not isinstance(row, list) or len(row) != 10:
+            continue
+        ts, ref, kind, title, text, contact, version, osname, steamos, diag = (str(v or '') for v in row)
+        print(f"== {ts[:16].replace('T', ' ')}  {ref}  [{kind}] {title}")
+        print(f"   {version} on {osname}, SteamOS {steamos or 'unknown'}{', reply to ' + contact if contact else ''}")
+        print('   ' + text.replace('\n', '\n   '))
+        if diag:
+            print('   --- diagnostics\n   ' + diag.replace('\n', '\n   '))
+        print()
+
+
+if __name__ == '__main__':
+    main()

ui/frame_telemetry.py

@@ -14,8 +14,8 @@ every event and property):
   home folders, user names, addresses and keys.
 
 The first-run notice offers compat and diagnostics together, and the page's
-Report a problem dialog (frame_report.py) files bug reports whatever is chosen
-here.
+Report a problem dialog (frame_report.py) sends bug reports privately to the
+same project whatever is chosen here.
 
 Events are identified by a random id made on first run, not by the person or
 computer, and sent without person profiles or GeoIP. Nothing is sent without a
@@ -53,7 +53,7 @@ SENT_KEEP = 200
 OUTBOX_MAX = 2000  # events kept while offline; the oldest go first
 FLUSH_EVERY = 60
 REPEAT_WINDOW = 600  # the same diagnostic error is sent at most once in this many seconds
-DEFAULT_HOST = 'https://eu.i.posthog.com'
+DEFAULT_HOST = 'https://us.i.posthog.com'
 
 LEVELS = ('usage', 'compat', 'diagnostics')
 # Events the page may send through /api/telemetry, and the properties each may carry.
@@ -270,11 +270,12 @@ def categorize(message):
 
 # ---- capturing ------------------------------------------------------------------
 
-def _common():
+def common():
     return {'app_version': app_version(), 'os': frame_host.NAME, 'arch': platform.machine().lower(),
             'python': '%d.%d' % sys.version_info[:2], '$lib': 'frame-control',
-            # Anonymous events: no person profile, no location lookup.
-            '$process_person_profile': False, '$geoip_disable': True}
+            # Anonymous events: no person profile, no location lookup, and a placeholder address,
+            # since PostHog stores the sender's IP unless an event gives one.
+            '$process_person_profile': False, '$geoip_disable': True, '$ip': '0.0.0.0'}
 
 
 def app_version():
@@ -296,7 +297,7 @@ def capture(event, props=None, level='usage'):
         s = settings()
         e = {'event': event, 'distinct_id': s['id'], 'uuid': str(uuid.uuid4()),
              'timestamp': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
-             'properties': {**_common(), **(props or {}), 'level': level}}
+             'properties': {**common(), **(props or {}), 'level': level}}
         with _lock:
             lines = _read_lines(OUTBOX) + [e]
             _write_lines(OUTBOX, lines[-OUTBOX_MAX:])
@@ -459,28 +460,50 @@ def _drop_unwanted(s):
         _write_lines(OUTBOX, kept)
 
 
+def post(batch, timeout=20):
+    """Send events to PostHog now. Raises SendError if they weren't accepted."""
+    cfg = config()
+    if not cfg['key']:
+        raise SendError('no PostHog project key in this build')
+    for e in batch:  # also events queued by versions that didn't add the placeholder address
+        e.setdefault('properties', {})['$ip'] = '0.0.0.0'
+    body = json.dumps({'api_key': cfg['key'], 'batch': batch}).encode()
+    req = urllib.request.Request(cfg['host'] + '/batch/', data=body, method='POST',
+                                 headers={'content-type': 'application/json',
+                                          'user-agent': f'FrameControl/{app_version()}'})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as r:
+            r.read()
+    except urllib.error.HTTPError as e:
+        e.close()
+        raise SendError(f'PostHog said HTTP {e.code}')
+    except (urllib.error.URLError, OSError, ValueError) as e:
+        raise SendError(f"couldn't reach PostHog: {e}")
+
+
+def record_sent(events):
+    """Add events sent outside the outbox to the log the page shows."""
+    with _lock:
+        _write_lines(SENT, (_read_lines(SENT) + list(events))[-SENT_KEEP:])
+
+
+class SendError(RuntimeError):
+    pass
+
+
 def flush(timeout=20):
     """Send what's queued. Returns how many were sent; on failure they stay queued."""
     with _send_lock:
         if blocked() or not settings()['notice_shown']:
             return 0
-        cfg = config()
         with _lock:
             _drop_unwanted(settings())
             batch = _read_lines(OUTBOX)[:100]
         if not batch:
             return 0
-        body = json.dumps({'api_key': cfg['key'], 'batch': batch}).encode()
-        req = urllib.request.Request(cfg['host'] + '/batch/', data=body, method='POST',
-                                     headers={'content-type': 'application/json',
-                                              'user-agent': f'FrameControl/{app_version()}'})
         try:
-            with urllib.request.urlopen(req, timeout=timeout) as r:
-                r.read()
-        except urllib.error.HTTPError as e:
-            e.close()
-            return 0
-        except (urllib.error.URLError, OSError, ValueError):
+            post(batch, timeout)
+        except SendError:
             return 0
         sent_ids = {e['uuid'] for e in batch}
         with _lock:

ui/index.html

@@ -759,19 +759,18 @@
       <option value="question">A question</option><option value="other">Something else</option></select></label>
     <label class="field">Title<input type="text" id="bugTitleIn" maxlength="120" required minlength="5"
       placeholder="e.g. Installing an APK stops at 'copying to the Frame'"></label>
-    <label class="field">What happened?<textarea id="bugText" maxlength="3500" required minlength="10"
+    <label class="field">What happened?<textarea id="bugText" maxlength="5000" required minlength="10"
       placeholder="What you did, what happened, and what you expected."></textarea></label>
-    <label class="field">GitHub username (optional, so you can follow replies)<input type="text" id="bugGithub" maxlength="40" placeholder="@yourname"></label>
+    <label class="field">How can we reach you? (optional, for a reply)<input type="text" id="bugContact" maxlength="120" placeholder="Email, GitHub or Discord name"></label>
     <label class="popt"><input type="checkbox" id="bugDiag" checked><b>Include diagnostics</b>
       <span class="sub">Frame Control's version, your OS and the Frame's SteamOS build.</span></label>
     <label class="popt"><input type="checkbox" id="bugLogs"><b>Also include recent activity and the server log</b>
       <span class="sub">Often shows what went wrong, but can contain file and app names. Check it below before sending.</span></label>
     <details id="bugDiagBox"><summary>Show exactly what's included</summary><div class="sentlog" id="bugDiagText">Loading…</div></details>
-    <p id="bugWarn">This becomes a public issue on GitHub (saphid/frame-control). No GitHub account is needed.</p>
+    <p id="bugWarn">Sent privately to the Frame Control developer. Nothing is published.</p>
     <div class="row rep-actions"><span class="sub" id="bugMsg"></span><span class="spacer"></span>
       <button type="button" class="small" id="bugCancel">Cancel</button>
       <button type="button" class="small" id="bugCopy">Copy report</button>
-      <a class="small" id="bugGithubLink" href="#" target="_blank" hidden>Open on GitHub instead</a>
       <button type="submit" class="action small" id="bugSend">Send report</button></div>
   </form>
 </dialog>
@@ -2331,18 +2330,14 @@ document.querySelectorAll("nav a").forEach(a => a.addEventListener("click", () =
   if (!tabsSeen.has(tab)) { tabsSeen.add(tab); pageEvent("tab_viewed", { tab }); }
 }));
 
-// ---- report a problem (ui/frame_report.py): a GitHub issue via the website, with diagnostics ----
-const bug = { opened: 0, preview: "" };
+// ---- report a problem (ui/frame_report.py): sent privately to PostHog, with diagnostics ----
+const bug = { preview: "" };
 const activityLines = () => [...$("log").children].slice(0, 25).map(el => el.textContent.trim());
 function bugReportText() {
-  const body = `${$("bugText").value.trim()}${bug.preview ? "\n\n---\nDiagnostics:\n```\n" + bug.preview + "\n```" : ""}`;
+  const contact = $("bugContact").value.trim();
+  const body = `Kind: ${$("bugKind").value}${contact ? `\nContact: ${contact}` : ""}\n\n${$("bugText").value.trim()}${bug.preview ? "\n\n---\nDiagnostics:\n```\n" + bug.preview + "\n```" : ""}`;
   return { title: $("bugTitleIn").value.trim(), body };
 }
-function bugGithubUrl() {
-  const { title, body } = bugReportText();
-  return "https://github.com/saphid/frame-control/issues/new?labels=feedback&title=" + encodeURIComponent(title)
-         + "&body=" + encodeURIComponent(body.slice(0, 6000));
-}
 // The preview is a snapshot: exactly this text is sent, even if more activity happens meanwhile.
 async function loadBugPreview() {
   if (!$("bugDiag").checked) { bug.preview = ""; $("bugDiagText").textContent = "Nothing: diagnostics are off."; return; }
@@ -2353,10 +2348,9 @@ async function loadBugPreview() {
 }
 function openBugReport() {
   $("bugForm").reset();
-  $("bugMsg").textContent = ""; $("bugGithubLink").hidden = true; $("bugSend").disabled = false;
+  $("bugMsg").textContent = ""; $("bugSend").disabled = false;
   $("bugCancel").textContent = "Cancel";
   $("bugDiagBox").open = false; $("bugLogs").disabled = false;
-  bug.opened = performance.now();
   $("bugDlg").showModal();
   loadBugPreview();
 }
@@ -2376,14 +2370,12 @@ $("bugForm").onsubmit = async e => {
   try {
     const res = await api("/api/report", {
       kind: $("bugKind").value, title: $("bugTitleIn").value, message: $("bugText").value,
-      github: $("bugGithub").value, diagnostics: $("bugDiag").checked ? bug.preview : "",
-      elapsed: Math.round(performance.now() - bug.opened) });
-    $("bugMsg").innerHTML = `Sent. <a href="${esc(res.url)}" target="_blank">Issue #${esc(String(res.number))}</a> on GitHub.`;
+      contact: $("bugContact").value, diagnostics: $("bugDiag").checked ? bug.preview : "" });
+    $("bugMsg").textContent = `Sent, thank you. Your reference is ${res.id}.`;
     $("bugCancel").textContent = "Close";
     log(res.message, "ok");
   } catch (err) {
-    $("bugMsg").textContent = `Couldn't send it: ${err.message}.`;
-    $("bugGithubLink").href = bugGithubUrl(); $("bugGithubLink").hidden = false;
+    $("bugMsg").textContent = `Couldn't send it: ${err.message}. Try again later, or use Copy report.`;
     $("bugSend").disabled = false;
   }
 };

ui/telemetry.json

@@ -1,5 +1,5 @@
 {
-  "host": "https://eu.i.posthog.com",
-  "key": "",
-  "project": ""
+  "host": "https://us.i.posthog.com",
+  "key": "phc_qkmbgQBvl2oBXGUVzfV6gG52EpmJdeaQyaRIxHRoQoL",
+  "project": "343535"
 }