diff --git a/.changeset/moody-dingos-happen.md b/.changeset/moody-dingos-happen.md
new file mode 100644
index 000000000000..d1b6bc504f54
--- /dev/null
+++ b/.changeset/moody-dingos-happen.md
@@ -0,0 +1,5 @@
+---
+'@sveltejs/kit': patch
+---
+
+Track scroll position without scroll listener, and recover on reload
diff --git a/packages/kit/src/runtime/client/router.js b/packages/kit/src/runtime/client/router.js
index d40cfe6192db..32f09f6aae99 100644
--- a/packages/kit/src/runtime/client/router.js
+++ b/packages/kit/src/runtime/client/router.js
@@ -2,6 +2,21 @@ import { onMount } from 'svelte';
 import { normalize_path } from '../../utils/url';
 import { get_base_uri } from './utils';
 
+// We track the scroll position associated with each history entry in sessionStorage,
+// rather than on history.state itself, because when navigation is driven by
+// popstate it's too late to update the scroll position associated with the
+// state we're navigating from
+const SCROLL_KEY = 'sveltekit:scroll';
+
+/** @typedef {{ x: number, y: number }} ScrollPosition */
+/** @type {Record<number, ScrollPosition>} */
+let scroll_positions = {};
+try {
+	scroll_positions = JSON.parse(sessionStorage[SCROLL_KEY]);
+} catch {
+	// do nothing
+}
+
 function scroll_state() {
 	return {
 		x: pageXOffset,
@@ -63,6 +78,11 @@ export class Router {
 			history.replaceState({ ...history.state, 'sveltekit:index': 0 }, '', location.href);
 		}
 
+		// if we reload the page, or Cmd-Shift-T back to it,
+		// recover scroll position
+		const scroll = scroll_positions[this.current_history_index];
+		if (scroll) scrollTo(scroll.x, scroll.y);
+
 		this.hash_navigating = false;
 
 		this.callbacks = {
@@ -75,9 +95,7 @@ export class Router {
 	}
 
 	init_listeners() {
-		if ('scrollRestoration' in history) {
-			history.scrollRestoration = 'manual';
-		}
+		history.scrollRestoration = 'manual';
 
 		// Adopted from Nuxt.js
 		// Reset scrollRestoration to auto when leaving page, allowing page reload
@@ -102,28 +120,16 @@ export class Router {
 			}
 		});
 
-		// Setting scrollRestoration to manual again when returning to this page.
-		addEventListener('load', () => {
-			history.scrollRestoration = 'manual';
-		});
-
-		// There's no API to capture the scroll location right before the user
-		// hits the back/forward button, so we listen for scroll events
+		addEventListener('visibilitychange', () => {
+			if (document.visibilityState === 'hidden') {
+				this.#update_scroll_positions();
 
-		/** @type {NodeJS.Timeout} */
-		let scroll_timer;
-		addEventListener('scroll', () => {
-			clearTimeout(scroll_timer);
-			scroll_timer = setTimeout(() => {
-				// Store the scroll location in the history
-				// This will persist even if we navigate away from the site and come back
-				const new_state = {
-					...(history.state || {}),
-					'sveltekit:scroll': scroll_state()
-				};
-				history.replaceState(new_state, document.title, window.location.href);
-				// iOS scroll event intervals happen between 30-150ms, sometimes around 200ms
-			}, 200);
+				try {
+					sessionStorage[SCROLL_KEY] = JSON.stringify(scroll_positions);
+				} catch {
+					// do nothing
+				}
+			}
 		});
 
 		/** @param {Event} event */
@@ -196,6 +202,8 @@ export class Router {
 				// clicking a hash link and those triggered by popstate
 				this.hash_navigating = true;
 
+				this.#update_scroll_positions();
+
 				const info = this.parse(url);
 				if (info) {
 					return this.renderer.update(info, [], false);
@@ -225,7 +233,7 @@ export class Router {
 
 				this._navigate({
 					url: new URL(location.href),
-					scroll: event.state['sveltekit:scroll'],
+					scroll: scroll_positions[event.state['sveltekit:index']],
 					keepfocus: false,
 					chain: [],
 					details: null,
@@ -254,6 +262,10 @@ export class Router {
 		});
 	}
 
+	#update_scroll_positions() {
+		scroll_positions[this.current_history_index] = scroll_state();
+	}
+
 	/**
 	 * Returns true if `url` has the same origin and basepath as the app
 	 * @param {URL} url
@@ -401,6 +413,8 @@ export class Router {
 			});
 		}
 
+		this.#update_scroll_positions();
+
 		accepted();
 
 		if (!this.navigating) {
diff --git a/packages/kit/test/apps/basics/test/test.js b/packages/kit/test/apps/basics/test/test.js
index acfa23fd32ac..917892a2a952 100644
--- a/packages/kit/test/apps/basics/test/test.js
+++ b/packages/kit/test/apps/basics/test/test.js
@@ -253,6 +253,19 @@ test.describe('Scrolling', () => {
 		expect(await in_view('#input')).toBe(true);
 		expect(await page.locator('#input')).toBeFocused();
 	});
+
+	test('scroll positions are recovered on reloading the page', async ({ page, back, app }) => {
+		await page.goto('/anchor');
+		await page.evaluate(() => window.scrollTo(0, 1000));
+		await app.goto('/anchor/anchor');
+		await page.evaluate(() => window.scrollTo(0, 1000));
+
+		await page.reload();
+		expect(await page.evaluate(() => window.scrollY)).toBe(1000);
+
+		await back();
+		expect(await page.evaluate(() => window.scrollY)).toBe(1000);
+	});
 });
 
 test.describe.parallel('Imports', () => {