diff --git a/routes/_components/Timeline.html b/routes/_components/Timeline.html
index 97c499b..2390bf2 100644
--- a/routes/_components/Timeline.html
+++ b/routes/_components/Timeline.html
@@ -15,7 +15,7 @@
import StatusListItem from './StatusListItem.html'
import VirtualList from './VirtualList.html'
import { splice, push } from 'svelte-extras'
- import worker from 'workerize-loader!../_utils/database/statuses'
+ import worker from 'workerize-loader!../_utils/database/database'
import { mergeStatuses } from '../_utils/statuses'
import { mark, stop } from '../_utils/marks'
import { timelines } from '../_static/timelines'
diff --git a/routes/_utils/database/cleanup.js b/routes/_utils/database/cleanup.js
new file mode 100644
index 0000000..e6463b5
--- /dev/null
+++ b/routes/_utils/database/cleanup.js
@@ -0,0 +1,42 @@
+import keyval from "idb-keyval"
+import debounce from 'lodash/debounce'
+import { OBJECT_STORE, getDatabase } from './shared'
+
+const MAX_NUM_STORED_STATUSES = 1000
+const CLEANUP_INTERVAL = 60000
+
+async function cleanup(instanceName, timeline) {
+ const db = await getDatabase(instanceName, timeline)
+ return await new Promise((resolve, reject) => {
+ const tx = db.transaction(OBJECT_STORE, 'readwrite')
+ const store = tx.objectStore(OBJECT_STORE)
+ const index = store.index('pinafore_id_as_negative_big_int')
+
+ store.count().onsuccess = (e) => {
+ let count = e.target.result
+ let openKeyCursor = index.openKeyCursor || index.openCursor
+ openKeyCursor.call(index, null, 'prev').onsuccess = (e) => {
+ let cursor = e.target.result
+ if (--count < MAX_NUM_STORED_STATUSES || !cursor) {
+ return
+ }
+ store.delete(cursor.primaryKey).onsuccess = () => {
+ cursor.continue()
+ }
+ }
+ }
+ tx.oncomplete = () => resolve()
+ tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
+ })
+}
+
+export const cleanupOldStatuses = debounce(async () => {
+ console.log('cleanupOldStatuses')
+ let knownDbs = (await keyval.get('known_dbs')) || {}
+ let dbNames = Object.keys(knownDbs)
+ for (let dbName of dbNames) {
+ let [ instanceName, timeline ] = knownDbs[dbName]
+ await cleanup(instanceName, timeline)
+ }
+ console.log('done cleanupOldStatuses')
+}, CLEANUP_INTERVAL)
\ No newline at end of file
diff --git a/routes/_utils/database/database.js b/routes/_utils/database/database.js
new file mode 100644
index 0000000..b0f7de2
--- /dev/null
+++ b/routes/_utils/database/database.js
@@ -0,0 +1,36 @@
+import { cleanupOldStatuses } from './cleanup'
+import { OBJECT_STORE, getDatabase, doTransaction } from './shared'
+import { toReversePaddedBigInt, transformStatusForStorage } from './utils'
+
+export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) {
+ const db = await getDatabase(instanceName, timeline)
+ return await new Promise((resolve, reject) => {
+ const tx = db.transaction(OBJECT_STORE, 'readonly')
+ const store = tx.objectStore(OBJECT_STORE)
+ const index = store.index('pinafore_id_as_negative_big_int')
+ let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id)
+ let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false)
+
+ let res
+ index.getAll(query, limit).onsuccess = (e) => {
+ res = e.target.result
+ }
+
+ tx.oncomplete = () => resolve(res)
+ tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
+ })
+}
+
+export async function insertStatuses(instanceName, timeline, statuses) {
+ cleanupOldStatuses()
+ const db = await getDatabase(instanceName, timeline)
+ return await new Promise((resolve, reject) => {
+ const tx = db.transaction(OBJECT_STORE, 'readwrite')
+ const store = tx.objectStore(OBJECT_STORE)
+ for (let status of statuses) {
+ store.put(transformStatusForStorage(status))
+ }
+ tx.oncomplete = () => resolve()
+ tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
+ })
+}
\ No newline at end of file
diff --git a/routes/_utils/database/shared.js b/routes/_utils/database/shared.js
new file mode 100644
index 0000000..1e08883
--- /dev/null
+++ b/routes/_utils/database/shared.js
@@ -0,0 +1,40 @@
+import keyval from "idb-keyval"
+
+const databaseCache = {}
+export const OBJECT_STORE = 'statuses'
+
+export function createDbName(instanceName, timeline) {
+ return `${OBJECT_STORE}_${instanceName}_${timeline}`
+}
+
+export function getDatabase(instanceName, timeline) {
+ const key = `${instanceName}_${timeline}`
+ if (databaseCache[key]) {
+ return Promise.resolve(databaseCache[key])
+ }
+
+ let dbName = createDbName(instanceName, timeline)
+
+ keyval.get('known_dbs').then(knownDbs => {
+ knownDbs = knownDbs || {}
+ knownDbs[dbName] = [instanceName, timeline]
+ keyval.set('known_dbs', knownDbs)
+ })
+
+ databaseCache[key] = new Promise((resolve, reject) => {
+ let req = indexedDB.open(dbName, 1)
+ req.onerror = reject
+ req.onblocked = () => {
+ console.log('idb blocked')
+ }
+ req.onupgradeneeded = () => {
+ let db = req.result;
+ let oStore = db.createObjectStore(OBJECT_STORE, {
+ keyPath: 'id'
+ })
+ oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int')
+ }
+ req.onsuccess = () => resolve(req.result)
+ })
+ return databaseCache[key]
+}
\ No newline at end of file
diff --git a/routes/_utils/database/statuses.js b/routes/_utils/database/statuses.js
deleted file mode 100644
index 0cec0a0..0000000
--- a/routes/_utils/database/statuses.js
+++ /dev/null
@@ -1,91 +0,0 @@
-import keyval from 'idb-keyval'
-import cloneDeep from 'lodash/cloneDeep'
-import padStart from 'lodash/padStart'
-
-const STORE = 'statuses'
-const databaseCache = {}
-
-function toPaddedBigInt(id) {
- return padStart(id, 30, '0')
-}
-
-function toReversePaddedBigInt(id) {
- let bigInt = toPaddedBigInt(id)
- let res = ''
- for (let i = 0; i < bigInt.length; i++) {
- res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10)
- }
- return res
-}
-
-function transformStatusForStorage(status) {
- status = cloneDeep(status)
- status.pinafore_id_as_big_int = toPaddedBigInt(status.id)
- status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id)
- status.pinafore_stale = true
- return status
-}
-
-function getDatabase(instanceName, timeline) {
- const key = `${instanceName}_${timeline}`
- if (databaseCache[key]) {
- return Promise.resolve(databaseCache[key])
- }
-
- let objectStoreName = `${STORE}_${key}`
-
- keyval.get('known_dbs').then(knownDbs => {
- knownDbs = knownDbs || {}
- knownDbs[objectStoreName] = true
- keyval.set('known_dbs', knownDbs)
- })
-
- databaseCache[key] = new Promise((resolve, reject) => {
- let req = indexedDB.open(objectStoreName, 1)
- req.onerror = reject
- req.onblocked = () => {
- console.log('idb blocked')
- }
- req.onupgradeneeded = () => {
- let db = req.result;
- let oStore = db.createObjectStore(STORE, {
- keyPath: 'id'
- })
- oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int')
- }
- req.onsuccess = () => resolve(req.result)
- })
- return databaseCache[key]
-}
-
-export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) {
- const db = await getDatabase(instanceName, timeline)
- return await new Promise((resolve, reject) => {
- const tx = db.transaction(STORE, 'readonly')
- const store = tx.objectStore(STORE)
- const index = store.index('pinafore_id_as_negative_big_int')
- let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id)
- let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false)
-
- let res
- index.getAll(query, limit).onsuccess = (e) => {
- res = e.target.result
- }
-
- tx.oncomplete = () => resolve(res)
- tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
- })
-}
-
-export async function insertStatuses(instanceName, timeline, statuses) {
- const db = await getDatabase(instanceName, timeline)
- return await new Promise((resolve, reject) => {
- const tx = db.transaction(STORE, 'readwrite')
- const store = tx.objectStore(STORE)
- for (let status of statuses) {
- store.put(transformStatusForStorage(status))
- }
- tx.oncomplete = () => resolve()
- tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
- })
-}
\ No newline at end of file
diff --git a/routes/_utils/database/utils.js b/routes/_utils/database/utils.js
new file mode 100644
index 0000000..7fd3d21
--- /dev/null
+++ b/routes/_utils/database/utils.js
@@ -0,0 +1,22 @@
+import cloneDeep from 'lodash/cloneDeep'
+import padStart from 'lodash/padStart'
+
+export function toPaddedBigInt (id) {
+ return padStart(id, 30, '0')
+}
+
+export function toReversePaddedBigInt (id) {
+ let bigInt = toPaddedBigInt(id)
+ let res = ''
+ for (let i = 0; i < bigInt.length; i++) {
+ res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10)
+ }
+ return res
+}
+
+export function transformStatusForStorage (status) {
+ status = cloneDeep(status)
+ status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id)
+ status.pinafore_stale = true
+ return status
+}
\ No newline at end of file