refactor db usage

routes/_components/Timeline.html

@@ -15,7 +15,7 @@
   import StatusListItem from './StatusListItem.html'
   import VirtualList from './VirtualList.html'
   import { splice, push } from 'svelte-extras'
-  import worker from 'workerize-loader!../_utils/database/statuses'
+  import worker from 'workerize-loader!../_utils/database/database'
   import { mergeStatuses } from '../_utils/statuses'
   import { mark, stop } from '../_utils/marks'
   import { timelines } from '../_static/timelines'

routes/_utils/database/cleanup.js

@@ -0,0 +1,42 @@
+import keyval from "idb-keyval"
+import debounce from 'lodash/debounce'
+import { OBJECT_STORE, getDatabase } from './shared'
+
+const MAX_NUM_STORED_STATUSES = 1000
+const CLEANUP_INTERVAL = 60000
+
+async function cleanup(instanceName, timeline) {
+  const db = await getDatabase(instanceName, timeline)
+  return await new Promise((resolve, reject) => {
+    const tx = db.transaction(OBJECT_STORE, 'readwrite')
+    const store = tx.objectStore(OBJECT_STORE)
+    const index = store.index('pinafore_id_as_negative_big_int')
+
+    store.count().onsuccess = (e) => {
+      let count = e.target.result
+      let openKeyCursor = index.openKeyCursor || index.openCursor
+      openKeyCursor.call(index, null, 'prev').onsuccess = (e) => {
+        let cursor = e.target.result
+        if (--count < MAX_NUM_STORED_STATUSES || !cursor) {
+          return
+        }
+        store.delete(cursor.primaryKey).onsuccess = () => {
+          cursor.continue()
+        }
+      }
+    }
+    tx.oncomplete = () => resolve()
+    tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
+  })
+}
+
+export const cleanupOldStatuses = debounce(async () => {
+  console.log('cleanupOldStatuses')
+  let knownDbs = (await keyval.get('known_dbs')) || {}
+  let dbNames = Object.keys(knownDbs)
+  for (let dbName of dbNames) {
+    let [ instanceName, timeline ] = knownDbs[dbName]
+    await cleanup(instanceName, timeline)
+  }
+  console.log('done cleanupOldStatuses')
+}, CLEANUP_INTERVAL)

routes/_utils/database/database.js

@@ -0,0 +1,36 @@
+import { cleanupOldStatuses } from './cleanup'
+import { OBJECT_STORE, getDatabase, doTransaction } from './shared'
+import { toReversePaddedBigInt, transformStatusForStorage } from './utils'
+
+export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) {
+  const db = await getDatabase(instanceName, timeline)
+  return await new Promise((resolve, reject) => {
+    const tx = db.transaction(OBJECT_STORE, 'readonly')
+    const store = tx.objectStore(OBJECT_STORE)
+    const index = store.index('pinafore_id_as_negative_big_int')
+    let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id)
+    let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false)
+
+    let res
+    index.getAll(query, limit).onsuccess = (e) => {
+      res = e.target.result
+    }
+
+    tx.oncomplete = () => resolve(res)
+    tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
+  })
+}
+
+export async function insertStatuses(instanceName, timeline, statuses) {
+  cleanupOldStatuses()
+  const db = await getDatabase(instanceName, timeline)
+  return await new Promise((resolve, reject) => {
+    const tx = db.transaction(OBJECT_STORE, 'readwrite')
+    const store = tx.objectStore(OBJECT_STORE)
+    for (let status of statuses) {
+      store.put(transformStatusForStorage(status))
+    }
+    tx.oncomplete = () => resolve()
+    tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
+  })
+}

routes/_utils/database/shared.js

@@ -0,0 +1,40 @@
+import keyval from "idb-keyval"
+
+const databaseCache = {}
+export const OBJECT_STORE = 'statuses'
+
+export function createDbName(instanceName, timeline) {
+  return `${OBJECT_STORE}_${instanceName}_${timeline}`
+}
+
+export function getDatabase(instanceName, timeline) {
+  const key = `${instanceName}_${timeline}`
+  if (databaseCache[key]) {
+    return Promise.resolve(databaseCache[key])
+  }
+
+  let dbName = createDbName(instanceName, timeline)
+
+  keyval.get('known_dbs').then(knownDbs => {
+    knownDbs = knownDbs || {}
+    knownDbs[dbName] = [instanceName, timeline]
+    keyval.set('known_dbs', knownDbs)
+  })
+
+  databaseCache[key] = new Promise((resolve, reject) => {
+    let req = indexedDB.open(dbName, 1)
+    req.onerror = reject
+    req.onblocked = () => {
+      console.log('idb blocked')
+    }
+    req.onupgradeneeded = () => {
+      let db = req.result;
+      let oStore = db.createObjectStore(OBJECT_STORE, {
+        keyPath: 'id'
+      })
+      oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int')
+    }
+    req.onsuccess = () => resolve(req.result)
+  })
+  return databaseCache[key]
+}

routes/_utils/database/statuses.js

@@ -1,91 +0,0 @@
-import keyval from 'idb-keyval'
-import cloneDeep from 'lodash/cloneDeep'
-import padStart from 'lodash/padStart'
-
-const STORE = 'statuses'
-const databaseCache = {}
-
-function toPaddedBigInt(id) {
-  return padStart(id, 30, '0')
-}
-
-function toReversePaddedBigInt(id) {
-  let bigInt = toPaddedBigInt(id)
-  let res = ''
-  for (let i = 0; i < bigInt.length; i++) {
-    res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10)
-  }
-  return res
-}
-
-function transformStatusForStorage(status) {
-  status = cloneDeep(status)
-  status.pinafore_id_as_big_int = toPaddedBigInt(status.id)
-  status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id)
-  status.pinafore_stale = true
-  return status
-}
-
-function getDatabase(instanceName, timeline) {
-  const key = `${instanceName}_${timeline}`
-  if (databaseCache[key]) {
-    return Promise.resolve(databaseCache[key])
-  }
-
-  let objectStoreName = `${STORE}_${key}`
-
-  keyval.get('known_dbs').then(knownDbs => {
-    knownDbs = knownDbs || {}
-    knownDbs[objectStoreName] = true
-    keyval.set('known_dbs', knownDbs)
-  })
-
-  databaseCache[key] = new Promise((resolve, reject) => {
-    let req = indexedDB.open(objectStoreName, 1)
-    req.onerror = reject
-    req.onblocked = () => {
-      console.log('idb blocked')
-    }
-    req.onupgradeneeded = () => {
-      let db = req.result;
-      let oStore = db.createObjectStore(STORE, {
-        keyPath: 'id'
-      })
-      oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int')
-    }
-    req.onsuccess = () => resolve(req.result)
-  })
-  return databaseCache[key]
-}
-
-export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) {
-  const db = await getDatabase(instanceName, timeline)
-  return await new Promise((resolve, reject) => {
-    const tx = db.transaction(STORE, 'readonly')
-    const store = tx.objectStore(STORE)
-    const index = store.index('pinafore_id_as_negative_big_int')
-    let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id)
-    let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false)
-
-    let res
-    index.getAll(query, limit).onsuccess = (e) => {
-      res = e.target.result
-    }
-
-    tx.oncomplete = () => resolve(res)
-    tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
-  })
-}
-
-export async function insertStatuses(instanceName, timeline, statuses) {
-  const db = await getDatabase(instanceName, timeline)
-  return await new Promise((resolve, reject) => {
-    const tx = db.transaction(STORE, 'readwrite')
-    const store = tx.objectStore(STORE)
-    for (let status of statuses) {
-      store.put(transformStatusForStorage(status))
-    }
-    tx.oncomplete = () => resolve()
-    tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
-  })
-}

routes/_utils/database/utils.js

@@ -0,0 +1,22 @@
+import cloneDeep from 'lodash/cloneDeep'
+import padStart from 'lodash/padStart'
+
+export function toPaddedBigInt (id) {
+  return padStart(id, 30, '0')
+}
+
+export function toReversePaddedBigInt (id) {
+  let bigInt = toPaddedBigInt(id)
+  let res = ''
+  for (let i = 0; i < bigInt.length; i++) {
+    res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10)
+  }
+  return res
+}
+
+export function transformStatusForStorage (status) {
+  status = cloneDeep(status)
+  status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id)
+  status.pinafore_stale = true
+  return status
+}