optimize indexedDB storage

routes/_database/timelines.js

@@ -1,33 +1,72 @@
 import { toReversePaddedBigInt } from './utils'
 import { dbPromise, getDatabase } from './databaseLifecycle'
-import { accountsCache, notificationsCache, setInCache, statusesCache } from './cache'
+import { accountsCache, getInCache, hasInCache, notificationsCache, setInCache, statusesCache } from './cache'
 import {
-  ACCOUNTS_STORE, NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE, STATUS_TIMELINES_STORE,
+  ACCOUNTS_STORE,
+  NOTIFICATION_TIMELINES_STORE,
+  NOTIFICATIONS_STORE,
+  STATUS_TIMELINES_STORE,
   STATUSES_STORE
 } from './constants'
-import { getGenericEntityWithId } from './helpers'
+
+const TIMESTAMP = '__pinafore_ts'
+const ACCOUNT_ID = '__pinafore_acct_id'
+const STATUS_ID = '__pinafore_status_id'
+const REBLOG_ID = '__pinafore_reblog_id'
 
 function createKeyRange (timeline, maxId) {
   let negBigInt = maxId && toReversePaddedBigInt(maxId)
   let start = negBigInt ? (timeline + '\u0000' + negBigInt) : (timeline + '\u0000')
   let end = timeline + '\u0000\uffff'
-  return IDBKeyRange.bound(start, end, false, false)
+  return IDBKeyRange.bound(start, end, true, true)
 }
 
+function cloneForStorage(obj) {
+  let res = {}
+  let keys = Object.keys(obj)
+  for (let key of keys) {
+    let value = obj[key]
+    // save storage space by skipping nulls, 0s, falses, empty strings, and empty arrays
+    if (!value || (Array.isArray(value) && value.length === 0)) {
+      continue
+    }
+    switch (key) {
+      case 'account':
+        res[ACCOUNT_ID] = value.id
+        break
+      case 'status':
+        res[STATUS_ID] = value.id
+        break
+      case 'reblog':
+        res[REBLOG_ID] = value.id
+        break
+      default:
+        res[key] = value
+        break
+    }
+  }
+  res[TIMESTAMP] = Date.now()
+  return res
+}
+
+//
+// pagination
+//
+
 async function getNotificationTimeline (instanceName, timeline, maxId, limit) {
-  let storeNames = [NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE]
+  let storeNames = [NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE, STATUSES_STORE, ACCOUNTS_STORE]
   const db = await getDatabase(instanceName)
   return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
-    let [ timelineStore, notificationsStore ] = stores
+    let [ timelineStore, notificationsStore, statusesStore, accountsStore ] = stores
     let keyRange = createKeyRange(timeline, maxId)
 
     timelineStore.getAll(keyRange, limit).onsuccess = e => {
       let timelineResults = e.target.result
       let res = new Array(timelineResults.length)
       timelineResults.forEach((timelineResult, i) => {
-        notificationsStore.get(timelineResult.notificationId).onsuccess = e => {
+        fetchNotification(notificationsStore, statusesStore, accountsStore, timelineResult.notificationId, notification => {
-          res[i] = e.target.result
+          res[i] = notification
-        }
+        })
       })
       callback(res)
     }
@@ -35,19 +74,19 @@ async function getNotificationTimeline (instanceName, timeline, maxId, limit) {
 }
 
 async function getStatusTimeline (instanceName, timeline, maxId, limit) {
-  let storeNames = [STATUS_TIMELINES_STORE, STATUSES_STORE]
+  let storeNames = [STATUS_TIMELINES_STORE, STATUSES_STORE, ACCOUNTS_STORE]
   const db = await getDatabase(instanceName)
   return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
-    let [ timelineStore, statusesStore ] = stores
+    let [ timelineStore, statusesStore, accountsStore ] = stores
     let keyRange = createKeyRange(timeline, maxId)
 
     timelineStore.getAll(keyRange, limit).onsuccess = e => {
       let timelineResults = e.target.result
       let res = new Array(timelineResults.length)
       timelineResults.forEach((timelineResult, i) => {
-        statusesStore.get(timelineResult.statusId).onsuccess = e => {
+        fetchStatus(statusesStore, accountsStore, timelineResult.statusId, status => {
-          res[i] = e.target.result
+          res[i] = status
-        }
+        })
       })
       callback(res)
     }
@@ -60,33 +99,104 @@ export async function getTimeline (instanceName, timeline, maxId = null, limit =
     : getStatusTimeline(instanceName, timeline, maxId, limit)
 }
 
+//
+// insertion
+//
+
+function putStatus(statusesStore, status) {
+  statusesStore.put(cloneForStorage(status))
+}
+
+function putAccount(accountsStore, account) {
+  accountsStore.put(cloneForStorage(account))
+}
+
+function putNotification(notificationsStore, notification) {
+  notificationsStore.put(cloneForStorage(notification))
+}
+
+function storeAccount(accountsStore, account) {
+  putAccount(accountsStore, account)
+}
+
+function storeStatus(statusesStore, accountsStore, status) {
+  putStatus(statusesStore, status)
+  putAccount(accountsStore, status.account)
+  if (status.reblog) {
+    putStatus(statusesStore, status.reblog)
+    putAccount(accountsStore, status.reblog.account)
+  }
+}
+
+function storeNotification(notificationsStore, statusesStore, accountsStore, notification) {
+  if (notification.status) {
+    storeStatus(statusesStore, accountsStore, notification.status)
+  }
+  storeAccount(accountsStore, notification.account)
+  putNotification(notificationsStore, notification)
+}
+
+function fetchAccount(accountsStore, id, callback) {
+  accountsStore.get(id).onsuccess = e => {
+    callback(e.target.result)
+  }
+}
+
+function fetchStatus(statusesStore, accountsStore, id, callback) {
+  statusesStore.get(id).onsuccess = e => {
+    let status = e.target.result
+    callback(status)
+    fetchAccount(accountsStore, status[ACCOUNT_ID], account => {
+      status.account = account
+    })
+    if (status[REBLOG_ID]) {
+      fetchStatus(statusesStore, accountsStore, status[REBLOG_ID], reblog => {
+        status.reblog = reblog
+      })
+    }
+  }
+}
+
+function fetchNotification(notificationsStore, statusesStore, accountsStore, id, callback) {
+  notificationsStore.get(id).onsuccess = e => {
+    let notification = e.target.result
+    callback(notification)
+    fetchAccount(accountsStore, notification[ACCOUNT_ID], account => {
+      notification.account = account
+    })
+    if (notification[STATUS_ID]) {
+      fetchStatus(statusesStore, accountsStore, notification[STATUS_ID], status => {
+        notification.status = status
+      })
+    }
+  }
+}
+
 function createTimelineId (timeline, id) {
   // reverse chronological order, prefixed by timeline
   return timeline + '\u0000' + toReversePaddedBigInt(id)
 }
 
 async function insertTimelineNotifications (instanceName, timeline, notifications) {
-  let storeNames = [NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE, ACCOUNTS_STORE]
   for (let notification of notifications) {
     setInCache(notificationsCache, instanceName, notification.id, notification)
     setInCache(accountsCache, instanceName, notification.account.id, notification.account)
   }
   const db = await getDatabase(instanceName)
+  let storeNames = [NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE, ACCOUNTS_STORE, STATUSES_STORE]
   await dbPromise(db, storeNames, 'readwrite', (stores) => {
-    let [ timelineStore, notificationsStore, accountsStore ] = stores
+    let [ timelineStore, notificationsStore, accountsStore, statusesStore ] = stores
     for (let notification of notifications) {
-      notificationsStore.put(notification)
+      storeNotification(notificationsStore, statusesStore, accountsStore, notification)
       timelineStore.put({
         id: createTimelineId(timeline, notification.id),
         notificationId: notification.id
       })
-      accountsStore.put(notification.account)
     }
   })
 }
 
 async function insertTimelineStatuses (instanceName, timeline, statuses) {
-  let storeNames = [STATUS_TIMELINES_STORE, STATUSES_STORE, ACCOUNTS_STORE]
   for (let status of statuses) {
     setInCache(statusesCache, instanceName, status.id, status)
     setInCache(accountsCache, instanceName, status.account.id, status.account)
@@ -95,18 +205,15 @@ async function insertTimelineStatuses (instanceName, timeline, statuses) {
     }
   }
   const db = await getDatabase(instanceName)
+  let storeNames = [STATUS_TIMELINES_STORE, STATUSES_STORE, ACCOUNTS_STORE]
   await dbPromise(db, storeNames, 'readwrite', (stores) => {
     let [ timelineStore, statusesStore, accountsStore ] = stores
     for (let status of statuses) {
-      statusesStore.put(status)
+      storeStatus(statusesStore, accountsStore, status)
       timelineStore.put({
         id: createTimelineId(timeline, status.id),
         statusId: status.id
       })
-      accountsStore.put(status.account)
-      if (status.reblog) {
-        accountsStore.put(status.reblog.account)
-      }
     }
   })
 }
@@ -117,10 +224,34 @@ export async function insertTimelineItems (instanceName, timeline, timelineItems
     : insertTimelineStatuses(instanceName, timeline, timelineItems)
 }
 
-export async function getStatus (instanceName, statusId) {
+//
-  return getGenericEntityWithId(STATUSES_STORE, statusesCache, instanceName, statusId)
+// get
+//
+
+export async function getStatus (instanceName, id) {
+  if (hasInCache(statusesCache, instanceName, id)) {
+    return getInCache(statusesCache, instanceName, id)
+  }
+  const db = await getDatabase(instanceName)
+  let storeNames = [STATUSES_STORE, ACCOUNTS_STORE]
+  let result = await dbPromise(db, storeNames, 'readonly', (stores, callback) => {
+    let [ statusesStore, accountsStore ] = stores
+    fetchStatus(statusesStore, accountsStore, id, callback)
+  })
+  setInCache(statusesCache, instanceName, id, result)
+  return result
 }
 
-export async function getNotification (instanceName, notificationId) {
+export async function getNotification (instanceName, id) {
-  return getGenericEntityWithId(NOTIFICATIONS_STORE, notificationsCache, instanceName, notificationId)
+  if (hasInCache(notificationsCache, instanceName, id)) {
+    return getInCache(notificationsCache, instanceName, id)
+  }
+  const db = await getDatabase(instanceName)
+  let storeNames = [NOTIFICATIONS_STORE, STATUSES_STORE, ACCOUNTS_STORE]
+  let result = await dbPromise(db, storeNames, 'readonly', (stores, callback) => {
+    let [ notificationsStore, statusesStore, accountsStore ] = stores
+    fetchNotification(notificationsStore, statusesStore, accountsStore, id, callback)
+  })
+  setInCache(notificationsCache, instanceName, id, result)
+  return result
 }