copyleftie
/
pinafore
1
0
Fork 0
Code Issues Pull Requests Releases Activity

optimize indexedDB storage

This commit is contained in:
Nolan Lawson 2018-02-09 19:48:52 -08:00
parent 85ef4f7fd5
commit 4f157596eb
1 changed files with 159 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
routes/_database/timelines.js
View File

@ -1,33 +1,72 @@
import { toReversePaddedBigInt } from './utils'
import { dbPromise, getDatabase } from './databaseLifecycle'
import { accountsCache, notificationsCache, setInCache, statusesCache } from './cache'
import { accountsCache, getInCache, hasInCache, notificationsCache, setInCache, statusesCache } from './cache'
import {
ACCOUNTS_STORE, NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE, STATUS_TIMELINES_STORE,
ACCOUNTS_STORE,
NOTIFICATION_TIMELINES_STORE,
NOTIFICATIONS_STORE,
STATUS_TIMELINES_STORE,
STATUSES_STORE
} from './constants'
import { getGenericEntityWithId } from './helpers'
const TIMESTAMP = '__pinafore_ts'
const ACCOUNT_ID = '__pinafore_acct_id'
const STATUS_ID = '__pinafore_status_id'
const REBLOG_ID = '__pinafore_reblog_id'
function createKeyRange (timeline, maxId) {
let negBigInt = maxId && toReversePaddedBigInt(maxId)
let start = negBigInt ? (timeline + '\u0000' + negBigInt) : (timeline + '\u0000')
let end = timeline + '\u0000\uffff'
return IDBKeyRange.bound(start, end, false, false)
return IDBKeyRange.bound(start, end, true, true)
}
function cloneForStorage(obj) {
let res = {}
let keys = Object.keys(obj)
for (let key of keys) {
let value = obj[key]
// save storage space by skipping nulls, 0s, falses, empty strings, and empty arrays
if (!value || (Array.isArray(value) && value.length === 0)) {
continue
}
switch (key) {
case 'account':
res[ACCOUNT_ID] = value.id
break
case 'status':
res[STATUS_ID] = value.id
break
case 'reblog':
res[REBLOG_ID] = value.id
break
default:
res[key] = value
break
}
}
res[TIMESTAMP] = Date.now()
return res
}
//
// pagination
//
async function getNotificationTimeline (instanceName, timeline, maxId, limit) {
let storeNames = [NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE]
let storeNames = [NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE, STATUSES_STORE, ACCOUNTS_STORE]
const db = await getDatabase(instanceName)
return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
let [ timelineStore, notificationsStore ] = stores
let [ timelineStore, notificationsStore, statusesStore, accountsStore ] = stores
let keyRange = createKeyRange(timeline, maxId)
timelineStore.getAll(keyRange, limit).onsuccess = e => {
let timelineResults = e.target.result
let res = new Array(timelineResults.length)
timelineResults.forEach((timelineResult, i) => {
notificationsStore.get(timelineResult.notificationId).onsuccess = e => {
res[i] = e.target.result
}
fetchNotification(notificationsStore, statusesStore, accountsStore, timelineResult.notificationId, notification => {
res[i] = notification
})
})
callback(res)
}
@ -35,19 +74,19 @@ async function getNotificationTimeline (instanceName, timeline, maxId, limit) {
}
async function getStatusTimeline (instanceName, timeline, maxId, limit) {
let storeNames = [STATUS_TIMELINES_STORE, STATUSES_STORE]
let storeNames = [STATUS_TIMELINES_STORE, STATUSES_STORE, ACCOUNTS_STORE]
const db = await getDatabase(instanceName)
return dbPromise(db, storeNames, 'readonly', (stores, callback) => {
let [ timelineStore, statusesStore ] = stores
let [ timelineStore, statusesStore, accountsStore ] = stores
let keyRange = createKeyRange(timeline, maxId)
timelineStore.getAll(keyRange, limit).onsuccess = e => {
let timelineResults = e.target.result
let res = new Array(timelineResults.length)
timelineResults.forEach((timelineResult, i) => {
statusesStore.get(timelineResult.statusId).onsuccess = e => {
res[i] = e.target.result
}
fetchStatus(statusesStore, accountsStore, timelineResult.statusId, status => {
res[i] = status
})
})
callback(res)
}
@ -60,33 +99,104 @@ export async function getTimeline (instanceName, timeline, maxId = null, limit =
: getStatusTimeline(instanceName, timeline, maxId, limit)
}
//
// insertion
//
function putStatus(statusesStore, status) {
statusesStore.put(cloneForStorage(status))
}
function putAccount(accountsStore, account) {
accountsStore.put(cloneForStorage(account))
}
function putNotification(notificationsStore, notification) {
notificationsStore.put(cloneForStorage(notification))
}
function storeAccount(accountsStore, account) {
putAccount(accountsStore, account)
}
function storeStatus(statusesStore, accountsStore, status) {
putStatus(statusesStore, status)
putAccount(accountsStore, status.account)
if (status.reblog) {
putStatus(statusesStore, status.reblog)
putAccount(accountsStore, status.reblog.account)
}
}
function storeNotification(notificationsStore, statusesStore, accountsStore, notification) {
if (notification.status) {
storeStatus(statusesStore, accountsStore, notification.status)
}
storeAccount(accountsStore, notification.account)
putNotification(notificationsStore, notification)
}
function fetchAccount(accountsStore, id, callback) {
accountsStore.get(id).onsuccess = e => {
callback(e.target.result)
}
}
function fetchStatus(statusesStore, accountsStore, id, callback) {
statusesStore.get(id).onsuccess = e => {
let status = e.target.result
callback(status)
fetchAccount(accountsStore, status[ACCOUNT_ID], account => {
status.account = account
})
if (status[REBLOG_ID]) {
fetchStatus(statusesStore, accountsStore, status[REBLOG_ID], reblog => {
status.reblog = reblog
})
}
}
}
function fetchNotification(notificationsStore, statusesStore, accountsStore, id, callback) {
notificationsStore.get(id).onsuccess = e => {
let notification = e.target.result
callback(notification)
fetchAccount(accountsStore, notification[ACCOUNT_ID], account => {
notification.account = account
})
if (notification[STATUS_ID]) {
fetchStatus(statusesStore, accountsStore, notification[STATUS_ID], status => {
notification.status = status
})
}
}
}
function createTimelineId (timeline, id) {
// reverse chronological order, prefixed by timeline
return timeline + '\u0000' + toReversePaddedBigInt(id)
}
async function insertTimelineNotifications (instanceName, timeline, notifications) {
let storeNames = [NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE, ACCOUNTS_STORE]
for (let notification of notifications) {
setInCache(notificationsCache, instanceName, notification.id, notification)
setInCache(accountsCache, instanceName, notification.account.id, notification.account)
}
const db = await getDatabase(instanceName)
let storeNames = [NOTIFICATION_TIMELINES_STORE, NOTIFICATIONS_STORE, ACCOUNTS_STORE, STATUSES_STORE]
await dbPromise(db, storeNames, 'readwrite', (stores) => {
let [ timelineStore, notificationsStore, accountsStore ] = stores
let [ timelineStore, notificationsStore, accountsStore, statusesStore ] = stores
for (let notification of notifications) {
notificationsStore.put(notification)
storeNotification(notificationsStore, statusesStore, accountsStore, notification)
timelineStore.put({
id: createTimelineId(timeline, notification.id),
notificationId: notification.id
})
accountsStore.put(notification.account)
}
})
}
async function insertTimelineStatuses (instanceName, timeline, statuses) {
let storeNames = [STATUS_TIMELINES_STORE, STATUSES_STORE, ACCOUNTS_STORE]
for (let status of statuses) {
setInCache(statusesCache, instanceName, status.id, status)
setInCache(accountsCache, instanceName, status.account.id, status.account)
@ -95,18 +205,15 @@ async function insertTimelineStatuses (instanceName, timeline, statuses) {
}
}
const db = await getDatabase(instanceName)
let storeNames = [STATUS_TIMELINES_STORE, STATUSES_STORE, ACCOUNTS_STORE]
await dbPromise(db, storeNames, 'readwrite', (stores) => {
let [ timelineStore, statusesStore, accountsStore ] = stores
for (let status of statuses) {
statusesStore.put(status)
storeStatus(statusesStore, accountsStore, status)
timelineStore.put({
id: createTimelineId(timeline, status.id),
statusId: status.id
})
accountsStore.put(status.account)
if (status.reblog) {
accountsStore.put(status.reblog.account)
}
}
})
}
@ -117,10 +224,34 @@ export async function insertTimelineItems (instanceName, timeline, timelineItems
: insertTimelineStatuses(instanceName, timeline, timelineItems)
}
export async function getStatus (instanceName, statusId) {
return getGenericEntityWithId(STATUSES_STORE, statusesCache, instanceName, statusId)
//
// get
//
export async function getStatus (instanceName, id) {
if (hasInCache(statusesCache, instanceName, id)) {
return getInCache(statusesCache, instanceName, id)
}
const db = await getDatabase(instanceName)
let storeNames = [STATUSES_STORE, ACCOUNTS_STORE]
let result = await dbPromise(db, storeNames, 'readonly', (stores, callback) => {
let [ statusesStore, accountsStore ] = stores
fetchStatus(statusesStore, accountsStore, id, callback)
})
setInCache(statusesCache, instanceName, id, result)
return result
}
export async function getNotification (instanceName, notificationId) {
return getGenericEntityWithId(NOTIFICATIONS_STORE, notificationsCache, instanceName, notificationId)
export async function getNotification (instanceName, id) {
if (hasInCache(notificationsCache, instanceName, id)) {
return getInCache(notificationsCache, instanceName, id)
}
const db = await getDatabase(instanceName)
let storeNames = [NOTIFICATIONS_STORE, STATUSES_STORE, ACCOUNTS_STORE]
let result = await dbPromise(db, storeNames, 'readonly', (stores, callback) => {
let [ notificationsStore, statusesStore, accountsStore ] = stores
fetchNotification(notificationsStore, statusesStore, accountsStore, id, callback)
})
setInCache(notificationsCache, instanceName, id, result)
return result
}