refactor db usage
This commit is contained in:
parent
d659578338
commit
a05400b06f
6 changed files with 141 additions and 92 deletions
|
|
@ -15,7 +15,7 @@
|
|||
import StatusListItem from './StatusListItem.html'
|
||||
import VirtualList from './VirtualList.html'
|
||||
import { splice, push } from 'svelte-extras'
|
||||
import worker from 'workerize-loader!../_utils/database/statuses'
|
||||
import worker from 'workerize-loader!../_utils/database/database'
|
||||
import { mergeStatuses } from '../_utils/statuses'
|
||||
import { mark, stop } from '../_utils/marks'
|
||||
import { timelines } from '../_static/timelines'
|
||||
|
|
|
|||
42
routes/_utils/database/cleanup.js
Normal file
42
routes/_utils/database/cleanup.js
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import keyval from "idb-keyval"
|
||||
import debounce from 'lodash/debounce'
|
||||
import { OBJECT_STORE, getDatabase } from './shared'
|
||||
|
||||
const MAX_NUM_STORED_STATUSES = 1000
|
||||
const CLEANUP_INTERVAL = 60000
|
||||
|
||||
async function cleanup(instanceName, timeline) {
|
||||
const db = await getDatabase(instanceName, timeline)
|
||||
return await new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(OBJECT_STORE, 'readwrite')
|
||||
const store = tx.objectStore(OBJECT_STORE)
|
||||
const index = store.index('pinafore_id_as_negative_big_int')
|
||||
|
||||
store.count().onsuccess = (e) => {
|
||||
let count = e.target.result
|
||||
let openKeyCursor = index.openKeyCursor || index.openCursor
|
||||
openKeyCursor.call(index, null, 'prev').onsuccess = (e) => {
|
||||
let cursor = e.target.result
|
||||
if (--count < MAX_NUM_STORED_STATUSES || !cursor) {
|
||||
return
|
||||
}
|
||||
store.delete(cursor.primaryKey).onsuccess = () => {
|
||||
cursor.continue()
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.oncomplete = () => resolve()
|
||||
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
|
||||
})
|
||||
}
|
||||
|
||||
export const cleanupOldStatuses = debounce(async () => {
|
||||
console.log('cleanupOldStatuses')
|
||||
let knownDbs = (await keyval.get('known_dbs')) || {}
|
||||
let dbNames = Object.keys(knownDbs)
|
||||
for (let dbName of dbNames) {
|
||||
let [ instanceName, timeline ] = knownDbs[dbName]
|
||||
await cleanup(instanceName, timeline)
|
||||
}
|
||||
console.log('done cleanupOldStatuses')
|
||||
}, CLEANUP_INTERVAL)
|
||||
36
routes/_utils/database/database.js
Normal file
36
routes/_utils/database/database.js
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import { cleanupOldStatuses } from './cleanup'
|
||||
import { OBJECT_STORE, getDatabase, doTransaction } from './shared'
|
||||
import { toReversePaddedBigInt, transformStatusForStorage } from './utils'
|
||||
|
||||
export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) {
|
||||
const db = await getDatabase(instanceName, timeline)
|
||||
return await new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(OBJECT_STORE, 'readonly')
|
||||
const store = tx.objectStore(OBJECT_STORE)
|
||||
const index = store.index('pinafore_id_as_negative_big_int')
|
||||
let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id)
|
||||
let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false)
|
||||
|
||||
let res
|
||||
index.getAll(query, limit).onsuccess = (e) => {
|
||||
res = e.target.result
|
||||
}
|
||||
|
||||
tx.oncomplete = () => resolve(res)
|
||||
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
|
||||
})
|
||||
}
|
||||
|
||||
export async function insertStatuses(instanceName, timeline, statuses) {
|
||||
cleanupOldStatuses()
|
||||
const db = await getDatabase(instanceName, timeline)
|
||||
return await new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(OBJECT_STORE, 'readwrite')
|
||||
const store = tx.objectStore(OBJECT_STORE)
|
||||
for (let status of statuses) {
|
||||
store.put(transformStatusForStorage(status))
|
||||
}
|
||||
tx.oncomplete = () => resolve()
|
||||
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
|
||||
})
|
||||
}
|
||||
40
routes/_utils/database/shared.js
Normal file
40
routes/_utils/database/shared.js
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import keyval from "idb-keyval"
|
||||
|
||||
const databaseCache = {}
|
||||
export const OBJECT_STORE = 'statuses'
|
||||
|
||||
export function createDbName(instanceName, timeline) {
|
||||
return `${OBJECT_STORE}_${instanceName}_${timeline}`
|
||||
}
|
||||
|
||||
export function getDatabase(instanceName, timeline) {
|
||||
const key = `${instanceName}_${timeline}`
|
||||
if (databaseCache[key]) {
|
||||
return Promise.resolve(databaseCache[key])
|
||||
}
|
||||
|
||||
let dbName = createDbName(instanceName, timeline)
|
||||
|
||||
keyval.get('known_dbs').then(knownDbs => {
|
||||
knownDbs = knownDbs || {}
|
||||
knownDbs[dbName] = [instanceName, timeline]
|
||||
keyval.set('known_dbs', knownDbs)
|
||||
})
|
||||
|
||||
databaseCache[key] = new Promise((resolve, reject) => {
|
||||
let req = indexedDB.open(dbName, 1)
|
||||
req.onerror = reject
|
||||
req.onblocked = () => {
|
||||
console.log('idb blocked')
|
||||
}
|
||||
req.onupgradeneeded = () => {
|
||||
let db = req.result;
|
||||
let oStore = db.createObjectStore(OBJECT_STORE, {
|
||||
keyPath: 'id'
|
||||
})
|
||||
oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int')
|
||||
}
|
||||
req.onsuccess = () => resolve(req.result)
|
||||
})
|
||||
return databaseCache[key]
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
import keyval from 'idb-keyval'
|
||||
import cloneDeep from 'lodash/cloneDeep'
|
||||
import padStart from 'lodash/padStart'
|
||||
|
||||
const STORE = 'statuses'
|
||||
const databaseCache = {}
|
||||
|
||||
function toPaddedBigInt(id) {
|
||||
return padStart(id, 30, '0')
|
||||
}
|
||||
|
||||
function toReversePaddedBigInt(id) {
|
||||
let bigInt = toPaddedBigInt(id)
|
||||
let res = ''
|
||||
for (let i = 0; i < bigInt.length; i++) {
|
||||
res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
function transformStatusForStorage(status) {
|
||||
status = cloneDeep(status)
|
||||
status.pinafore_id_as_big_int = toPaddedBigInt(status.id)
|
||||
status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id)
|
||||
status.pinafore_stale = true
|
||||
return status
|
||||
}
|
||||
|
||||
function getDatabase(instanceName, timeline) {
|
||||
const key = `${instanceName}_${timeline}`
|
||||
if (databaseCache[key]) {
|
||||
return Promise.resolve(databaseCache[key])
|
||||
}
|
||||
|
||||
let objectStoreName = `${STORE}_${key}`
|
||||
|
||||
keyval.get('known_dbs').then(knownDbs => {
|
||||
knownDbs = knownDbs || {}
|
||||
knownDbs[objectStoreName] = true
|
||||
keyval.set('known_dbs', knownDbs)
|
||||
})
|
||||
|
||||
databaseCache[key] = new Promise((resolve, reject) => {
|
||||
let req = indexedDB.open(objectStoreName, 1)
|
||||
req.onerror = reject
|
||||
req.onblocked = () => {
|
||||
console.log('idb blocked')
|
||||
}
|
||||
req.onupgradeneeded = () => {
|
||||
let db = req.result;
|
||||
let oStore = db.createObjectStore(STORE, {
|
||||
keyPath: 'id'
|
||||
})
|
||||
oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int')
|
||||
}
|
||||
req.onsuccess = () => resolve(req.result)
|
||||
})
|
||||
return databaseCache[key]
|
||||
}
|
||||
|
||||
export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) {
|
||||
const db = await getDatabase(instanceName, timeline)
|
||||
return await new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(STORE, 'readonly')
|
||||
const store = tx.objectStore(STORE)
|
||||
const index = store.index('pinafore_id_as_negative_big_int')
|
||||
let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id)
|
||||
let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false)
|
||||
|
||||
let res
|
||||
index.getAll(query, limit).onsuccess = (e) => {
|
||||
res = e.target.result
|
||||
}
|
||||
|
||||
tx.oncomplete = () => resolve(res)
|
||||
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
|
||||
})
|
||||
}
|
||||
|
||||
export async function insertStatuses(instanceName, timeline, statuses) {
|
||||
const db = await getDatabase(instanceName, timeline)
|
||||
return await new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(STORE, 'readwrite')
|
||||
const store = tx.objectStore(STORE)
|
||||
for (let status of statuses) {
|
||||
store.put(transformStatusForStorage(status))
|
||||
}
|
||||
tx.oncomplete = () => resolve()
|
||||
tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message)
|
||||
})
|
||||
}
|
||||
22
routes/_utils/database/utils.js
Normal file
22
routes/_utils/database/utils.js
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import cloneDeep from 'lodash/cloneDeep'
|
||||
import padStart from 'lodash/padStart'
|
||||
|
||||
export function toPaddedBigInt (id) {
|
||||
return padStart(id, 30, '0')
|
||||
}
|
||||
|
||||
export function toReversePaddedBigInt (id) {
|
||||
let bigInt = toPaddedBigInt(id)
|
||||
let res = ''
|
||||
for (let i = 0; i < bigInt.length; i++) {
|
||||
res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
export function transformStatusForStorage (status) {
|
||||
status = cloneDeep(status)
|
||||
status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id)
|
||||
status.pinafore_stale = true
|
||||
return status
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue