From a05400b06f05b03e3d7e131ea1cecd8732fb5b7b Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Fri, 19 Jan 2018 09:18:14 -0800 Subject: [PATCH] refactor db usage --- routes/_components/Timeline.html | 2 +- routes/_utils/database/cleanup.js | 42 ++++++++++++++ routes/_utils/database/database.js | 36 ++++++++++++ routes/_utils/database/shared.js | 40 +++++++++++++ routes/_utils/database/statuses.js | 91 ------------------------------ routes/_utils/database/utils.js | 22 ++++++++ 6 files changed, 141 insertions(+), 92 deletions(-) create mode 100644 routes/_utils/database/cleanup.js create mode 100644 routes/_utils/database/database.js create mode 100644 routes/_utils/database/shared.js delete mode 100644 routes/_utils/database/statuses.js create mode 100644 routes/_utils/database/utils.js diff --git a/routes/_components/Timeline.html b/routes/_components/Timeline.html index 97c499b..2390bf2 100644 --- a/routes/_components/Timeline.html +++ b/routes/_components/Timeline.html @@ -15,7 +15,7 @@ import StatusListItem from './StatusListItem.html' import VirtualList from './VirtualList.html' import { splice, push } from 'svelte-extras' - import worker from 'workerize-loader!../_utils/database/statuses' + import worker from 'workerize-loader!../_utils/database/database' import { mergeStatuses } from '../_utils/statuses' import { mark, stop } from '../_utils/marks' import { timelines } from '../_static/timelines' diff --git a/routes/_utils/database/cleanup.js b/routes/_utils/database/cleanup.js new file mode 100644 index 0000000..e6463b5 --- /dev/null +++ b/routes/_utils/database/cleanup.js @@ -0,0 +1,42 @@ +import keyval from "idb-keyval" +import debounce from 'lodash/debounce' +import { OBJECT_STORE, getDatabase } from './shared' + +const MAX_NUM_STORED_STATUSES = 1000 +const CLEANUP_INTERVAL = 60000 + +async function cleanup(instanceName, timeline) { + const db = await getDatabase(instanceName, timeline) + return await new Promise((resolve, reject) => { + const tx = db.transaction(OBJECT_STORE, 'readwrite') + const store = tx.objectStore(OBJECT_STORE) + const index = store.index('pinafore_id_as_negative_big_int') + + store.count().onsuccess = (e) => { + let count = e.target.result + let openKeyCursor = index.openKeyCursor || index.openCursor + openKeyCursor.call(index, null, 'prev').onsuccess = (e) => { + let cursor = e.target.result + if (--count < MAX_NUM_STORED_STATUSES || !cursor) { + return + } + store.delete(cursor.primaryKey).onsuccess = () => { + cursor.continue() + } + } + } + tx.oncomplete = () => resolve() + tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) + }) +} + +export const cleanupOldStatuses = debounce(async () => { + console.log('cleanupOldStatuses') + let knownDbs = (await keyval.get('known_dbs')) || {} + let dbNames = Object.keys(knownDbs) + for (let dbName of dbNames) { + let [ instanceName, timeline ] = knownDbs[dbName] + await cleanup(instanceName, timeline) + } + console.log('done cleanupOldStatuses') +}, CLEANUP_INTERVAL) \ No newline at end of file diff --git a/routes/_utils/database/database.js b/routes/_utils/database/database.js new file mode 100644 index 0000000..b0f7de2 --- /dev/null +++ b/routes/_utils/database/database.js @@ -0,0 +1,36 @@ +import { cleanupOldStatuses } from './cleanup' +import { OBJECT_STORE, getDatabase, doTransaction } from './shared' +import { toReversePaddedBigInt, transformStatusForStorage } from './utils' + +export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) { + const db = await getDatabase(instanceName, timeline) + return await new Promise((resolve, reject) => { + const tx = db.transaction(OBJECT_STORE, 'readonly') + const store = tx.objectStore(OBJECT_STORE) + const index = store.index('pinafore_id_as_negative_big_int') + let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id) + let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false) + + let res + index.getAll(query, limit).onsuccess = (e) => { + res = e.target.result + } + + tx.oncomplete = () => resolve(res) + tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) + }) +} + +export async function insertStatuses(instanceName, timeline, statuses) { + cleanupOldStatuses() + const db = await getDatabase(instanceName, timeline) + return await new Promise((resolve, reject) => { + const tx = db.transaction(OBJECT_STORE, 'readwrite') + const store = tx.objectStore(OBJECT_STORE) + for (let status of statuses) { + store.put(transformStatusForStorage(status)) + } + tx.oncomplete = () => resolve() + tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) + }) +} \ No newline at end of file diff --git a/routes/_utils/database/shared.js b/routes/_utils/database/shared.js new file mode 100644 index 0000000..1e08883 --- /dev/null +++ b/routes/_utils/database/shared.js @@ -0,0 +1,40 @@ +import keyval from "idb-keyval" + +const databaseCache = {} +export const OBJECT_STORE = 'statuses' + +export function createDbName(instanceName, timeline) { + return `${OBJECT_STORE}_${instanceName}_${timeline}` +} + +export function getDatabase(instanceName, timeline) { + const key = `${instanceName}_${timeline}` + if (databaseCache[key]) { + return Promise.resolve(databaseCache[key]) + } + + let dbName = createDbName(instanceName, timeline) + + keyval.get('known_dbs').then(knownDbs => { + knownDbs = knownDbs || {} + knownDbs[dbName] = [instanceName, timeline] + keyval.set('known_dbs', knownDbs) + }) + + databaseCache[key] = new Promise((resolve, reject) => { + let req = indexedDB.open(dbName, 1) + req.onerror = reject + req.onblocked = () => { + console.log('idb blocked') + } + req.onupgradeneeded = () => { + let db = req.result; + let oStore = db.createObjectStore(OBJECT_STORE, { + keyPath: 'id' + }) + oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int') + } + req.onsuccess = () => resolve(req.result) + }) + return databaseCache[key] +} \ No newline at end of file diff --git a/routes/_utils/database/statuses.js b/routes/_utils/database/statuses.js deleted file mode 100644 index 0cec0a0..0000000 --- a/routes/_utils/database/statuses.js +++ /dev/null @@ -1,91 +0,0 @@ -import keyval from 'idb-keyval' -import cloneDeep from 'lodash/cloneDeep' -import padStart from 'lodash/padStart' - -const STORE = 'statuses' -const databaseCache = {} - -function toPaddedBigInt(id) { - return padStart(id, 30, '0') -} - -function toReversePaddedBigInt(id) { - let bigInt = toPaddedBigInt(id) - let res = '' - for (let i = 0; i < bigInt.length; i++) { - res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10) - } - return res -} - -function transformStatusForStorage(status) { - status = cloneDeep(status) - status.pinafore_id_as_big_int = toPaddedBigInt(status.id) - status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id) - status.pinafore_stale = true - return status -} - -function getDatabase(instanceName, timeline) { - const key = `${instanceName}_${timeline}` - if (databaseCache[key]) { - return Promise.resolve(databaseCache[key]) - } - - let objectStoreName = `${STORE}_${key}` - - keyval.get('known_dbs').then(knownDbs => { - knownDbs = knownDbs || {} - knownDbs[objectStoreName] = true - keyval.set('known_dbs', knownDbs) - }) - - databaseCache[key] = new Promise((resolve, reject) => { - let req = indexedDB.open(objectStoreName, 1) - req.onerror = reject - req.onblocked = () => { - console.log('idb blocked') - } - req.onupgradeneeded = () => { - let db = req.result; - let oStore = db.createObjectStore(STORE, { - keyPath: 'id' - }) - oStore.createIndex('pinafore_id_as_negative_big_int', 'pinafore_id_as_negative_big_int') - } - req.onsuccess = () => resolve(req.result) - }) - return databaseCache[key] -} - -export async function getTimeline(instanceName, timeline, max_id = null, limit = 20) { - const db = await getDatabase(instanceName, timeline) - return await new Promise((resolve, reject) => { - const tx = db.transaction(STORE, 'readonly') - const store = tx.objectStore(STORE) - const index = store.index('pinafore_id_as_negative_big_int') - let sinceAsNegativeBigInt = max_id === null ? null : toReversePaddedBigInt(max_id) - let query = sinceAsNegativeBigInt === null ? null : IDBKeyRange.lowerBound(sinceAsNegativeBigInt, false) - - let res - index.getAll(query, limit).onsuccess = (e) => { - res = e.target.result - } - - tx.oncomplete = () => resolve(res) - tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) - }) -} - -export async function insertStatuses(instanceName, timeline, statuses) { - const db = await getDatabase(instanceName, timeline) - return await new Promise((resolve, reject) => { - const tx = db.transaction(STORE, 'readwrite') - const store = tx.objectStore(STORE) - for (let status of statuses) { - store.put(transformStatusForStorage(status)) - } - tx.oncomplete = () => resolve() - tx.onerror = () => reject(tx.error.name + ' ' + tx.error.message) - }) -} \ No newline at end of file diff --git a/routes/_utils/database/utils.js b/routes/_utils/database/utils.js new file mode 100644 index 0000000..7fd3d21 --- /dev/null +++ b/routes/_utils/database/utils.js @@ -0,0 +1,22 @@ +import cloneDeep from 'lodash/cloneDeep' +import padStart from 'lodash/padStart' + +export function toPaddedBigInt (id) { + return padStart(id, 30, '0') +} + +export function toReversePaddedBigInt (id) { + let bigInt = toPaddedBigInt(id) + let res = '' + for (let i = 0; i < bigInt.length; i++) { + res += (9 - parseInt(bigInt.charAt(i), 10)).toString(10) + } + return res +} + +export function transformStatusForStorage (status) { + status = cloneDeep(status) + status.pinafore_id_as_negative_big_int = toReversePaddedBigInt(status.id) + status.pinafore_stale = true + return status +} \ No newline at end of file