From d599f2f308825b9cd2cc08981224daa93518f1ea Mon Sep 17 00:00:00 2001 From: Nolan Lawson Date: Wed, 29 Aug 2018 19:03:12 -0700 Subject: [PATCH] run idb operations in a web worker (#517) --- package-lock.json | 39 ++-- package.json | 2 + routes/_actions/accounts.js | 18 +- routes/_actions/addInstance.js | 4 +- routes/_actions/addStatusOrNotification.js | 6 +- routes/_actions/compose.js | 4 +- routes/_actions/deleteStatuses.js | 6 +- routes/_actions/emoji.js | 9 +- routes/_actions/favorite.js | 6 +- routes/_actions/instances.js | 18 +- routes/_actions/lists.js | 9 +- routes/_actions/muteConversation.js | 4 +- routes/_actions/pin.js | 4 +- routes/_actions/pinnedStatuses.js | 9 +- routes/_actions/reblog.js | 4 +- routes/_actions/statuses.js | 14 +- routes/_actions/timeline.js | 13 +- routes/_components/timeline/Timeline.html | 9 +- routes/_database/accounts.js | 3 +- routes/_database/cache.js | 2 +- routes/_database/cleanup.js | 14 +- routes/_database/database.dev.js | 3 + routes/_database/database.js | 7 + routes/_database/databaseLifecycle.js | 5 +- routes/_database/databaseWorker.js | 187 ++++++++++++++++++ routes/_database/knownInstances.js | 17 ++ routes/_database/timelines/pagination.js | 4 +- .../_store/observers/autosuggestObservers.js | 6 +- routes/_utils/marks.js | 2 +- webpack.client.config.js | 12 ++ 30 files changed, 316 insertions(+), 124 deletions(-) create mode 100644 routes/_database/database.dev.js create mode 100644 routes/_database/database.js create mode 100644 routes/_database/databaseWorker.js create mode 100644 routes/_database/knownInstances.js diff --git a/package-lock.json b/package-lock.json index db4b753..c2f62fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2243,13 +2243,11 @@ }, "balanced-match": { "version": "1.0.0", - "bundled": true, - "optional": true + "bundled": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, - "optional": true, "requires": { "balanced-match": "1.0.0", "concat-map": "0.0.1" @@ -2262,18 +2260,15 @@ }, "code-point-at": { "version": "1.1.0", - "bundled": true, - "optional": true + "bundled": true }, "concat-map": { "version": "0.0.1", - "bundled": true, - "optional": true + "bundled": true }, "console-control-strings": { "version": "1.1.0", - "bundled": true, - "optional": true + "bundled": true }, "core-util-is": { "version": "1.0.2", @@ -2376,8 +2371,7 @@ }, "inherits": { "version": "2.0.3", - "bundled": true, - "optional": true + "bundled": true }, "ini": { "version": "1.3.5", @@ -2387,7 +2381,6 @@ "is-fullwidth-code-point": { "version": "1.0.0", "bundled": true, - "optional": true, "requires": { "number-is-nan": "1.0.1" } @@ -2400,15 +2393,13 @@ "minimatch": { "version": "3.0.4", "bundled": true, - "optional": true, "requires": { "brace-expansion": "1.1.11" } }, "minimist": { "version": "0.0.8", - "bundled": true, - "optional": true + "bundled": true }, "minipass": { "version": "2.2.4", @@ -2429,7 +2420,6 @@ "mkdirp": { "version": "0.5.1", "bundled": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -2502,8 +2492,7 @@ }, "number-is-nan": { "version": "1.0.1", - "bundled": true, - "optional": true + "bundled": true }, "object-assign": { "version": "4.1.1", @@ -2618,7 +2607,6 @@ "string-width": { "version": "1.0.2", "bundled": true, - "optional": true, "requires": { "code-point-at": "1.1.0", "is-fullwidth-code-point": "1.0.0", @@ -6697,6 +6685,11 @@ "postcss": "^6.0.1" } }, + "idb-keyval": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-3.1.0.tgz", + "integrity": "sha512-iFwFN5n00KNNnVxlOOK280SJJfXWY7pbMUOQXdIXehvvc/mGCV/6T2Ae+Pk2KwAkkATDTwfMavOiDH5lrJKWXQ==" + }, "ieee754": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.12.tgz", @@ -12594,6 +12587,14 @@ "errno": "~0.1.7" } }, + "workerize-loader": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/workerize-loader/-/workerize-loader-1.0.4.tgz", + "integrity": "sha512-HMTr/zpuZhm8dbhcK52cMYmn57uf7IJeMZJil+5lL/vC5+AO9wzxZ0FISkGVj78No7HcpaINwAWHGCYx3dnsTw==", + "requires": { + "loader-utils": "^1.1.0" + } + }, "wrap-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", diff --git a/package.json b/package.json index c4d47ad..26da081 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "form-data": "^2.3.2", "glob": "^7.1.2", "helmet": "^3.13.0", + "idb-keyval": "^3.1.0", "indexeddb-getall-shim": "^1.3.5", "intersection-observer": "^0.5.0", "lodash-es": "^4.17.10", @@ -98,6 +99,7 @@ "web-animations-js": "^2.3.1", "webpack": "^4.17.1", "webpack-bundle-analyzer": "^2.13.1", + "workerize-loader": "^1.0.4", "yargs": "^12.0.1" }, "devDependencies": { diff --git a/routes/_actions/accounts.js b/routes/_actions/accounts.js index 4bffe9e..dc2ec9e 100644 --- a/routes/_actions/accounts.js +++ b/routes/_actions/accounts.js @@ -1,18 +1,12 @@ import { getAccount } from '../_api/user' import { getRelationship } from '../_api/relationships' -import { - getAccount as getAccountFromDatabase, - setAccount as setAccountInDatabase} from '../_database/accounts' -import { - getRelationship as getRelationshipFromDatabase, - setRelationship as setRelationshipInDatabase -} from '../_database/relationships' +import { database } from '../_database/database' import { store } from '../_store/store' async function _updateAccount (accountId, instanceName, accessToken) { - let localPromise = getAccountFromDatabase(instanceName, accountId) + let localPromise = database.getAccount(instanceName, accountId) let remotePromise = getAccount(instanceName, accessToken, accountId).then(account => { - /* no await */ setAccountInDatabase(instanceName, account) + /* no await */ database.setAccount(instanceName, account) return account }) @@ -29,9 +23,9 @@ async function _updateAccount (accountId, instanceName, accessToken) { } async function _updateRelationship (accountId, instanceName, accessToken) { - let localPromise = getRelationshipFromDatabase(instanceName, accountId) + let localPromise = database.getRelationship(instanceName, accountId) let remotePromise = getRelationship(instanceName, accessToken, accountId).then(relationship => { - /* no await */ setRelationshipInDatabase(instanceName, relationship) + /* no await */ database.setRelationship(instanceName, relationship) return relationship }) try { @@ -47,7 +41,7 @@ async function _updateRelationship (accountId, instanceName, accessToken) { } export async function updateLocalRelationship (instanceName, accountId, relationship) { - await setRelationshipInDatabase(instanceName, relationship) + await database.setRelationship(instanceName, relationship) try { store.set({currentAccountRelationship: relationship}) } catch (e) { diff --git a/routes/_actions/addInstance.js b/routes/_actions/addInstance.js index f64e1d7..b7a3789 100644 --- a/routes/_actions/addInstance.js +++ b/routes/_actions/addInstance.js @@ -5,7 +5,7 @@ import { switchToTheme } from '../_utils/themeEngine' import { store } from '../_store/store' import { updateVerifyCredentialsForInstance } from './instances' import { updateCustomEmojiForInstance } from './emoji' -import { setInstanceInfo as setInstanceInfoInDatabase } from '../_database/meta' +import { database } from '../_database/database' const REDIRECT_URI = (typeof location !== 'undefined' ? location.origin : 'https://pinafore.social') + '/settings/instances/add' @@ -19,7 +19,7 @@ async function redirectToOauth () { } let registrationPromise = registerApplication(instanceNameInSearch, REDIRECT_URI) let instanceInfo = await getInstanceInfo(instanceNameInSearch) - await setInstanceInfoInDatabase(instanceNameInSearch, instanceInfo) // cache for later + await database.setInstanceInfo(instanceNameInSearch, instanceInfo) // cache for later let instanceData = await registrationPromise store.set({ currentRegisteredInstanceName: instanceNameInSearch, diff --git a/routes/_actions/addStatusOrNotification.js b/routes/_actions/addStatusOrNotification.js index 06a6564..ba12ae2 100644 --- a/routes/_actions/addStatusOrNotification.js +++ b/routes/_actions/addStatusOrNotification.js @@ -4,9 +4,7 @@ import { store } from '../_store/store' import uniqBy from 'lodash-es/uniqBy' import uniq from 'lodash-es/uniq' import isEqual from 'lodash-es/isEqual' -import { - insertTimelineItems as insertTimelineItemsInDatabase -} from '../_database/timelines/insertion' +import { database } from '../_database/database' import { runMediumPriorityTask } from '../_utils/runMediumPriorityTask' const STREAMING_THROTTLE_DELAY = 3000 @@ -29,7 +27,7 @@ async function insertUpdatesIntoTimeline (instanceName, timelineName, updates) { return } - await insertTimelineItemsInDatabase(instanceName, timelineName, updates) + await database.insertTimelineItems(instanceName, timelineName, updates) let itemIdsToAdd = store.getForTimeline(instanceName, timelineName, 'itemIdsToAdd') || [] let newItemIdsToAdd = uniq([].concat(itemIdsToAdd).concat(updates.map(_ => _.id))) diff --git a/routes/_actions/compose.js b/routes/_actions/compose.js index 82e5565..1bc69e0 100644 --- a/routes/_actions/compose.js +++ b/routes/_actions/compose.js @@ -2,13 +2,13 @@ import { store } from '../_store/store' import { toast } from '../_utils/toast' import { postStatus as postStatusToServer } from '../_api/statuses' import { addStatusOrNotification } from './addStatusOrNotification' -import { getStatus as getStatusFromDatabase } from '../_database/timelines/getStatusOrNotification' +import { database } from '../_database/database' import { emit } from '../_utils/eventBus' import { putMediaDescription } from '../_api/media' export async function insertHandleForReply (statusId) { let { currentInstance } = store.get() - let status = await getStatusFromDatabase(currentInstance, statusId) + let status = await database.getStatus(currentInstance, statusId) let { currentVerifyCredentials } = store.get() let originalStatus = status.reblog || status let accounts = [originalStatus.account].concat(originalStatus.mentions || []) diff --git a/routes/_actions/deleteStatuses.js b/routes/_actions/deleteStatuses.js index cff6bd6..0ce6eed 100644 --- a/routes/_actions/deleteStatuses.js +++ b/routes/_actions/deleteStatuses.js @@ -2,9 +2,7 @@ import { getIdsThatRebloggedThisStatus, getNotificationIdsForStatuses } from './ import { store } from '../_store/store' import { scheduleIdleTask } from '../_utils/scheduleIdleTask' import isEqual from 'lodash-es/isEqual' -import { - deleteStatusesAndNotifications as deleteStatusesAndNotificationsFromDatabase -} from '../_database/timelines/deletion' +import { database } from '../_database/database' function filterItemIdsFromTimelines (instanceName, timelineFilter, idFilter) { let keys = ['timelineItemIds', 'itemIdsToAdd'] @@ -45,7 +43,7 @@ function deleteNotificationIdsFromStore (instanceName, idsToDelete) { async function deleteStatusesAndNotifications (instanceName, statusIdsToDelete, notificationIdsToDelete) { deleteStatusIdsFromStore(instanceName, statusIdsToDelete) deleteNotificationIdsFromStore(instanceName, notificationIdsToDelete) - await deleteStatusesAndNotificationsFromDatabase(instanceName, statusIdsToDelete, notificationIdsToDelete) + await database.deleteStatusesAndNotifications(instanceName, statusIdsToDelete, notificationIdsToDelete) } async function doDeleteStatus (instanceName, statusId) { diff --git a/routes/_actions/emoji.js b/routes/_actions/emoji.js index 7877bfe..83122a0 100644 --- a/routes/_actions/emoji.js +++ b/routes/_actions/emoji.js @@ -1,16 +1,13 @@ import { cacheFirstUpdateAfter } from '../_utils/sync' -import { - getCustomEmoji as getCustomEmojiFromDatabase, - setCustomEmoji as setCustomEmojiInDatabase -} from '../_database/meta' +import { database } from '../_database/database' import { getCustomEmoji } from '../_api/emoji' import { store } from '../_store/store' export async function updateCustomEmojiForInstance (instanceName) { await cacheFirstUpdateAfter( () => getCustomEmoji(instanceName), - () => getCustomEmojiFromDatabase(instanceName), - emoji => setCustomEmojiInDatabase(instanceName, emoji), + () => database.getCustomEmoji(instanceName), + emoji => database.setCustomEmoji(instanceName, emoji), emoji => { let { customEmoji } = store.get() customEmoji[instanceName] = emoji diff --git a/routes/_actions/favorite.js b/routes/_actions/favorite.js index d667b88..581f8d8 100644 --- a/routes/_actions/favorite.js +++ b/routes/_actions/favorite.js @@ -1,9 +1,7 @@ import { favoriteStatus, unfavoriteStatus } from '../_api/favorite' import { store } from '../_store/store' import { toast } from '../_utils/toast' -import { - setStatusFavorited as setStatusFavoritedInDatabase -} from '../_database/timelines/updateStatus' +import { database } from '../_database/database' export async function setFavorited (statusId, favorited) { let { online } = store.get() @@ -18,7 +16,7 @@ export async function setFavorited (statusId, favorited) { store.setStatusFavorited(currentInstance, statusId, favorited) // optimistic update try { await networkPromise - await setStatusFavoritedInDatabase(currentInstance, statusId, favorited) + await database.setStatusFavorited(currentInstance, statusId, favorited) } catch (e) { console.error(e) toast.say(`Failed to ${favorited ? 'favorite' : 'unfavorite'}. ` + (e.message || '')) diff --git a/routes/_actions/instances.js b/routes/_actions/instances.js index 377fb77..4204425 100644 --- a/routes/_actions/instances.js +++ b/routes/_actions/instances.js @@ -5,13 +5,7 @@ import { toast } from '../_utils/toast' import { goto } from 'sapper/runtime.js' import { cacheFirstUpdateAfter } from '../_utils/sync' import { getInstanceInfo } from '../_api/instance' -import { clearDatabaseForInstance } from '../_database/clear' -import { - getInstanceVerifyCredentials as getInstanceVerifyCredentialsFromDatabase, - setInstanceVerifyCredentials as setInstanceVerifyCredentialsInDatabase, - getInstanceInfo as getInstanceInfoFromDatabase, - setInstanceInfo as setInstanceInfoInDatabase -} from '../_database/meta' +import { database } from '../_database/database' export function changeTheme (instanceName, newTheme) { let { instanceThemes } = store.get() @@ -62,7 +56,7 @@ export async function logOutOfInstance (instanceName) { store.save() toast.say(`Logged out of ${instanceName}`) switchToTheme(instanceThemes[newInstance] || 'default') - await clearDatabaseForInstance(instanceName) + await database.clearDatabaseForInstance(instanceName) goto('/settings/instances') } @@ -77,8 +71,8 @@ export async function updateVerifyCredentialsForInstance (instanceName) { let accessToken = loggedInInstances[instanceName].access_token await cacheFirstUpdateAfter( () => getVerifyCredentials(instanceName, accessToken), - () => getInstanceVerifyCredentialsFromDatabase(instanceName), - verifyCredentials => setInstanceVerifyCredentialsInDatabase(instanceName, verifyCredentials), + () => database.getInstanceVerifyCredentials(instanceName), + verifyCredentials => database.setInstanceVerifyCredentials(instanceName, verifyCredentials), verifyCredentials => setStoreVerifyCredentials(instanceName, verifyCredentials) ) } @@ -91,8 +85,8 @@ export async function updateVerifyCredentialsForCurrentInstance () { export async function updateInstanceInfo (instanceName) { await cacheFirstUpdateAfter( () => getInstanceInfo(instanceName), - () => getInstanceInfoFromDatabase(instanceName), - info => setInstanceInfoInDatabase(instanceName, info), + () => database.getInstanceInfo(instanceName), + info => database.setInstanceInfo(instanceName, info), info => { let { instanceInfos } = store.get() instanceInfos[instanceName] = info diff --git a/routes/_actions/lists.js b/routes/_actions/lists.js index df51073..753b630 100644 --- a/routes/_actions/lists.js +++ b/routes/_actions/lists.js @@ -1,18 +1,15 @@ import { store } from '../_store/store' import { getLists } from '../_api/lists' import { cacheFirstUpdateAfter } from '../_utils/sync' -import { - getLists as getListsFromDatabase, - setLists as setListsInDatabase -} from '../_database/meta' +import { database } from '../_database/database' export async function updateLists () { let { currentInstance, accessToken } = store.get() await cacheFirstUpdateAfter( () => getLists(currentInstance, accessToken), - () => getListsFromDatabase(currentInstance), - lists => setListsInDatabase(currentInstance, lists), + () => database.getLists(currentInstance), + lists => database.setLists(currentInstance, lists), lists => { let { instanceLists } = store.get() instanceLists[currentInstance] = lists diff --git a/routes/_actions/muteConversation.js b/routes/_actions/muteConversation.js index 1e5c1d4..c3ebc9e 100644 --- a/routes/_actions/muteConversation.js +++ b/routes/_actions/muteConversation.js @@ -1,7 +1,7 @@ import { store } from '../_store/store' import { muteConversation, unmuteConversation } from '../_api/muteConversation' import { toast } from '../_utils/toast' -import { setStatusMuted as setStatusMutedInDatabase } from '../_database/timelines/updateStatus' +import { database } from '../_database/database' export async function setConversationMuted (statusId, mute, toastOnSuccess) { let { currentInstance, accessToken } = store.get() @@ -11,7 +11,7 @@ export async function setConversationMuted (statusId, mute, toastOnSuccess) { } else { await unmuteConversation(currentInstance, accessToken, statusId) } - await setStatusMutedInDatabase(currentInstance, statusId, mute) + await database.setStatusMuted(currentInstance, statusId, mute) if (toastOnSuccess) { if (mute) { toast.say('Muted conversation') diff --git a/routes/_actions/pin.js b/routes/_actions/pin.js index b74a1e1..0d11a24 100644 --- a/routes/_actions/pin.js +++ b/routes/_actions/pin.js @@ -1,7 +1,7 @@ import { store } from '../_store/store' import { toast } from '../_utils/toast' import { pinStatus, unpinStatus } from '../_api/pin' -import { setStatusPinned as setStatusPinnedInDatabase } from '../_database/timelines/updateStatus' +import { database } from '../_database/database' import { emit } from '../_utils/eventBus' export async function setStatusPinnedOrUnpinned (statusId, pinned, toastOnSuccess) { @@ -20,7 +20,7 @@ export async function setStatusPinnedOrUnpinned (statusId, pinned, toastOnSucces } } store.setStatusPinned(currentInstance, statusId, pinned) - await setStatusPinnedInDatabase(currentInstance, statusId, pinned) + await database.setStatusPinned(currentInstance, statusId, pinned) emit('updatePinnedStatuses') } catch (e) { console.error(e) diff --git a/routes/_actions/pinnedStatuses.js b/routes/_actions/pinnedStatuses.js index 52d3704..bf35783 100644 --- a/routes/_actions/pinnedStatuses.js +++ b/routes/_actions/pinnedStatuses.js @@ -1,9 +1,6 @@ import { store } from '../_store/store' import { cacheFirstUpdateAfter } from '../_utils/sync' -import { - getPinnedStatuses as getPinnedStatusesFromDatabase, - insertPinnedStatuses as insertPinnedStatusesInDatabase -} from '../_database/timelines/pinnedStatuses' +import { database } from '../_database/database' import { getPinnedStatuses } from '../_api/pinnedStatuses' @@ -13,8 +10,8 @@ export async function updatePinnedStatusesForAccount (accountId) { await cacheFirstUpdateAfter( () => getPinnedStatuses(currentInstance, accessToken, accountId), - () => getPinnedStatusesFromDatabase(currentInstance, accountId), - statuses => insertPinnedStatusesInDatabase(currentInstance, accountId, statuses), + () => database.getPinnedStatuses(currentInstance, accountId), + statuses => database.insertPinnedStatuses(currentInstance, accountId, statuses), statuses => { let { pinnedStatuses } = store.get() pinnedStatuses[currentInstance] = pinnedStatuses[currentInstance] || {} diff --git a/routes/_actions/reblog.js b/routes/_actions/reblog.js index b583ef8..f092dce 100644 --- a/routes/_actions/reblog.js +++ b/routes/_actions/reblog.js @@ -1,7 +1,7 @@ import { store } from '../_store/store' import { toast } from '../_utils/toast' import { reblogStatus, unreblogStatus } from '../_api/reblog' -import { setStatusReblogged as setStatusRebloggedInDatabase } from '../_database/timelines/updateStatus' +import { database } from '../_database/database' export async function setReblogged (statusId, reblogged) { let online = store.get() @@ -16,7 +16,7 @@ export async function setReblogged (statusId, reblogged) { store.setStatusReblogged(currentInstance, statusId, reblogged) // optimistic update try { await networkPromise - await setStatusRebloggedInDatabase(currentInstance, statusId, reblogged) + await database.setStatusReblogged(currentInstance, statusId, reblogged) } catch (e) { console.error(e) toast.say(`Failed to ${reblogged ? 'boost' : 'unboost'}. ` + (e.message || '')) diff --git a/routes/_actions/statuses.js b/routes/_actions/statuses.js index 3d0cdec..de07418 100644 --- a/routes/_actions/statuses.js +++ b/routes/_actions/statuses.js @@ -1,13 +1,7 @@ -import { - getNotificationIdsForStatuses as getNotificationIdsForStatusesFromDatabase, - getReblogsForStatus as getReblogsForStatusFromDatabase -} from '../_database/timelines/lookup' -import { - getStatus as getStatusFromDatabase -} from '../_database/timelines/getStatusOrNotification' +import { database } from '../_database/database' export async function getIdThatThisStatusReblogged (instanceName, statusId) { - let status = await getStatusFromDatabase(instanceName, statusId) + let status = await database.getStatus(instanceName, statusId) return status.reblog && status.reblog.id } @@ -19,9 +13,9 @@ export async function getIdsThatTheseStatusesReblogged (instanceName, statusIds) } export async function getIdsThatRebloggedThisStatus (instanceName, statusId) { - return getReblogsForStatusFromDatabase(instanceName, statusId) + return database.getReblogsForStatus(instanceName, statusId) } export async function getNotificationIdsForStatuses (instanceName, statusIds) { - return getNotificationIdsForStatusesFromDatabase(instanceName, statusIds) + return database.getNotificationIdsForStatuses(instanceName, statusIds) } diff --git a/routes/_actions/timeline.js b/routes/_actions/timeline.js index f15d843..647acbc 100644 --- a/routes/_actions/timeline.js +++ b/routes/_actions/timeline.js @@ -5,19 +5,14 @@ import { mark, stop } from '../_utils/marks' import { concat, mergeArrays } from '../_utils/arrays' import { byItemIds } from '../_utils/sorting' import isEqual from 'lodash-es/isEqual' -import { - insertTimelineItems as insertTimelineItemsInDatabase -} from '../_database/timelines/insertion' -import { - getTimeline as getTimelineFromDatabase -} from '../_database/timelines/pagination' +import { database } from '../_database/database' import { getStatus, getStatusContext } from '../_api/statuses' import { emit } from '../_utils/eventBus' const FETCH_LIMIT = 20 async function storeFreshTimelineItemsInDatabase (instanceName, timelineName, items) { - await insertTimelineItemsInDatabase(instanceName, timelineName, items) + await database.insertTimelineItems(instanceName, timelineName, items) if (timelineName.startsWith('status/')) { // For status threads, we want to be sure to update the favorite/reblog counts even if // this is a stale "view" of the status. See 119-status-counts-update.js for @@ -45,7 +40,7 @@ async function fetchTimelineItems (instanceName, accessToken, timelineName, last let items let stale = false if (!online) { - items = await getTimelineFromDatabase(instanceName, timelineName, lastTimelineItemId, FETCH_LIMIT) + items = await database.getTimeline(instanceName, timelineName, lastTimelineItemId, FETCH_LIMIT) stale = true } else { try { @@ -54,7 +49,7 @@ async function fetchTimelineItems (instanceName, accessToken, timelineName, last } catch (e) { console.error(e) toast.say('Internet request failed. Showing offline content.') - items = await getTimelineFromDatabase(instanceName, timelineName, lastTimelineItemId, FETCH_LIMIT) + items = await database.getTimeline(instanceName, timelineName, lastTimelineItemId, FETCH_LIMIT) stale = true } } diff --git a/routes/_components/timeline/Timeline.html b/routes/_components/timeline/Timeline.html index 9db09bb..646a286 100644 --- a/routes/_components/timeline/Timeline.html +++ b/routes/_components/timeline/Timeline.html @@ -41,10 +41,7 @@ importNotificationVirtualListItem } from '../../_utils/asyncModules' import { timelines } from '../../_static/timelines' - import { - getStatus as getStatusFromDatabase, - getNotification as getNotificationFromDatabase - } from '../../_database/timelines/getStatusOrNotification' + import { database } from '../../_database/database' import { fetchTimelineItemsOnScrollToBottom, setupTimeline, @@ -101,9 +98,9 @@ timelineValue } if (timelineType === 'notifications') { - res.notification = await getNotificationFromDatabase($currentInstance, itemId) + res.notification = await database.getNotification($currentInstance, itemId) } else { - res.status = await getStatusFromDatabase($currentInstance, itemId) + res.status = await database.getStatus($currentInstance, itemId) } return res }, diff --git a/routes/_database/accounts.js b/routes/_database/accounts.js index 5f36e84..5d8e079 100644 --- a/routes/_database/accounts.js +++ b/routes/_database/accounts.js @@ -12,7 +12,8 @@ export async function setAccount (instanceName, account) { return setGenericEntityWithId(ACCOUNTS_STORE, accountsCache, instanceName, cloneForStorage(account)) } -export async function searchAccountsByUsername (instanceName, usernamePrefix, limit = 20) { +export async function searchAccountsByUsername (instanceName, usernamePrefix, limit) { + limit = limit || 20 const db = await getDatabase(instanceName) return dbPromise(db, ACCOUNTS_STORE, 'readonly', (accountsStore, callback) => { let keyRange = createAccountUsernamePrefixKeyRange(usernamePrefix.toLowerCase()) diff --git a/routes/_database/cache.js b/routes/_database/cache.js index 2f3eed2..d6950db 100644 --- a/routes/_database/cache.js +++ b/routes/_database/cache.js @@ -22,7 +22,7 @@ export const notificationsCache = { } if (process.browser && process.env.NODE_ENV !== 'production') { - window.cacheStats = { + (typeof self !== 'undefined' ? self : window).cacheStats = { statuses: statusesCache, accounts: accountsCache, relationships: relationshipsCache, diff --git a/routes/_database/cleanup.js b/routes/_database/cleanup.js index 7d2124a..a93e3c8 100644 --- a/routes/_database/cleanup.js +++ b/routes/_database/cleanup.js @@ -1,5 +1,4 @@ import { dbPromise, getDatabase } from './databaseLifecycle' -import { scheduleIdleTask } from '../_utils/scheduleIdleTask' import { ACCOUNTS_STORE, NOTIFICATION_TIMELINES_STORE, @@ -12,10 +11,10 @@ import { TIMESTAMP } from './constants' import debounce from 'lodash-es/debounce' -import { store } from '../_store/store' import { mark, stop } from '../_utils/marks' import { deleteAll } from './utils' import { createPinnedStatusKeyRange, createThreadKeyRange } from './keys' +import { getKnownInstances } from './knownInstances' const BATCH_SIZE = 20 const TIME_AGO = 7 * 24 * 60 * 60 * 1000 // one week ago @@ -135,15 +134,16 @@ async function cleanup (instanceName) { } function doCleanup (instanceName) { - scheduleIdleTask(() => cleanup(instanceName)) + // run in setTimeout because we're in a worker and there's no requestIdleCallback + setTimeout(() => cleanup(instanceName)) } -function scheduledCleanup () { +async function scheduledCleanup () { console.log('scheduledCleanup') - let { loggedInInstancesInOrder } = store.get() - for (let instance of loggedInInstancesInOrder) { + let knownInstances = await getKnownInstances() + for (let instance of knownInstances) { doCleanup(instance) } } -export const scheduleCleanup = debounce(() => scheduleIdleTask(scheduledCleanup), DELAY) +export const scheduleCleanup = debounce(scheduledCleanup, DELAY) diff --git a/routes/_database/database.dev.js b/routes/_database/database.dev.js new file mode 100644 index 0000000..0c88104 --- /dev/null +++ b/routes/_database/database.dev.js @@ -0,0 +1,3 @@ +// vanilla version +import * as database from './databaseWorker' +export { database } diff --git a/routes/_database/database.js b/routes/_database/database.js new file mode 100644 index 0000000..d07e1af --- /dev/null +++ b/routes/_database/database.js @@ -0,0 +1,7 @@ +// workerize version +let database +if (process.browser) { + const worker = require('./databaseWorker') + database = worker() +} +export { database } diff --git a/routes/_database/databaseLifecycle.js b/routes/_database/databaseLifecycle.js index c1c46da..42c74da 100644 --- a/routes/_database/databaseLifecycle.js +++ b/routes/_database/databaseLifecycle.js @@ -13,6 +13,7 @@ import { STATUS_ID, USERNAME_LOWERCASE } from './constants' +import { addKnownInstance, deleteKnownInstance } from './knownInstances' const openReqs = {} const databaseCache = {} @@ -86,6 +87,8 @@ export function getDatabase (instanceName) { } } req.onsuccess = () => resolve(req.result) + }).then(res => { + return addKnownInstance(instanceName).then(() => res) }) return databaseCache[instanceName] } @@ -118,5 +121,5 @@ export function deleteDatabase (instanceName) { let req = indexedDB.deleteDatabase(instanceName) req.onsuccess = () => resolve() req.onerror = () => reject(req.error) - }) + }).then(() => deleteKnownInstance(instanceName)) } diff --git a/routes/_database/databaseWorker.js b/routes/_database/databaseWorker.js new file mode 100644 index 0000000..d652e16 --- /dev/null +++ b/routes/_database/databaseWorker.js @@ -0,0 +1,187 @@ +import { + getAccount as _getAccount, + searchAccountsByUsername as _searchAccountsByUsername, + setAccount as _setAccount +} from './accounts' +import { + clearDatabaseForInstance as _clearDatabaseForInstance +} from './clear' +import { + getNotificationIdsForStatuses as _getNotificationIdsForStatuses, + getReblogsForStatus as _getReblogsForStatus +} from './timelines/lookup' +import { + getPinnedStatuses as _getPinnedStatuses, + insertPinnedStatuses as _insertPinnedStatuses +} from './timelines/pinnedStatuses' +import { + getNotificationTimeline as _getNotificationTimeline, + getStatusThread as _getStatusThread, + getStatusTimeline as _getStatusTimeline, + getTimeline as _getTimeline +} from './timelines/pagination' +import { + getNotification as _getNotification, + getStatus as _getStatus +} from './timelines/getStatusOrNotification' +import { + setStatusFavorited as _setStatusFavorited, + setStatusMuted as _setStatusMuted, + setStatusPinned as _setStatusPinned, + setStatusReblogged as _setStatusReblogged +} from './timelines/updateStatus' +import { + deleteStatusesAndNotifications as _deleteStatusesAndNotifications +} from './timelines/deletion' +import { + insertStatusThread as _insertStatusThread, + insertTimelineItems as _insertTimelineItems, + insertTimelineNotifications as _insertTimelineNotifications, + insertTimelineStatuses as _insertTimelineStatuses +} from './timelines/insertion' +import { + getCustomEmoji as _getCustomEmoji, + getInstanceInfo as _getInstanceInfo, + getInstanceVerifyCredentials as _getInstanceVerifyCredentials, + getLists as _getLists, + setCustomEmoji as _setCustomEmoji, + setInstanceInfo as _setInstanceInfo, + setInstanceVerifyCredentials as _setInstanceVerifyCredentials, + setLists as _setLists +} from './meta' +import { + getRelationship as _getRelationship, + setRelationship as _setRelationship +} from './relationships' + +export async function getAccount (instanceName, accountId) { + return _getAccount(instanceName, accountId) +} + +export async function setAccount (instanceName, account) { + return _setAccount(instanceName, account) +} + +export async function searchAccountsByUsername (instanceName, usernamePrefix, limit) { + return _searchAccountsByUsername(instanceName, usernamePrefix, limit) +} + +export async function clearDatabaseForInstance (instanceName) { + return _clearDatabaseForInstance(instanceName) +} + +export async function getReblogsForStatus (instanceName, id) { + return _getReblogsForStatus(instanceName, id) +} + +export async function getNotificationIdsForStatuses (instanceName, statusIds) { + return _getNotificationIdsForStatuses(instanceName, statusIds) +} + +export async function insertPinnedStatuses (instanceName, accountId, statuses) { + return _insertPinnedStatuses(instanceName, accountId, statuses) +} + +export async function getPinnedStatuses (instanceName, accountId) { + return _getPinnedStatuses(instanceName, accountId) +} + +export async function getNotificationTimeline (instanceName, timeline, maxId, limit) { + return _getNotificationTimeline(instanceName, timeline, maxId, limit) +} + +export async function getStatusTimeline (instanceName, timeline, maxId, limit) { + return _getStatusTimeline(instanceName, timeline, maxId, limit) +} + +export async function getStatusThread (instanceName, statusId) { + return _getStatusThread(instanceName, statusId) +} + +export async function getTimeline (instanceName, timeline, maxId, limit) { + return _getTimeline(instanceName, timeline, maxId, limit) +} + +export async function getStatus (instanceName, id) { + return _getStatus(instanceName, id) +} + +export async function getNotification (instanceName, id) { + return _getNotification(instanceName, id) +} + +export async function setStatusFavorited (instanceName, statusId, favorited) { + return _setStatusFavorited(instanceName, statusId, favorited) +} + +export async function setStatusReblogged (instanceName, statusId, reblogged) { + return _setStatusReblogged(instanceName, statusId, reblogged) +} + +export async function setStatusPinned (instanceName, statusId, pinned) { + return _setStatusPinned(instanceName, statusId, pinned) +} + +export async function setStatusMuted (instanceName, statusId, muted) { + return _setStatusMuted(instanceName, statusId, muted) +} + +export async function deleteStatusesAndNotifications (instanceName, statusIds, notificationIds) { + return _deleteStatusesAndNotifications(instanceName, statusIds, notificationIds) +} + +export async function insertTimelineNotifications (instanceName, timeline, notifications) { + return _insertTimelineNotifications(instanceName, timeline, notifications) +} + +export async function insertTimelineStatuses (instanceName, timeline, statuses) { + return _insertTimelineStatuses(instanceName, timeline, statuses) +} + +export async function insertStatusThread (instanceName, statusId, statuses) { + return _insertStatusThread(instanceName, statusId, statuses) +} + +export async function insertTimelineItems (instanceName, timeline, timelineItems) { + return _insertTimelineItems(instanceName, timeline, timelineItems) +} + +export async function getInstanceVerifyCredentials (instanceName) { + return _getInstanceVerifyCredentials(instanceName) +} + +export async function setInstanceVerifyCredentials (instanceName, value) { + return _setInstanceVerifyCredentials(instanceName, value) +} + +export async function getInstanceInfo (instanceName) { + return _getInstanceInfo(instanceName) +} + +export async function setInstanceInfo (instanceName, value) { + return _setInstanceInfo(instanceName, value) +} + +export async function getLists (instanceName) { + return _getLists(instanceName) +} + +export async function setLists (instanceName, value) { + return _setLists(instanceName, value) +} + +export async function getCustomEmoji (instanceName) { + return _getCustomEmoji(instanceName) +} + +export async function setCustomEmoji (instanceName, value) { + return _setCustomEmoji(instanceName, value) +} + +export async function getRelationship (instanceName, accountId) { + return _getRelationship(instanceName, accountId) +} + +export async function setRelationship (instanceName, relationship) { + return _setRelationship(instanceName, relationship) +} diff --git a/routes/_database/knownInstances.js b/routes/_database/knownInstances.js new file mode 100644 index 0000000..8c5469b --- /dev/null +++ b/routes/_database/knownInstances.js @@ -0,0 +1,17 @@ +import { set, keys, del } from 'idb-keyval' + +const PREFIX = 'known-instance-' + +export async function getKnownInstances () { + return (await keys()) + .filter(_ => _.startsWith(PREFIX)) + .map(_ => _.substring(PREFIX.length)) +} + +export async function addKnownInstance (instanceName) { + return set(PREFIX + instanceName, true) +} + +export async function deleteKnownInstance (instanceName) { + return del(PREFIX + instanceName) +} diff --git a/routes/_database/timelines/pagination.js b/routes/_database/timelines/pagination.js index ffa88c4..4036507 100644 --- a/routes/_database/timelines/pagination.js +++ b/routes/_database/timelines/pagination.js @@ -80,7 +80,9 @@ export async function getStatusThread (instanceName, statusId) { }) } -export async function getTimeline (instanceName, timeline, maxId = null, limit = 20) { +export async function getTimeline (instanceName, timeline, maxId, limit) { + maxId = maxId || null + limit = limit || 20 if (timeline === 'notifications') { return getNotificationTimeline(instanceName, timeline, maxId, limit) } else if (timeline.startsWith('status/')) { diff --git a/routes/_store/observers/autosuggestObservers.js b/routes/_store/observers/autosuggestObservers.js index a39aa0b..c213c21 100644 --- a/routes/_store/observers/autosuggestObservers.js +++ b/routes/_store/observers/autosuggestObservers.js @@ -1,6 +1,4 @@ -import { - searchAccountsByUsername as searchAccountsByUsernameInDatabase -} from '../../_database/accounts' +import { database } from '../../_database/database' const SEARCH_RESULTS_LIMIT = 4 const DATABASE_SEARCH_RESULTS_LIMIT = 30 @@ -8,7 +6,7 @@ const DATABASE_SEARCH_RESULTS_LIMIT = 30 async function searchAccounts (store, searchText) { searchText = searchText.substring(1) let { currentInstance } = store.get() - let results = await searchAccountsByUsernameInDatabase( + let results = await database.searchAccountsByUsername( currentInstance, searchText, DATABASE_SEARCH_RESULTS_LIMIT) return results.slice(0, SEARCH_RESULTS_LIMIT) } diff --git a/routes/_utils/marks.js b/routes/_utils/marks.js index fc17830..35d1dc5 100644 --- a/routes/_utils/marks.js +++ b/routes/_utils/marks.js @@ -1,6 +1,6 @@ const enabled = process.browser && performance.mark && ( process.env.NODE_ENV !== 'production' || - location.search.includes('marks=true') + (typeof location !== 'undefined' && location.search.includes('marks=true')) ) const perf = process.browser && performance diff --git a/webpack.client.config.js b/webpack.client.config.js index 74b04c1..1f81bce 100644 --- a/webpack.client.config.js +++ b/webpack.client.config.js @@ -10,6 +10,8 @@ const isDev = config.dev module.exports = { entry: config.client.entry(), + // uncomment to enable HMR within workers + // output: Object.assign(config.client.output(), { globalObject: 'this' }), output: config.client.output(), resolve: { extensions: ['.js', '.json', '.html'] @@ -45,6 +47,12 @@ module.exports = { MiniCssExtractPlugin.loader, 'css-loader' ] + }, + !isDev && { // workerize-loader makes dev mode hard (e.g. HMR) + test: /\/_database\/databaseWorker\.js$/, + use: [ + 'workerize-loader' + ] } ].filter(Boolean) }, @@ -80,6 +88,10 @@ module.exports = { paths: true }) ].concat(isDev ? [ + new webpack.NormalModuleReplacementPlugin( + /\/_database\/database\.js$/, + './database.dev.js' + ), new webpack.HotModuleReplacementPlugin({ requestTimeout: 120000 })