diff --git a/routes/_actions/deleteStatuses.js b/routes/_actions/deleteStatuses.js index d0cb6a4..2939ade 100644 --- a/routes/_actions/deleteStatuses.js +++ b/routes/_actions/deleteStatuses.js @@ -35,9 +35,7 @@ async function doDeleteStatus (instanceName, statusId) { let rebloggedIds = await getIdsThatRebloggedThisStatus(instanceName, statusId) let statusIdsToDelete = Array.from(new Set([statusId].concat(rebloggedIds).filter(Boolean))) let notificationIdsToDelete = new Set(await getNotificationIdsForStatuses(instanceName, statusIdsToDelete)) - await Promise.all([ - deleteStatusesAndNotifications(instanceName, statusIdsToDelete, notificationIdsToDelete) - ]) + await deleteStatusesAndNotifications(instanceName, statusIdsToDelete, notificationIdsToDelete) } export function deleteStatus (instanceName, statusId) { diff --git a/routes/_database/cleanup.js b/routes/_database/cleanup.js index af6fa43..b24c382 100644 --- a/routes/_database/cleanup.js +++ b/routes/_database/cleanup.js @@ -14,6 +14,8 @@ import { import debounce from 'lodash/debounce' import { store } from '../_store/store' import { mark, stop } from '../_utils/marks' +import { deleteAll } from './utils' +import { createPinnedStatusKeyRange, createThreadKeyRange } from './keys' const BATCH_SIZE = 20 const TIME_AGO = 14 * 24 * 60 * 60 * 1000 // two weeks ago @@ -34,18 +36,20 @@ function batchedGetAll (callGetAll, callback) { function cleanupStatuses (statusesStore, statusTimelinesStore, threadsStore, cutoff) { batchedGetAll( - () => statusesStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), + () => statusesStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), results => { - results.forEach(result => { - statusesStore.delete(result.id) - threadsStore.delete(result.id) - let req = statusTimelinesStore.index('statusId').getAllKeys(IDBKeyRange.only(result.id)) - req.onsuccess = e => { - let keys = e.target.result - keys.forEach(key => { - statusTimelinesStore.delete(key) - }) - } + results.forEach(statusId => { + statusesStore.delete(statusId) + deleteAll( + statusTimelinesStore, + statusTimelinesStore.index('statusId'), + IDBKeyRange.only(statusId) + ) + deleteAll( + threadsStore, + threadsStore, + createThreadKeyRange(statusId) + ) }) } ) @@ -53,17 +57,15 @@ function cleanupStatuses (statusesStore, statusTimelinesStore, threadsStore, cut function cleanupNotifications (notificationsStore, notificationTimelinesStore, cutoff) { batchedGetAll( - () => notificationsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), + () => notificationsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), results => { - results.forEach(result => { - notificationsStore.delete(result.id) - let req = notificationTimelinesStore.index('notificationId').getAllKeys(IDBKeyRange.only(result.id)) - req.onsuccess = e => { - let keys = e.target.result - keys.forEach(key => { - notificationTimelinesStore.delete(key) - }) - } + results.forEach(notificationId => { + notificationsStore.delete(notificationId) + deleteAll( + notificationTimelinesStore, + notificationTimelinesStore.index('notificationId'), + IDBKeyRange.only(notificationId) + ) }) } ) @@ -71,18 +73,15 @@ function cleanupNotifications (notificationsStore, notificationTimelinesStore, c function cleanupAccounts (accountsStore, pinnedStatusesStore, cutoff) { batchedGetAll( - () => accountsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), - (results) => { - results.forEach(result => { - accountsStore.delete(result.id) - let keyRange = IDBKeyRange.bound(result.id + '\u0000', result.id + '\u0000\uffff') - let req = pinnedStatusesStore.getAllKeys(keyRange) - req.onsuccess = e => { - let keys = e.target.result - keys.forEach(key => { - pinnedStatusesStore.delete(key) - }) - } + () => accountsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), + results => { + results.forEach(accountId => { + accountsStore.delete(accountId) + deleteAll( + pinnedStatusesStore, + pinnedStatusesStore, + createPinnedStatusKeyRange(accountId) + ) }) } ) @@ -90,10 +89,10 @@ function cleanupAccounts (accountsStore, pinnedStatusesStore, cutoff) { function cleanupRelationships (relationshipsStore, cutoff) { batchedGetAll( - () => relationshipsStore.index(TIMESTAMP).getAll(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), - (results) => { - results.forEach(result => { - relationshipsStore.delete(result.id) + () => relationshipsStore.index(TIMESTAMP).getAllKeys(IDBKeyRange.upperBound(cutoff), BATCH_SIZE), + results => { + results.forEach(relationshipId => { + relationshipsStore.delete(relationshipId) }) } ) diff --git a/routes/_database/constants.js b/routes/_database/constants.js index f148722..f3e0abc 100644 --- a/routes/_database/constants.js +++ b/routes/_database/constants.js @@ -1,12 +1,12 @@ -export const STATUSES_STORE = 'statuses-v1' -export const STATUS_TIMELINES_STORE = 'status_timelines-v1' -export const META_STORE = 'meta-v1' -export const ACCOUNTS_STORE = 'accounts-v1' -export const RELATIONSHIPS_STORE = 'relationships-v1' -export const NOTIFICATIONS_STORE = 'notifications-v1' -export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines-v1' -export const PINNED_STATUSES_STORE = 'pinned_statuses-v1' -export const THREADS_STORE = 'threads-v1' +export const STATUSES_STORE = 'statuses-v2' +export const STATUS_TIMELINES_STORE = 'status_timelines-v2' +export const META_STORE = 'meta-v2' +export const ACCOUNTS_STORE = 'accounts-v2' +export const RELATIONSHIPS_STORE = 'relationships-v2' +export const NOTIFICATIONS_STORE = 'notifications-v2' +export const NOTIFICATION_TIMELINES_STORE = 'notification_timelines-v2' +export const PINNED_STATUSES_STORE = 'pinned_statuses-v2' +export const THREADS_STORE = 'threads-v2' export const TIMESTAMP = '__pinafore_ts' export const ACCOUNT_ID = '__pinafore_acct_id' diff --git a/routes/_database/databaseLifecycle.js b/routes/_database/databaseLifecycle.js index 685ec1e..8fb1e44 100644 --- a/routes/_database/databaseLifecycle.js +++ b/routes/_database/databaseLifecycle.js @@ -13,10 +13,12 @@ import { STATUS_ID } from './constants' +import forEach from 'lodash/forEach' + const openReqs = {} const databaseCache = {} -const DB_VERSION = 6 +const DB_VERSION = 7 export function getDatabase (instanceName) { if (!instanceName) { @@ -35,28 +37,46 @@ export function getDatabase (instanceName) { } req.onupgradeneeded = (e) => { let db = req.result - let tx = e.currentTarget.transaction - if (e.oldVersion < 5) { - db.createObjectStore(STATUSES_STORE, {keyPath: 'id'}) - .createIndex(TIMESTAMP, TIMESTAMP) - db.createObjectStore(STATUS_TIMELINES_STORE) - .createIndex('statusId', '') - db.createObjectStore(NOTIFICATIONS_STORE, {keyPath: 'id'}) - .createIndex(TIMESTAMP, TIMESTAMP) - db.createObjectStore(NOTIFICATION_TIMELINES_STORE) - .createIndex('notificationId', '') - db.createObjectStore(ACCOUNTS_STORE, {keyPath: 'id'}) - .createIndex(TIMESTAMP, TIMESTAMP) - db.createObjectStore(RELATIONSHIPS_STORE, {keyPath: 'id'}) - .createIndex(TIMESTAMP, TIMESTAMP) - db.createObjectStore(META_STORE) - db.createObjectStore(PINNED_STATUSES_STORE) - tx.objectStore(STATUSES_STORE).createIndex(REBLOG_ID, REBLOG_ID) - tx.objectStore(NOTIFICATIONS_STORE).createIndex('statusId', 'statusId') - db.createObjectStore(THREADS_STORE) + + function createObjectStore (name, init, indexes) { + let store = init + ? db.createObjectStore(name, init) + : db.createObjectStore(name) + if (indexes) { + forEach(indexes, (indexValue, indexKey) => { + store.createIndex(indexKey, indexValue) + }) + } } - if (e.oldVersion < 6) { - tx.objectStore(NOTIFICATIONS_STORE).createIndex(STATUS_ID, STATUS_ID) + + if (e.oldVersion < 7) { + createObjectStore(STATUSES_STORE, {keyPath: 'id'}, { + [TIMESTAMP]: TIMESTAMP, + [REBLOG_ID]: REBLOG_ID + }) + createObjectStore(STATUS_TIMELINES_STORE, null, { + 'statusId': '' + }) + createObjectStore(NOTIFICATIONS_STORE, {keyPath: 'id'}, { + [TIMESTAMP]: TIMESTAMP, + [STATUS_ID]: STATUS_ID + }) + createObjectStore(NOTIFICATION_TIMELINES_STORE, null, { + 'notificationId': '' + }) + createObjectStore(ACCOUNTS_STORE, {keyPath: 'id'}, { + [TIMESTAMP]: TIMESTAMP + }) + createObjectStore(RELATIONSHIPS_STORE, {keyPath: 'id'}, { + [TIMESTAMP]: TIMESTAMP + }) + createObjectStore(THREADS_STORE, null, { + 'statusId': '' + }) + createObjectStore(PINNED_STATUSES_STORE, null, { + 'statusId': '' + }) + createObjectStore(META_STORE) } } req.onsuccess = () => resolve(req.result) diff --git a/routes/_database/keys.js b/routes/_database/keys.js new file mode 100644 index 0000000..a5ca480 --- /dev/null +++ b/routes/_database/keys.js @@ -0,0 +1,47 @@ +import { toReversePaddedBigInt, zeroPad } from '../_utils/sorting' + +// +// timelines +// + +export function createTimelineId (timeline, id) { + // reverse chronological order, prefixed by timeline + return timeline + '\u0000' + toReversePaddedBigInt(id) +} + +export function createTimelineKeyRange (timeline, maxId) { + let negBigInt = maxId && toReversePaddedBigInt(maxId) + let start = negBigInt ? (timeline + '\u0000' + negBigInt) : (timeline + '\u0000') + let end = timeline + '\u0000\uffff' + return IDBKeyRange.bound(start, end, true, true) +} + +// +// threads +// + +export function createThreadId (statusId, i) { + return statusId + '\u0000' + zeroPad(i, 5) +} + +export function createThreadKeyRange (statusId) { + return IDBKeyRange.bound( + statusId + '\u0000', + statusId + '\u0000\uffff' + ) +} + +// +// pinned statues +// + +export function createPinnedStatusId (accountId, i) { + return accountId + '\u0000' + zeroPad(i, 3) +} + +export function createPinnedStatusKeyRange (accountId) { + return IDBKeyRange.bound( + accountId + '\u0000', + accountId + '\u0000\uffff' + ) +} diff --git a/routes/_database/timelines.js b/routes/_database/timelines.js index e693fd8..df35885 100644 --- a/routes/_database/timelines.js +++ b/routes/_database/timelines.js @@ -1,4 +1,5 @@ -import { toPaddedBigInt, toReversePaddedBigInt } from '../_utils/sorting' +import difference from 'lodash/difference' +import times from 'lodash/times' import { cloneForStorage } from './helpers' import { dbPromise, getDatabase } from './databaseLifecycle' import { @@ -16,13 +17,15 @@ import { REBLOG_ID, STATUS_ID, THREADS_STORE } from './constants' - -function createTimelineKeyRange (timeline, maxId) { - let negBigInt = maxId && toReversePaddedBigInt(maxId) - let start = negBigInt ? (timeline + '\u0000' + negBigInt) : (timeline + '\u0000') - let end = timeline + '\u0000\uffff' - return IDBKeyRange.bound(start, end, true, true) -} +import { + createThreadKeyRange, + createTimelineKeyRange, + createTimelineId, + createThreadId, + createPinnedStatusKeyRange, + createPinnedStatusId +} from './keys' +import { deleteAll } from './utils' function cacheStatus (status, instanceName) { setInCache(statusesCache, instanceName, status.id, status) @@ -80,7 +83,8 @@ async function getStatusThread (instanceName, statusId) { const db = await getDatabase(instanceName) return dbPromise(db, storeNames, 'readonly', (stores, callback) => { let [ threadsStore, statusesStore, accountsStore ] = stores - threadsStore.get(statusId).onsuccess = e => { + let keyRange = createThreadKeyRange(statusId) + threadsStore.getAll(keyRange).onsuccess = e => { let thread = e.target.result let res = new Array(thread.length) thread.forEach((otherStatusId, i) => { @@ -177,11 +181,6 @@ function fetchNotification (notificationsStore, statusesStore, accountsStore, id } } -function createTimelineId (timeline, id) { - // reverse chronological order, prefixed by timeline - return timeline + '\u0000' + toReversePaddedBigInt(id) -} - async function insertTimelineNotifications (instanceName, timeline, notifications) { for (let notification of notifications) { setInCache(notificationsCache, instanceName, notification.id, notification) @@ -224,10 +223,18 @@ async function insertStatusThread (instanceName, statusId, statuses) { let storeNames = [THREADS_STORE, STATUSES_STORE, ACCOUNTS_STORE] await dbPromise(db, storeNames, 'readwrite', (stores) => { let [ threadsStore, statusesStore, accountsStore ] = stores - threadsStore.put(statuses.map(_ => _.id), statusId) - for (let status of statuses) { - storeStatus(statusesStore, accountsStore, status) + threadsStore.getAllKeys(createThreadKeyRange(statusId)).onsuccess = e => { + let existingKeys = e.target.result + let newKeys = times(statuses.length, i => createThreadId(statusId, i)) + let keysToDelete = difference(existingKeys, newKeys) + for (let key of keysToDelete) { + threadsStore.delete(key) + } } + statuses.forEach((otherStatus, i) => { + storeStatus(statusesStore, accountsStore, otherStatus) + threadsStore.put(otherStatus.id, createThreadId(statusId, i)) + }) }) } @@ -323,7 +330,8 @@ export async function deleteStatusesAndNotifications (instanceName, statusIds, n STATUS_TIMELINES_STORE, NOTIFICATIONS_STORE, NOTIFICATION_TIMELINES_STORE, - PINNED_STATUSES_STORE + PINNED_STATUSES_STORE, + THREADS_STORE ] await dbPromise(db, storeNames, 'readwrite', (stores) => { let [ @@ -331,33 +339,41 @@ export async function deleteStatusesAndNotifications (instanceName, statusIds, n statusTimelinesStore, notificationsStore, notificationTimelinesStore, - pinnedStatusesStore + pinnedStatusesStore, + threadsStore ] = stores function deleteStatus (statusId) { - pinnedStatusesStore.delete(statusId).onerror = e => { - e.preventDefault() - e.stopPropagation() - } statusesStore.delete(statusId) - let getAllReq = statusTimelinesStore.index('statusId') - .getAllKeys(IDBKeyRange.only(statusId)) - getAllReq.onsuccess = e => { - for (let result of e.target.result) { - statusTimelinesStore.delete(result) - } - } + deleteAll( + pinnedStatusesStore, + pinnedStatusesStore.index('statusId'), + IDBKeyRange.only(statusId) + ) + deleteAll( + statusTimelinesStore, + statusTimelinesStore.index('statusId'), + IDBKeyRange.only(statusId) + ) + deleteAll( + threadsStore, + threadsStore.index('statusId'), + IDBKeyRange.only(statusId) + ) + deleteAll( + threadsStore, + threadsStore, + createThreadKeyRange(statusId) + ) } function deleteNotification (notificationId) { notificationsStore.delete(notificationId) - let getAllReq = notificationTimelinesStore.index('statusId') - .getAllKeys(IDBKeyRange.only(notificationId)) - getAllReq.onsuccess = e => { - for (let result of e.target.result) { - notificationTimelinesStore.delete(result) - } - } + deleteAll( + notificationTimelinesStore, + notificationTimelinesStore.index('statusId'), + IDBKeyRange.only(notificationId) + ) } for (let statusId of statusIds) { @@ -383,7 +399,7 @@ export async function insertPinnedStatuses (instanceName, accountId, statuses) { let [ pinnedStatusesStore, statusesStore, accountsStore ] = stores statuses.forEach((status, i) => { storeStatus(statusesStore, accountsStore, status) - pinnedStatusesStore.put(status.id, accountId + '\u0000' + toPaddedBigInt(i)) + pinnedStatusesStore.put(status.id, createPinnedStatusId(accountId, i)) }) }) } @@ -393,10 +409,7 @@ export async function getPinnedStatuses (instanceName, accountId) { const db = await getDatabase(instanceName) return dbPromise(db, storeNames, 'readonly', (stores, callback) => { let [ pinnedStatusesStore, statusesStore, accountsStore ] = stores - let keyRange = IDBKeyRange.bound( - accountId + '\u0000', - accountId + '\u0000\uffff' - ) + let keyRange = createPinnedStatusKeyRange(accountId) pinnedStatusesStore.getAll(keyRange).onsuccess = e => { let pinnedResults = e.target.result let res = new Array(pinnedResults.length) @@ -410,19 +423,6 @@ export async function getPinnedStatuses (instanceName, accountId) { }) } -// -// notifications by status -// - -export async function getNotificationIdsForStatus (instanceName, statusId) { - const db = await getDatabase(instanceName) - return dbPromise(db, NOTIFICATIONS_STORE, 'readonly', (notificationStore, callback) => { - notificationStore.index(statusId).getAllKeys(IDBKeyRange.only(statusId)).onsuccess = e => { - callback(Array.from(e.target.result)) - } - }) -} - // // update statuses // diff --git a/routes/_database/utils.js b/routes/_database/utils.js new file mode 100644 index 0000000..b881a5b --- /dev/null +++ b/routes/_database/utils.js @@ -0,0 +1,7 @@ +export function deleteAll (store, index, keyRange) { + index.getAllKeys(keyRange).onsuccess = e => { + for (let result of e.target.result) { + store.delete(result) + } + } +} diff --git a/routes/_utils/sorting.js b/routes/_utils/sorting.js index 6f0289c..a2a3a83 100644 --- a/routes/_utils/sorting.js +++ b/routes/_utils/sorting.js @@ -1,7 +1,11 @@ import padStart from 'lodash/padStart' +export function zeroPad (str, toSize) { + return padStart(str, toSize, '0') +} + export function toPaddedBigInt (id) { - return padStart(id, 30, '0') + return zeroPad(id, 30) } export function toReversePaddedBigInt (id) { diff --git a/tests/serverActions.js b/tests/serverActions.js index 81e7c14..03344b0 100644 --- a/tests/serverActions.js +++ b/tests/serverActions.js @@ -20,6 +20,11 @@ export async function postAsAdmin (text) { null, null, false, null, 'public') } +export async function postReplyAsAdmin (text, inReplyTo) { + return postStatus(instanceName, users.admin.accessToken, text, + inReplyTo, null, false, null, 'public') +} + export async function deleteAsAdmin (statusId) { return deleteStatus(instanceName, users.admin.accessToken, statusId) } diff --git a/tests/spec/105-deletes.js b/tests/spec/105-deletes.js index 3cb8310..1954a1b 100644 --- a/tests/spec/105-deletes.js +++ b/tests/spec/105-deletes.js @@ -1,6 +1,9 @@ import { foobarRole } from '../roles' -import { getNthStatus } from '../utils' -import { deleteAsAdmin, postAsAdmin } from '../serverActions' +import { + clickToNotificationsAndBackHome, forceOffline, forceOnline, getNthStatus, getUrl, homeNavButton, + sleep +} from '../utils' +import { deleteAsAdmin, postAsAdmin, postReplyAsAdmin } from '../serverActions' fixture`105-deletes.js` .page`http://localhost:4002` @@ -9,7 +12,44 @@ test('deleted statuses are removed from the timeline', async t => { await t.useRole(foobarRole) .hover(getNthStatus(0)) let status = await postAsAdmin("I'm gonna delete this") + await sleep(1000) await t.expect(getNthStatus(0).innerText).contains("I'm gonna delete this") await deleteAsAdmin(status.id) + await sleep(1000) await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this") + await clickToNotificationsAndBackHome(t) + await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this") + await t.navigateTo('/notifications') + await forceOffline() + await t.click(homeNavButton) + await t.expect(getNthStatus(0).innerText).notContains("I'm gonna delete this") + await forceOnline() + await t + .navigateTo('/') + .expect(getNthStatus(0).innerText).notContains("I'm gonna delete this") +}) + +test('deleted statuses are removed from threads', async t => { + await t.useRole(foobarRole) + .hover(getNthStatus(0)) + let status = await postAsAdmin("I won't delete this") + let reply = await postReplyAsAdmin('But I will delete this', status.id) + await sleep(5000) + await t.expect(getNthStatus(0).innerText).contains('But I will delete this') + .expect(getNthStatus(1).innerText).contains("I won't delete this") + .click(getNthStatus(1)) + .expect(getUrl()).contains('/statuses') + .expect(getNthStatus(0).innerText).contains("I won't delete this") + .expect(getNthStatus(1).innerText).contains('But I will delete this') + await deleteAsAdmin(reply.id) + await sleep(1000) + await t.expect(getNthStatus(1).exists).notOk() + .expect(getNthStatus(0).innerText).contains("I won't delete this") + await t.navigateTo('/') + await forceOffline() + await t.click(getNthStatus(0)) + .expect(getUrl()).contains('/statuses') + .expect(getNthStatus(1).exists).notOk() + .expect(getNthStatus(0).innerText).contains("I won't delete this") + await forceOnline() }) diff --git a/tests/utils.js b/tests/utils.js index b54f932..0e49fb6 100644 --- a/tests/utils.js +++ b/tests/utils.js @@ -186,3 +186,10 @@ export async function scrollToStatus (t, n) { } await t.hover(getNthStatus(n)) } + +export async function clickToNotificationsAndBackHome (t) { + await t.click(notificationsNavButton) + .expect(getUrl()).eql('http://localhost:4002/notifications') + .click(homeNavButton) + .expect(getUrl()).eql('http://localhost:4002/') +}