From 8f0edf7e0183cb4e65a54d440e3e1f1b6ba63573 Mon Sep 17 00:00:00 2001 From: Robbie Antenesse Date: Thu, 6 Feb 2020 13:55:17 -0700 Subject: [PATCH] Make search api search BookReferences first (Untested) --- .../{books => bookData}/Inventaire.js | 21 +- .../controllers/{books => bookData}/index.js | 0 server/controllers/search/Inventaire.js | 53 +++++ .../{search.js => search/index.js} | 215 ++++++++++++------ server/routes/search.js | 23 +- 5 files changed, 214 insertions(+), 98 deletions(-) rename server/controllers/{books => bookData}/Inventaire.js (90%) rename server/controllers/{books => bookData}/index.js (100%) create mode 100644 server/controllers/search/Inventaire.js rename server/controllers/{search.js => search/index.js} (51%) diff --git a/server/controllers/books/Inventaire.js b/server/controllers/bookData/Inventaire.js similarity index 90% rename from server/controllers/books/Inventaire.js rename to server/controllers/bookData/Inventaire.js index 6d83dc8..63c4819 100644 --- a/server/controllers/books/Inventaire.js +++ b/server/controllers/bookData/Inventaire.js @@ -1,13 +1,16 @@ const fetch = require('node-fetch'); class Inventaire { - constructor(bookURI, language) { + constructor(language = 'en') { this.url = 'https://inventaire.io'; - this.uri = bookURI; this.lang = language; } - handleQuickEntity(entityObject) { + static getURL() { // Use a method instead of `get` to avoid collisions with `this.url` + return this.url; + } + + static handleQuickEntity(entityObject) { return { name: ( typeof entityObject.label !== 'undefined' @@ -42,14 +45,14 @@ class Inventaire { }; } - handleEntity(entityObject) { + static handleEntity(entityObject, language) { const hasLabels = typeof entityObject.labels !== 'undefined'; const hasDescriptions = typeof entityObject.descriptions !== 'undefined'; return { name: ( - hasLabels && typeof entityObject.labels[this.lang] !== 'undefined' - ? entityObject.labels[this.lang] + hasLabels && typeof entityObject.labels[language] !== 'undefined' + ? entityObject.labels[language] : ( hasLabels && Object.keys(entityObject.labels).length > 0 ? entityObject.labels[Object.keys(entityObject.labels)[0]] @@ -57,8 +60,8 @@ class Inventaire { ) ), description: ( - hasDescriptions && typeof entityObject.descriptions[this.lang] !== 'undefined' - ? entityObject.descriptions[this.lang] + hasDescriptions && typeof entityObject.descriptions[language] !== 'undefined' + ? entityObject.descriptions[language] : ( hasDescriptions && Object.keys(entityObject.descriptions).length > 0 ? entityObject.descriptions[Object.keys(entityObject.descriptions)[0]] @@ -100,7 +103,7 @@ class Inventaire { const bookData = await json; if (typeof bookData.entities !== 'undefined' && typeof bookData.entities[uri] !== 'undefined') { - const bookData = this.handleEntity(bookData.entities[uri], this.lang); + const bookData = Inventaire.handleEntity(bookData.entities[uri], this.lang); bookData['covers'] = await this.getCovers(); return bookData; diff --git a/server/controllers/books/index.js b/server/controllers/bookData/index.js similarity index 100% rename from server/controllers/books/index.js rename to server/controllers/bookData/index.js diff --git a/server/controllers/search/Inventaire.js b/server/controllers/search/Inventaire.js new file mode 100644 index 0000000..ac6b2a1 --- /dev/null +++ b/server/controllers/search/Inventaire.js @@ -0,0 +1,53 @@ +const fetch = require('node-fetch'); + +const Inventaire = require('../bookData/Inventaire'); + +function quickSearchInventaire(searchTerm, language) { + const request = fetch(`${Inventaire.getURL()}/api/search?types=works&search=${encodeURIComponent(searchTerm)}&lang=${encodeURIComponent(language)}&limit=10`) + request.catch(exception => { + console.error(exception); + return { + error: exception, + message: 'An error occurred when trying to reach the Inventaire API.', + } + }); + + const json = request.then(response => response.json()); + json.catch(exception => { + console.error(exception); + return { + error: exception, + message: 'An error occurred when trying read the response from Inventaire as JSON.', + } + }); + + // Map the results to the correct format. + return json.then(responseJSON => responseJSON.results.map(work => Inventaire.handleQuickEntity(work))); +} + +function searchInventaire(searchTerm, language) { + if (this.hasQuery) { + const request = fetch(`${Inventaire.getURL()}/api/entities?action=search&search=${encodeURIComponent(searchTerm)}&lang=${encodeURIComponent(language)}`) + request.catch(exception => { + console.error(exception); + return { + error: exception, + message: 'An error occurred when trying to reach the Inventaire API.', + } + }); + const json = request.then(response => response.json()); + json.catch(exception => { + console.error(exception); + return { + error: exception, + message: 'An error occurred when trying read the response from Inventaire as JSON.', + } + }); + return json.then(responseJSON => responseJSON.results.map(work => Inventaire.handleEntity(work, language))); + } +} + +module.exports = { + quickSearchInventaire, + searchInventaire, +}; \ No newline at end of file diff --git a/server/controllers/search.js b/server/controllers/search/index.js similarity index 51% rename from server/controllers/search.js rename to server/controllers/search/index.js index 26a2384..4e5f9b9 100644 --- a/server/controllers/search.js +++ b/server/controllers/search/index.js @@ -1,83 +1,158 @@ const fetch = require('node-fetch'); +const { Op, fn } = require('sequelize'); -const BooksController = require('./books'); +const BooksController = require('../bookData'); +const { quickSearchInventaire } = require('./Inventaire'); + +const defaultSearchOptions = { + searchBy: 'name', // A column name in the BookReference model, mainly 'name' or 'description' + source: 'inventaire', + language: 'en', +} class SearchController { - constructor(searchTerm, language = 'en') { - this.term = searchTerm; - this.lang = language; + constructor(sequelizeModels, searchTerm, options = defaultSearchOptions) { + this.models = sequelizeModels; + this.searchTerm = searchTerm; + this.searchBy = options.searchBy; + this.source = options.source; + this.lang = options.language; } get hasQuery() { - return typeof this.term !== 'undefined' && this.term !== ''; + return typeof this.searchTerm !== 'undefined' && this.searchTerm !== ''; } - quickSearchInventaire() { - if (this.hasQuery) { - const request = fetch(`https://inventaire.io/api/search?types=works&search=${encodeURIComponent(this.term)}&lang=${encodeURIComponent(this.lang)}&limit=10`) - request.catch(exception => { - console.error(exception); - return { - error: exception, - message: 'An error occurred when trying to reach the Inventaire API.', - } - }); - const json = request.then(response => response.json()); - json.catch(exception => { - console.error(exception); - return { - error: exception, - message: 'An error occurred when trying read the response from Inventaire as JSON.', - } - }); - return json.then(responseJSON => { - const booksController = new BooksController('inventaire', undefined, this.lang); - - return responseJSON.results.map(work => { - const bookData = booksController.Inventaire.handleQuickEntity(work); - booksController.uri = bookData.uri; // Update booksController.uri for each book when fetching community data. - const communityData = booksController.getCommunityData(5); - - return { - ...bookData, - ...communityData, - } - }); - }); - } + get includeQuery() { + return [ + { + model: this.models.Review, + where: { text: { [Op.not]: null } }, + attributes: [[fn('COUNT', 'id'), 'total']], // Get the total number of text reviews + as: 'reviews', + }, + { + model: this.models.Review, + where: { rating: { [Op.not]: null } }, + attributes: [[fn('AVG', 'rating'), 'average']], // Get the average star rating + as: 'rating', + }, + ] } - searchInventaire(searchBy = 'title') { - if (this.hasQuery) { - const request = fetch(`https://inventaire.io/api/entities?action=search&search=${encodeURIComponent(this.term)}&lang=${encodeURIComponent(this.lang)}`) - request.catch(exception => { - console.error(exception); - return { - error: exception, - message: 'An error occurred when trying to reach the Inventaire API.', - } - }); - const json = request.then(response => response.json()); - json.catch(exception => { - console.error(exception); - return { - error: exception, - message: 'An error occurred when trying read the response from Inventaire as JSON.', - } - }); - return json.then(responseJSON => { - const booksController = new BooksController('inventaire', undefined, this.lang); - return responseJSON.works.map(work => { - const bookData = booksController.Inventaire.handleEntity(work); - const communityData = booksController.getCommunityData(5); - - return { - ...bookData, - ...communityData, - } - }); - }); + get orderQuery() { + return [{ + model: this.models.Review, + attributes: [[fn('COUNT', 'id'), 'total']], // Get the total number of text reviews + }, 'total', 'DESC']; // Order references from most to least interaction + } + + async search() { + const bookReferences = await this.searchReferences(); + let searchResults; + switch (this.source) { + case 'openlibrary': { + searchResults = await this.searchOpenLibrary(this.searchBy); + break; + } + case 'inventaire': + default: { + searchResults = await quickSearchInventaire(this.searchTerm, this.lang); + break; + } } + + // Add any search results that match refs with the same URI and delete from results array + searchResults.forEach((result, i) => { + // If the result is not already in bookReferences + if (!bookReferences.some(ref => result.uri === ref.sources[this.source])) { + // Check if the URI is already in references table + const reference = await this.searchReferencesBySourceCode(this.source, result.uri); + if (reference) { + bookReferences.push(reference); + searchResults[i] = null; + } + } else { // If the result is already in references, null it out. + searchResults[i] = null; + } + }); + + return [ + ...bookReferences, + ...searchResults.filter(result => result !== null), + ]; + } + + async searchReferences() { + const { BookReference, Review } = this.models; + + // const includeQuery = [{ + // model: Review, + // include: [ + // { + // model: Reaction.scope('Review'), + // group: ['reactionType'], + // attributes: ['reactionType', [fn('COUNT', 'reactionType'), 'count']] + // }, + // ], + // order: [{ + // model: Reaction.scope('Review'), + // attributes: [[fn('COUNT', 'id'), 'total']], + // limit: 1, + // }, 'total', 'DESC'], + // limit: 5, + // }]; + + const exact = await BookReference.findAll({ + where: { + [Op.and]: [ // All of the contained cases are true + { + [this.searchBy]: this.searchTerm, // searchBy is exactly searchTerm + }, + { + locale: this.lang, + }, + ] + }, + include: includeQuery(Review), + order: orderQuery(Review), + }); + + if (exact.length > 0) { + return exact; + } + + // If no exact matches are found, return any approximate ones. + return await BookReference.findAll({ + where: { + [Op.and]: [ // All of the contained cases are true + { + [this.searchBy]: { // `name` or `description` + [Op.substring]: this.searchTerm, // LIKE '%searchTerm%' + }, + }, + { + locale: this.lang, + }, + ] + }, + include: this.includeQuery, + order: this.orderQuery, + }); + } + + async searchReferencesBySourceCode(source, sourceId) { + const sourceJSONKey = `"${source}"`; // Enable searching withing JSON column. + return await this.models.BookReference.findOne({ + where: { + source: { + [sourceJSONKey]: { // Where the object key is the source + [Op.eq]: sourceId, + }, + }, + }, + include: this.includeQuery, + }); } /** @@ -136,7 +211,7 @@ class SearchController { const query = this.mediaWikiQuery('https://en.wikibooks.org/w/api.php', { action: 'query', list: 'search', - srsearch: this.term, + srsearch: this.searchTerm, srprop: '', }); query(response => { @@ -173,7 +248,7 @@ class SearchController { return []; } - return fetch(`https://openlibrary.org/search.json?${searchBy}=${encodeURIComponent(this.term)}`) + return fetch(`https://openlibrary.org/search.json?${searchBy}=${encodeURIComponent(this.searchTerm)}`) .then(res => res.json()) .then(response => { if (!response.hasOwnProperty('docs')) { diff --git a/server/routes/search.js b/server/routes/search.js index 89297fd..c9280df 100644 --- a/server/routes/search.js +++ b/server/routes/search.js @@ -5,31 +5,16 @@ async function routes(fastify, options) { const searchTerm = typeof request.query.for !== 'undefined' ? request.query.for.trim() : ''; const searchBy = typeof request.query.by !== 'undefined' ? request.query.by.trim() : 'title'; const language = typeof request.query.lang !== 'undefined' ? request.query.lang.trim().split('-')[0] : undefined; // Get base language in cases like 'en-US' - const searchSource = typeof request.query.source !== 'undefined' ? request.query.source.trim() : undefined; // Get base language in cases like 'en-US' - const search = new SearchController(searchTerm, language); + const source = typeof request.query.source !== 'undefined' ? request.query.source.trim() : undefined; // Get base language in cases like 'en-US' + const controller = new SearchController(fastify.models, searchTerm, { searchBy, source, language }); - switch (searchSource) { - case 'openLibrary': { - return await search.searchOpenLibrary(searchBy); - } - case 'bookBrainz': { - return await search.searchOpenLibrary(searchBy); - } - case 'inventaire': - default: { - if (searchBy === 'title') { - return await search.quickSearchInventaire(); - } else { - return await search.searchInventaire(searchBy); - } - } - } + return await controller.search(); }); fastify.get('/api/search/cover', async (request, reply) => { const inventaireURI = typeof request.query.uri !== 'undefined' ? request.query.uri.trim() : false; const language = typeof request.query.lang !== 'undefined' ? request.query.lang.trim().split('-')[0] : undefined; // Get base language in cases like 'en-US' - const search = new SearchController(fastify.siteConfig.inventaireDomain, null, language); + const search = new SearchController(fastify.models, null, language); return await search.getInventaireCovers(inventaireURI); });