Compare commits

...

6 Commits

11 changed files with 428 additions and 271 deletions

View File

@ -1,13 +1,15 @@
const fetch = require('node-fetch');
class Inventaire {
constructor(bookURI, language) {
this.url = 'https://inventaire.io';
this.uri = bookURI;
constructor(language = 'en') {
this.lang = language;
}
handleQuickEntity(entityObject) {
static get url() {
return 'https://inventaire.io';
}
static handleQuickEntity(entityObject) {
return {
name: (
typeof entityObject.label !== 'undefined'
@ -42,14 +44,14 @@ class Inventaire {
};
}
handleEntity(entityObject) {
static handleEntity(entityObject, language) {
const hasLabels = typeof entityObject.labels !== 'undefined';
const hasDescriptions = typeof entityObject.descriptions !== 'undefined';
return {
name: (
hasLabels && typeof entityObject.labels[this.lang] !== 'undefined'
? entityObject.labels[this.lang]
hasLabels && typeof entityObject.labels[language] !== 'undefined'
? entityObject.labels[language]
: (
hasLabels && Object.keys(entityObject.labels).length > 0
? entityObject.labels[Object.keys(entityObject.labels)[0]]
@ -57,8 +59,8 @@ class Inventaire {
)
),
description: (
hasDescriptions && typeof entityObject.descriptions[this.lang] !== 'undefined'
? entityObject.descriptions[this.lang]
hasDescriptions && typeof entityObject.descriptions[language] !== 'undefined'
? entityObject.descriptions[language]
: (
hasDescriptions && Object.keys(entityObject.descriptions).length > 0
? entityObject.descriptions[Object.keys(entityObject.descriptions)[0]]
@ -100,7 +102,7 @@ class Inventaire {
const bookData = await json;
if (typeof bookData.entities !== 'undefined' && typeof bookData.entities[uri] !== 'undefined') {
const bookData = this.handleEntity(bookData.entities[uri], this.lang);
const bookData = Inventaire.handleEntity(bookData.entities[uri], this.lang);
bookData['covers'] = await this.getCovers();
return bookData;

View File

@ -1,237 +0,0 @@
const fetch = require('node-fetch');
const BooksController = require('./books');
class SearchController {
constructor(searchTerm, language = 'en') {
this.term = searchTerm;
this.lang = language;
}
get hasQuery() {
return typeof this.term !== 'undefined' && this.term !== '';
}
quickSearchInventaire() {
if (this.hasQuery) {
const request = fetch(`https://inventaire.io/api/search?types=works&search=${encodeURIComponent(this.term)}&lang=${encodeURIComponent(this.lang)}&limit=10`)
request.catch(exception => {
console.error(exception);
return {
error: exception,
message: 'An error occurred when trying to reach the Inventaire API.',
}
});
const json = request.then(response => response.json());
json.catch(exception => {
console.error(exception);
return {
error: exception,
message: 'An error occurred when trying read the response from Inventaire as JSON.',
}
});
return json.then(responseJSON => {
const booksController = new BooksController('inventaire', undefined, this.lang);
return responseJSON.results.map(work => {
const bookData = booksController.Inventaire.handleQuickEntity(work);
booksController.uri = bookData.uri; // Update booksController.uri for each book when fetching community data.
const communityData = booksController.getCommunityData(5);
return {
...bookData,
...communityData,
}
});
});
}
}
searchInventaire(searchBy = 'title') {
if (this.hasQuery) {
const request = fetch(`https://inventaire.io/api/entities?action=search&search=${encodeURIComponent(this.term)}&lang=${encodeURIComponent(this.lang)}`)
request.catch(exception => {
console.error(exception);
return {
error: exception,
message: 'An error occurred when trying to reach the Inventaire API.',
}
});
const json = request.then(response => response.json());
json.catch(exception => {
console.error(exception);
return {
error: exception,
message: 'An error occurred when trying read the response from Inventaire as JSON.',
}
});
return json.then(responseJSON => {
const booksController = new BooksController('inventaire', undefined, this.lang);
return responseJSON.works.map(work => {
const bookData = booksController.Inventaire.handleEntity(work);
const communityData = booksController.getCommunityData(5);
return {
...bookData,
...communityData,
}
});
});
}
}
/**
* Query a MediaWiki api.php instance with the given options
*/
mediaWikiQuery(endpoint, options) {
/**
* Create a uniquely-named callback that will process the JSONP results
*/
var createCallback = function (k) {
var i = 1;
var callbackName;
do {
callbackName = 'searchCallback' + i;
i = i + 1;
} while (window[callbackName])
window[callbackName] = k;
return callbackName;
}
/**
* Flatten an object into a URL query string.
* For example: { foo: 'bar', baz: 42 } becomes 'foo=bar&baz=42'
*/
var queryStr = function (options) {
var query = [];
for (var i in options) {
if (options.hasOwnProperty(i)) {
query.push(encodeURIComponent(i) + '=' + encodeURIComponent(options[i]));
}
}
return query.join('&');
}
/**
* Build a function that can be applied to a callback. The callback processes
* the JSON results of the API call.
*/
return function (k) {
options.format = 'json';
options.callback = createCallback(k);
var script = document.createElement('script');
script.id = 'searchResults';
script.src = endpoint + '?' + queryStr(options);
var head = document.getElementsByTagName('head')[0];
head.appendChild(script);
};
}
async searchWikiBooks(term) {
if (!this.hasQuery) {
return [];
}
const query = this.mediaWikiQuery('https://en.wikibooks.org/w/api.php', {
action: 'query',
list: 'search',
srsearch: this.term,
srprop: '',
});
query(response => {
console.log(response);
const searchScript = document.getElementById('searchResults');
searchScript.parentNode.removeChild(searchScript);
for (let property in window) {
if (property.includes('searchCallback')) {
delete window[property];
}
}
const bookResults = [];
const pageids = response.query.search.map(item => item.pageid);
const propsQuery = this.mediaWikiQuery('https://en.wikibooks.org/w/api.php', {
action: 'query',
pageids: pageids.join('|'),
prop: 'categories|pageprops',
});
propsQuery(propsResponse => {
console.log(propsResponse);
for (let pageid in propsResponse.query.pages) {
if (propsResponse.query.pages[pageid].hasOwnProperty('categories')) {
}
}
});
return bookResults;
});
}
async searchOpenLibrary(searchBy = 'title') {
if (!this.hasQuery) {
return [];
}
return fetch(`https://openlibrary.org/search.json?${searchBy}=${encodeURIComponent(this.term)}`)
.then(res => res.json())
.then(response => {
if (!response.hasOwnProperty('docs')) {
return [];
}
const booksController = new BooksController('openLibrary', undefined, this.lang);
// Format the response into usable objects
const docs = response.docs.map(doc => {
return booksController.handleOpenLibraryEntity(doc);
});
let results = [];
// Filter out duplicate items with the same title and author
docs.forEach(doc => {
const existingDoc = results.find((filterResult) => {
return filterResult.title === doc.title && filterResult.description === doc.description;
});
if (!existingDoc) {
results.push(doc);
}
});
results = results.map(result => {
// Find any duplicates in case they have different cover data
const duplicates = docs.filter(doc => {
return doc.name.toLowerCase() === result.name.toLowerCase() && doc.description === result.description;
});
result.covers = [];
duplicates.forEach(duplicate => {
if (duplicate.coverId !== false) {
result.covers.push({
uri: duplicate.coverId,
url: `//covers.openlibrary.org/b/id/${duplicate.coverId}-M.jpg`,
});
}
});
delete result.coverId;
return result;
}).map(bookData => {
// Use bookController to get community data
booksController.uri = bookData.uri; // Update booksController.uri for each book when fetching community data.
const communityData = booksController.getCommunityData(5);
return {
...bookData,
...communityData,
};
});
return results;
}).catch(error => {
console.log(error);
return [];
});
}
}
module.exports = SearchController;

View File

@ -0,0 +1,53 @@
const fetch = require('node-fetch');
const Inventaire = require('../bookData/Inventaire');
function quickSearchInventaire(searchTerm, language) {
const request = fetch(`${Inventaire.url}/api/search?types=works&search=${encodeURIComponent(searchTerm)}&lang=${encodeURIComponent(language)}&limit=10`)
request.catch(exception => {
console.error(exception);
return {
error: exception,
message: 'An error occurred when trying to reach the Inventaire API.',
}
});
const json = request.then(response => response.json());
json.catch(exception => {
console.error(exception);
return {
error: exception,
message: 'An error occurred when trying read the response from Inventaire as JSON.',
}
});
// Map the results to the correct format.
return json.then(responseJSON => responseJSON.results.map(work => Inventaire.handleQuickEntity(work)));
}
function searchInventaire(searchTerm, language) {
if (this.hasQuery) {
const request = fetch(`${Inventaire.url}/api/entities?action=search&search=${encodeURIComponent(searchTerm)}&lang=${encodeURIComponent(language)}`)
request.catch(exception => {
console.error(exception);
return {
error: exception,
message: 'An error occurred when trying to reach the Inventaire API.',
}
});
const json = request.then(response => response.json());
json.catch(exception => {
console.error(exception);
return {
error: exception,
message: 'An error occurred when trying read the response from Inventaire as JSON.',
}
});
return json.then(responseJSON => responseJSON.results.map(work => Inventaire.handleEntity(work, language)));
}
}
module.exports = {
quickSearchInventaire,
searchInventaire,
};

View File

@ -0,0 +1,326 @@
const fetch = require('node-fetch');
const { Op, fn, col } = require('sequelize');
const BooksController = require('../bookData');
const { quickSearchInventaire } = require('./Inventaire');
const defaultSearchOptions = {
searchBy: 'name', // A column name in the BookReference model, mainly 'name' or 'description'
source: 'inventaire',
language: 'en',
}
class SearchController {
constructor(sequelizeModels, searchTerm, options = defaultSearchOptions) {
this.models = sequelizeModels;
this.searchTerm = searchTerm;
this.searchBy = options.searchBy.replace('title', 'name').replace('author', 'description');
this.source = options.source;
this.lang = options.language;
}
get hasQuery() {
return typeof this.searchTerm !== 'undefined' && this.searchTerm !== '';
}
get bookReferenceSearchAttributes() {
return {
include: [
{
as: 'Interactions',
model: this.models.Review,
attributes: ['id'],
required: false,
},
{
as: 'Reviews',
model: this.models.Review,
attributes: ['id'],
required: false,
},
{
as: 'Ratings',
model: this.models.Review,
attributes: ['rating'],
required: false,
},
], // These are all subsets of Review model specified in BookReference associations
attributes: [
[col('BookReference.id'), 'id'],
'name',
'description',
'sources',
'covers',
[fn('COUNT', col('Interactions.id')), 'totalInteractions'],
[fn('COUNT', col('Reviews.id')), 'numReviews'],
[fn('AVG', col('Ratings.rating')), 'averageRating'],
],
order: [[col('totalInteractions'), 'DESC']],
};
}
async search() {
const bookReferences = await this.searchReferences();
let searchResults;
switch (this.source) {
case 'openlibrary': {
searchResults = await this.searchOpenLibrary(this.searchBy);
break;
}
case 'inventaire':
default: {
searchResults = await quickSearchInventaire(this.searchTerm, this.lang);
break;
}
}
// Add any search results that match refs with the same URI and delete from results array
const urisToCheck = searchResults.filter(
result => !bookReferences.some(ref => result.uri === ref.sources[this.source])
).map(result => result.uri);
if (urisToCheck.length > 0) {
const foundReferences = await this.searchReferencesBySourceCodes(this.source, urisToCheck);
return [
...bookReferences,
...foundReferences,
...searchResults.filter(result => !urisToCheck.includes(result.uri)),
];
}
return [
...bookReferences,
...searchResults.filter(result => result !== null),
];
}
async searchReferences() {
const { BookReference } = this.models;
const exact = await BookReference.findAll({
where: {
[Op.and]: [ // All of the contained cases are true
{
[this.searchBy]: this.searchTerm, // searchBy is exactly searchTerm
},
{
locale: this.lang,
},
]
},
...this.bookReferenceSearchAttributes,
}).then( // Empty results give 1 empty model in an array, so filter those out
references => references.filter(ref => typeof ref.id !== 'undefined' && ref.id !== null)
);
if (exact.length > 0) {
return exact;
}
// If no exact matches are found, return any approximate ones.
return await BookReference.findAll({
where: {
[Op.and]: [ // All of the contained cases are true
{
[this.searchBy]: { // `name` or `description`
[Op.substring]: this.searchTerm, // LIKE '%searchTerm%'
},
},
{
locale: this.lang,
},
]
},
...this.bookReferenceSearchAttributes,
}).then( // Empty results give 1 empty model in an array, so filter those out
references => references.filter(ref => typeof ref.id !== 'undefined' && ref.id !== null)
);
}
async searchReferencesBySourceCode(source, sourceId) {
const sourceJSONKey = `"${source}"`; // Enable searching withing JSON column.
return await this.models.BookReference.findOne({
where: {
source: {
[sourceJSONKey]: { // Where the object key is the source
[Op.eq]: sourceId,
},
},
},
...this.bookReferenceSearchAttributes,
}).then( // Empty results give 1 empty model in an array, so filter those out
references => references.filter(ref => typeof ref.id !== 'undefined' && ref.id !== null)
);
}
async searchReferencesBySourceCodes(source, sourceIds) {
const sourceJSONKey = `"${source}"`; // Enable searching withing JSON column.
return await this.models.BookReference.findOne({
where: {
[Op.or]: sourceIds.map(sourceId => ({
source: {
[sourceJSONKey]: sourceId,
},
})),
},
...this.bookReferenceSearchAttributes,
}).then( // Empty results give 1 empty model in an array, so filter those out
references => references.filter(ref => typeof ref.id !== 'undefined' && ref.id !== null)
);
}
/**
* Query a MediaWiki api.php instance with the given options
*/
mediaWikiQuery(endpoint, options) {
/**
* Create a uniquely-named callback that will process the JSONP results
*/
var createCallback = function (k) {
var i = 1;
var callbackName;
do {
callbackName = 'searchCallback' + i;
i = i + 1;
} while (window[callbackName])
window[callbackName] = k;
return callbackName;
}
/**
* Flatten an object into a URL query string.
* For example: { foo: 'bar', baz: 42 } becomes 'foo=bar&baz=42'
*/
var queryStr = function (options) {
var query = [];
for (var i in options) {
if (options.hasOwnProperty(i)) {
query.push(encodeURIComponent(i) + '=' + encodeURIComponent(options[i]));
}
}
return query.join('&');
}
/**
* Build a function that can be applied to a callback. The callback processes
* the JSON results of the API call.
*/
return function (k) {
options.format = 'json';
options.callback = createCallback(k);
var script = document.createElement('script');
script.id = 'searchResults';
script.src = endpoint + '?' + queryStr(options);
var head = document.getElementsByTagName('head')[0];
head.appendChild(script);
};
}
async searchWikiBooks(term) {
if (!this.hasQuery) {
return [];
}
const query = this.mediaWikiQuery('https://en.wikibooks.org/w/api.php', {
action: 'query',
list: 'search',
srsearch: this.searchTerm,
srprop: '',
});
query(response => {
console.log(response);
const searchScript = document.getElementById('searchResults');
searchScript.parentNode.removeChild(searchScript);
for (let property in window) {
if (property.includes('searchCallback')) {
delete window[property];
}
}
const bookResults = [];
const pageids = response.query.search.map(item => item.pageid);
const propsQuery = this.mediaWikiQuery('https://en.wikibooks.org/w/api.php', {
action: 'query',
pageids: pageids.join('|'),
prop: 'categories|pageprops',
});
propsQuery(propsResponse => {
console.log(propsResponse);
for (let pageid in propsResponse.query.pages) {
if (propsResponse.query.pages[pageid].hasOwnProperty('categories')) {
}
}
});
return bookResults;
});
}
async searchOpenLibrary(searchBy = 'title') {
if (!this.hasQuery) {
return [];
}
return fetch(`https://openlibrary.org/search.json?${searchBy}=${encodeURIComponent(this.searchTerm)}`)
.then(res => res.json())
.then(response => {
if (!response.hasOwnProperty('docs')) {
return [];
}
const booksController = new BooksController('openLibrary', undefined, this.lang);
// Format the response into usable objects
const docs = response.docs.map(doc => {
return booksController.handleOpenLibraryEntity(doc);
});
let results = [];
// Filter out duplicate items with the same title and author
docs.forEach(doc => {
const existingDoc = results.find((filterResult) => {
return filterResult.title === doc.title && filterResult.description === doc.description;
});
if (!existingDoc) {
results.push(doc);
}
});
results = results.map(result => {
// Find any duplicates in case they have different cover data
const duplicates = docs.filter(doc => {
return doc.name.toLowerCase() === result.name.toLowerCase() && doc.description === result.description;
});
result.covers = [];
duplicates.forEach(duplicate => {
if (duplicate.coverId !== false) {
result.covers.push({
uri: duplicate.coverId,
url: `//covers.openlibrary.org/b/id/${duplicate.coverId}-M.jpg`,
});
}
});
delete result.coverId;
return result;
}).map(bookData => {
// Use bookController to get community data
booksController.uri = bookData.uri; // Update booksController.uri for each book when fetching community data.
const communityData = booksController.getCommunityData(5);
return {
...bookData,
...communityData,
};
});
return results;
}).catch(error => {
console.log(error);
return [];
});
}
}
module.exports = SearchController;

View File

@ -1,4 +1,4 @@
const BooksController = require('../controllers/books');
const BooksController = require('../controllers/bookData');
async function routes(fastify, options) {
fastify.get('/api/books', async (request, reply) => {

View File

@ -5,31 +5,16 @@ async function routes(fastify, options) {
const searchTerm = typeof request.query.for !== 'undefined' ? request.query.for.trim() : '';
const searchBy = typeof request.query.by !== 'undefined' ? request.query.by.trim() : 'title';
const language = typeof request.query.lang !== 'undefined' ? request.query.lang.trim().split('-')[0] : undefined; // Get base language in cases like 'en-US'
const searchSource = typeof request.query.source !== 'undefined' ? request.query.source.trim() : undefined; // Get base language in cases like 'en-US'
const search = new SearchController(searchTerm, language);
const source = typeof request.query.source !== 'undefined' ? request.query.source.trim() : undefined; // Get base language in cases like 'en-US'
const controller = new SearchController(fastify.models, searchTerm, { searchBy, source, language });
switch (searchSource) {
case 'openLibrary': {
return await search.searchOpenLibrary(searchBy);
}
case 'bookBrainz': {
return await search.searchOpenLibrary(searchBy);
}
case 'inventaire':
default: {
if (searchBy === 'title') {
return await search.quickSearchInventaire();
} else {
return await search.searchInventaire(searchBy);
}
}
}
return await controller.search();
});
fastify.get('/api/search/cover', async (request, reply) => {
const inventaireURI = typeof request.query.uri !== 'undefined' ? request.query.uri.trim() : false;
const language = typeof request.query.lang !== 'undefined' ? request.query.lang.trim().split('-')[0] : undefined; // Get base language in cases like 'en-US'
const search = new SearchController(fastify.siteConfig.inventaireDomain, null, language);
const search = new SearchController(fastify.models, null, language);
return await search.getInventaireCovers(inventaireURI);
});

View File

@ -5,6 +5,17 @@ module.exports = models => {
} = models;
BookReference.hasMany(Review, {
as: 'Interactions',
foreignKey: 'bookReferenceId',
});
BookReference.hasMany(Review.scope('Text'), {
as: 'Reviews',
foreignKey: 'bookReferenceId',
});
BookReference.hasMany(Review.scope('Rating'), {
as: 'Ratings',
foreignKey: 'bookReferenceId',
});

View File

@ -18,8 +18,8 @@ module.exports = sequelize => sequelize.define('BookReference', {
sources: {
type: Sequelize.JSON,
allowNull: false,
defaultValue: [],
comment: 'A JSON array with each element being an object with named source <STRING> and source id <STRING>',
defaultValue: {},
comment: 'A JSON object with each key a source name and value a source id',
},
covers: {
type: Sequelize.JSON,
@ -27,6 +27,12 @@ module.exports = sequelize => sequelize.define('BookReference', {
defaultValue: [],
comment: 'A JSON array with each element being an object with image url <STRING> and source id <STRING>',
},
locale: {
type: Sequelize.STRING,
allowNull: false,
defaultValue: 'en',
comment: 'A basic locale string',
},
}, {
timestamps: false,
indexes: [
@ -40,5 +46,8 @@ module.exports = sequelize => sequelize.define('BookReference', {
{
fields: ['description'],
},
{
fields: ['locale'],
},
],
});

View File

@ -49,10 +49,10 @@ module.exports = sequelize => sequelize.define('Follow', {
}, {
indexes: [
{
fields: ['follower', 'domain'],
fields: ['follower', 'followerDomain'],
},
{
fields: ['following', 'domain'],
fields: ['following', 'followingDomain'],
},
],
scopes: {

View File

@ -67,4 +67,12 @@ module.exports = sequelize => sequelize.define('Review', {
fields: ['bookReferenceId'],
},
],
scopes: {
Rating: {
where: { rating: { [Sequelize.Op.not]: null } },
},
Text: {
where: { text: { [Sequelize.Op.not]: null } },
},
},
});