diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7527253dfbe..5f167e37090 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: - types-all - repo: https://github.com/asottile/pyupgrade - rev: v2.38.0 + rev: v2.38.2 hooks: - id: pyupgrade args: # Solr on Cython is not yet ready for 3.10 type hints diff --git a/bundlesize.config.json b/bundlesize.config.json index 755e2caa735..0f71a51421b 100644 --- a/bundlesize.config.json +++ b/bundlesize.config.json @@ -58,7 +58,7 @@ }, { "path": "static/build/admin.*.js", - "maxSize": "1KB" + "maxSize": "1.3KB" }, { "path": "static/build/search.*.js", diff --git a/openlibrary/components/MergeUI.vue b/openlibrary/components/MergeUI.vue index 6573db9a849..2e73b256a7c 100644 --- a/openlibrary/components/MergeUI.vue +++ b/openlibrary/components/MergeUI.vue @@ -114,6 +114,13 @@ export default { const splitKey = master.key.split('/') const primaryRecord = splitKey[splitKey.length - 1] await createMergeRequest(workIds, primaryRecord, 'create-pending', this.comment) + .then(response => response.json()) + .then(data => { + if (data.status === 'ok') { + // Redirect to merge table on success: + window.location.replace(`/merges#mrid-${data.id}`) + } + }) } this.mergeStatus = 'Done'; }, diff --git a/openlibrary/core/bookshelves.py b/openlibrary/core/bookshelves.py index a1f74849f26..6c24bfec280 100644 --- a/openlibrary/core/bookshelves.py +++ b/openlibrary/core/bookshelves.py @@ -88,6 +88,7 @@ def most_logged_books( users. This query is limited to a specific shelf_id (e.g. 1 for "Want to Read"). """ + page = int(page or 1) offset = (page - 1) * limit oldb = db.get_db() where = 'WHERE bookshelf_id' + ('=$shelf_id' if shelf_id else ' IS NOT NULL ') diff --git a/openlibrary/core/models.py b/openlibrary/core/models.py index 0493104fa25..0d8c58e81f1 100644 --- a/openlibrary/core/models.py +++ b/openlibrary/core/models.py @@ -646,7 +646,7 @@ def resolve_redirects_bulk( start_offset=0, grace_period_days=7, cutoff_date=datetime.datetime(year=2017, month=1, day=1), - test=True, + test=False, ): """ batch_size - how many records to fetch per batch diff --git a/openlibrary/core/schema.sql b/openlibrary/core/schema.sql index ccae4df1a1f..ca3a37ef639 100644 --- a/openlibrary/core/schema.sql +++ b/openlibrary/core/schema.sql @@ -1,5 +1,4 @@ - CREATE TABLE ratings ( username text NOT NULL, work_id integer NOT NULL, @@ -9,6 +8,7 @@ CREATE TABLE ratings ( created timestamp without time zone default (current_timestamp at time zone 'utc'), primary key (username, work_id) ); +CREATE INDEX ratings_work_id_idx ON ratings (work_id); CREATE TABLE booknotes ( username text NOT NULL, @@ -19,6 +19,7 @@ CREATE TABLE booknotes ( created timestamp without time zone default (current_timestamp at time zone 'utc'), primary key (username, work_id, edition_id) ); +CREATE INDEX booknotes_work_id_idx ON booknotes (work_id); CREATE TABLE bookshelves ( id serial not null primary key, @@ -39,7 +40,8 @@ CREATE TABLE bookshelves_books ( created timestamp without time zone default (current_timestamp at time zone 'utc'), primary key (username, work_id, bookshelf_id) ); - +CREATE INDEX bookshelves_books_work_id_idx ON bookshelves_books (work_id); +-- bookshelves_votes currently unused CREATE TABLE bookshelves_votes ( username text NOT NULL, bookshelf_id serial NOT NULL REFERENCES bookshelves(id) ON DELETE CASCADE ON UPDATE CASCADE, @@ -62,6 +64,7 @@ CREATE TABLE observations ( created timestamp without time zone default (current_timestamp at time zone 'utc'), primary key (work_id, edition_id, username, observation_value, observation_type) ); +CREATE INDEX observations_username_idx ON observations (username); CREATE TABLE community_edits_queue ( id serial not null primary key, diff --git a/openlibrary/core/sponsorships.py b/openlibrary/core/sponsorships.py index 8d0028686b4..525c88436e1 100644 --- a/openlibrary/core/sponsorships.py +++ b/openlibrary/core/sponsorships.py @@ -103,9 +103,12 @@ def do_we_want_it(isbn): } url = '%s/book/marc/ol_dedupe.php' % lending.config_ia_domain try: - data = requests.get(url, params=params).json() + data = requests.get(url, params=params, timeout=2).json() dwwi = data.get('response', 0) return dwwi == 1, data.get('books', []) + except requests.exceptions.Timeout: + logger.exception('DWWI Timeout') + return False, [] except: logger.error("DWWI Failed for isbn %s" % isbn, exc_info=True) # err on the side of false negative diff --git a/openlibrary/plugins/admin/code.py b/openlibrary/plugins/admin/code.py index d342a32c375..3be20bd6d6f 100644 --- a/openlibrary/plugins/admin/code.py +++ b/openlibrary/plugins/admin/code.py @@ -82,6 +82,12 @@ def handle(self, cls, args=(), librarians=False): if not m: raise web.nomethod(cls=cls) else: + if ( + context.user + and context.user.is_usergroup_member('/usergroup/librarians') + and web.ctx.path == '/admin/solr' + ): + return m(*args) if self.is_admin() or ( librarians and context.user diff --git a/openlibrary/plugins/openlibrary/borrow_home.py b/openlibrary/plugins/openlibrary/borrow_home.py index 6d0c3407c2d..39848ec8267 100644 --- a/openlibrary/plugins/openlibrary/borrow_home.py +++ b/openlibrary/plugins/openlibrary/borrow_home.py @@ -1,4 +1,8 @@ -"""Controller for /borrow page. +""" +Controllers for /borrow pages. + +These endpoints are largely deprecated, and only maintained for +backwards compatibility. """ import web @@ -8,227 +12,40 @@ import eventer -from infogami.plugins.api.code import jsonapi from infogami.utils import delegate from infogami.utils.view import render_template # noqa: F401 used for its side effects -from openlibrary.core import helpers as h from openlibrary.core import statsdb -from openlibrary.plugins.worksearch.subjects import SubjectEngine -class borrow(delegate.page): # TODO: Why is this function defined twice? +class borrow(delegate.page): path = "/borrow" def GET(self): raise web.seeother('/subjects/in_library#ebooks=true') -class borrow(delegate.page): # type: ignore[no-redef] +class borrow_json(delegate.page): path = "/borrow" encoding = "json" - def is_enabled(self): - return "inlibrary" in web.ctx.features - - @jsonapi def GET(self): - i = web.input( - offset=0, limit=12, rand=-1, details="false", has_fulltext="false" - ) - - filters = {} - if i.get("has_fulltext") == "true": - filters["has_fulltext"] = "true" - - if i.get("published_in"): - if "-" in i.published_in: - begin, end = i.published_in.split("-", 1) - - if ( - h.safeint(begin, None) is not None - and h.safeint(end, None) is not None - ): - filters["publish_year"] = [begin, end] - else: - y = h.safeint(i.published_in, None) - if y is not None: - filters["publish_year"] = i.published_in + raise web.seeother('/subjects/in_library.json' + web.ctx.query) - i.limit = h.safeint(i.limit, 12) - i.offset = h.safeint(i.offset, 0) - i.rand = h.safeint(i.rand, -1) - - if i.rand > 0: - sort = 'random_%d desc' % i.rand - filters['sort'] = sort - - subject = get_lending_library( - web.ctx.site, - offset=i.offset, - limit=i.limit, - details=i.details.lower() == "true", - inlibrary=False, - **filters, - ) - return json.dumps(subject) - - -class read(delegate.page): # TODO: Why is this function defined twice? +class read(delegate.page): path = "/read" def GET(self): web.seeother('/subjects/accessible_book#ebooks=true') -class read(delegate.page): # type: ignore[no-redef] +class read_json(delegate.page): path = "/read" encoding = "json" - @jsonapi - def GET(self): - i = web.input( - offset=0, limit=12, rand=-1, details="false", has_fulltext="false" - ) - - filters = {} - if i.get("has_fulltext") == "true": - filters["has_fulltext"] = "true" - - if i.get("published_in"): - if "-" in i.published_in: - begin, end = i.published_in.split("-", 1) - - if ( - h.safeint(begin, None) is not None - and h.safeint(end, None) is not None - ): - filters["publish_year"] = [begin, end] - else: - y = h.safeint(i.published_in, None) - if y is not None: - filters["publish_year"] = i.published_in - - i.limit = h.safeint(i.limit, 12) - i.offset = h.safeint(i.offset, 0) - - i.rand = h.safeint(i.rand, -1) - - if i.rand > 0: - sort = 'random_%d desc' % i.rand - filters['sort'] = sort - - subject = get_readable_books( - web.ctx.site, - offset=i.offset, - limit=i.limit, - details=i.details.lower() == "true", - **filters, - ) - return json.dumps(subject) - - -class borrow_about(delegate.page): - path = "/borrow/about" - def GET(self): - raise web.notfound() - - -def convert_works_to_editions(site, works): - """Takes work docs got from solr and converts them into appropriate editions required for lending library.""" - ekeys = [ - '/books/' + w['lending_edition'] for w in works if w.get('lending_edition') - ] - editions = {} - for e in site.get_many(ekeys): - editions[e['key']] = e.dict() - - for w in works: - if w.get('lending_edition'): - ekey = '/books/' + w['lending_edition'] - e = editions.get(ekey) - if e and 'ocaid' in e: - covers = e.get('covers') or [None] - w['key'] = e['key'] - w['cover_id'] = covers[0] - w['ia'] = e['ocaid'] - w['title'] = e.get('title') or w['title'] - - -def get_lending_library(site, inlibrary=False, **kw): - kw.setdefault("sort", "first_publish_year desc") - - if inlibrary: - subject = CustomSubjectEngine().get_subject( - "/subjects/lending_library", in_library=True, **kw - ) - else: - subject = CustomSubjectEngine().get_subject( - "/subjects/lending_library", in_library=False, **kw - ) - - subject['key'] = '/borrow' - convert_works_to_editions(site, subject['works']) - return subject - - -def get_readable_books(site, **kw): - kw.setdefault("sort", "first_publish_year desc") - subject = ReadableBooksEngine().get_subject("/subjects/dummy", **kw) - subject['key'] = '/read' - return subject - - -class ReadableBooksEngine(SubjectEngine): - """SubjectEngine for readable books. - - This doesn't take subject into account, but considers the public_scan_b - field, which is derived from ia collections. - - There is a subject "/subjects/accessible_book", but it has some - inlibrary/lendinglibrary books as well because of errors in OL data. Using - public_scan_b derived from ia collections is more accurate. - """ - - def make_query(self, key, filters): - return {"public_scan_b": "true"} - - def get_ebook_count(self, name, value, publish_year): - # we are not displaying ebook count. - # No point making a solr query - return 0 - - -class CustomSubjectEngine(SubjectEngine): - """SubjectEngine for inlibrary and lending_library combined.""" - - def make_query(self, key, filters): - meta = self.get_meta(key) - - q = { - meta.facet_key: ["lending_library"], - 'public_scan_b': "false", - 'NOT borrowed_b': "true", - # show only books in last 20 or so years - 'publish_year': (str(1990), str(datetime.date.today().year)), # range - } - - if filters: - if filters.get('in_library') is True: - q[meta.facet_key].append('in_library') - if filters.get("has_fulltext") == "true": - q['has_fulltext'] = "true" - if filters.get("publish_year"): - q['publish_year'] = filters['publish_year'] - - return q - - def get_ebook_count(self, name, value, publish_year): - # we are not displaying ebook count. - # No point making a solr query - return 0 + web.seeother('/subjects/accessible_book.json' + web.ctx.query) def on_loan_created_statsdb(loan): diff --git a/openlibrary/plugins/openlibrary/js/admin.js b/openlibrary/plugins/openlibrary/js/admin.js index 5b1df2c95d6..93e9cfef789 100644 --- a/openlibrary/plugins/openlibrary/js/admin.js +++ b/openlibrary/plugins/openlibrary/js/admin.js @@ -25,10 +25,27 @@ export function initAdmin() { export function initAnonymizationButton(button) { const displayName = button.dataset.displayName; - const confirmMessage = `Really anonymize ${displayName}'s account? This will delete ${displayName}'s profile page and booknotes, and anonymize ${displayName}'s reading log, reviews, and star ratings.`; + const confirmMessage = `Really anonymize ${displayName}'s account? This will delete ${displayName}'s profile page and booknotes, and anonymize ${displayName}'s reading log, reviews, star ratings, and merge request submissions.`; button.addEventListener('click', function(event) { if (!confirm(confirmMessage)) { event.preventDefault(); } }) } + +/** + * Adds click listener to each given button. When the button is clicked, + * the patron is prompted to confirm the action via a dialog. + * + * @param {NodeList} buttons + */ +export function initConfirmationButtons(buttons) { + const confirmMessage = 'Are you sure?' + for (const button of buttons) { + button.addEventListener('click', function(event) { + if (!confirm(confirmMessage)) { + event.preventDefault(); + } + }) + } +} diff --git a/openlibrary/plugins/openlibrary/js/edit.js b/openlibrary/plugins/openlibrary/js/edit.js index 6507a79fa9e..9023952a050 100644 --- a/openlibrary/plugins/openlibrary/js/edit.js +++ b/openlibrary/plugins/openlibrary/js/edit.js @@ -103,30 +103,15 @@ function isFormatValidIsbn10(isbn) { * @returns {boolean} true if ISBN string is a valid ISBN 10 */ export function isChecksumValidIsbn10(isbn) { - const chars = isbn.split(''); - let last = chars.pop(); - let check; - - // With ISBN 10, the last character can be [0-9] or string 'X'. - if (last !== 'X') { - last = parseInt(last); - } + const chars = isbn.replace('X', 'A').split(''); - // Compute the ISBN-10 check digit chars.reverse(); const sum = chars - .map((char, idx) => ((idx + 2) * parseInt(char, 10))) - .reduce((acc, sum) => acc + sum, 0) - - check = 11 - (sum % 11); - if (check === 10) { - check = 'X'; - } else if (check === 11) { - check = 0; - } + .map((char, idx) => ((idx + 1) * parseInt(char, 16))) + .reduce((acc, sum) => acc + sum, 0); - // The ISBN 10 is valid if the check digit and last digit match. - return check === last; + // The ISBN 10 is valid if the checksum mod 11 is 0. + return sum % 11 === 0; } /** @@ -148,21 +133,12 @@ function isFormatValidIsbn13(isbn) { */ export function isChecksumValidIsbn13(isbn) { const chars = isbn.split(''); - // Remove the final ISBN digit from `chars`, and assign it to `last` for comparison. - const last = parseInt(chars.pop()); - let check; - const sum = chars .map((char, idx) => ((idx % 2 * 2 + 1) * parseInt(char, 10))) .reduce((sum, num) => sum + num, 0); - check = 10 - (sum % 10); - if (check === 10) { - check = 0; - } - - // The ISBN 13 is valid if the check digit and last digit match. - return check === last; + // The ISBN 13 is valid if the checksum mod 10 is 0. + return sum % 10 === 0; } /** diff --git a/openlibrary/plugins/openlibrary/js/ile/utils/SelectionManager/SelectionManager.js b/openlibrary/plugins/openlibrary/js/ile/utils/SelectionManager/SelectionManager.js index f46fceb5e0e..c5956cec03f 100644 --- a/openlibrary/plugins/openlibrary/js/ile/utils/SelectionManager/SelectionManager.js +++ b/openlibrary/plugins/openlibrary/js/ile/utils/SelectionManager/SelectionManager.js @@ -57,6 +57,9 @@ export default class SelectionManager { * @param {MouseEvent & { currentTarget: HTMLElement }} clickEvent */ toggleSelected(clickEvent) { + // If there is text selection, dont do anything + if (window.getSelection()?.toString() !== '') return; + const el = clickEvent.currentTarget; el.classList.toggle('ile-selected'); const selected = el.classList.contains('ile-selected'); diff --git a/openlibrary/plugins/openlibrary/js/index.js b/openlibrary/plugins/openlibrary/js/index.js index 60f0754b8b7..0767e40069b 100644 --- a/openlibrary/plugins/openlibrary/js/index.js +++ b/openlibrary/plugins/openlibrary/js/index.js @@ -312,7 +312,8 @@ jQuery(function () { } const anonymizationButton = document.querySelector('.account-anonymization-button') const adminLinks = document.getElementById('adminLinks') - if (adminLinks || anonymizationButton) { + const confirmButtons = document.querySelectorAll('.do-confirm') + if (adminLinks || anonymizationButton || confirmButtons.length) { import(/* webpackChunkName: "admin" */ './admin') .then(module => { if (adminLinks) { @@ -321,6 +322,9 @@ jQuery(function () { if (anonymizationButton) { module.initAnonymizationButton(anonymizationButton); } + if (confirmButtons.length) { + module.initConfirmationButtons(confirmButtons); + } }); } diff --git a/openlibrary/plugins/openlibrary/js/page-barcodescanner/index.js b/openlibrary/plugins/openlibrary/js/page-barcodescanner/index.js index 453039034b4..f1929e3c4e0 100644 --- a/openlibrary/plugins/openlibrary/js/page-barcodescanner/index.js +++ b/openlibrary/plugins/openlibrary/js/page-barcodescanner/index.js @@ -1,26 +1,82 @@ -import Quagga from 'quagga'; // barcode scanning library +// @ts-check +import countBy from 'lodash/countBy'; +import maxBy from 'lodash/maxBy'; +import Quagga from '@ericblade/quagga2'; // barcode scanning library import LazyBookCard from './LazyBookCard'; const BOX_STYLE = {color: 'green', lineWidth: 2}; const RESULT_BOX_STYLE = {color: 'blue', lineWidth: 2}; const RESULT_LINE_STYLE = {color: 'red', lineWidth: 15}; -export function init() { - Quagga.init({ - inputStream: { - name: 'Live', - type: 'LiveStream', - target: $('#interactive')[0], - }, - decoder: { - readers: ['ean_reader'] - }, - }, function(err) { - if (err) throw err; - Quagga.start(); - }); - - Quagga.onProcessed(result => { +class OLBarcodeScanner { + constructor() { + this.lastISBN = null; + + const urlParams = new URLSearchParams(location.search) + this.returnTo = urlParams.get('returnTo'); + + // If we get noise, group and choose latest + this.submitISBNThrottled = new ThrottleGrouping({ + func: this.submitISBN.bind(this), + // Use the most frequent + reducer: (groupOfAargs) => { + const isbnCounts = Array.from( + Object.entries( + countBy(groupOfAargs, (arg) => arg[0]) + ) + ); + + /* eslint-disable no-unused-vars */ + const mostFrequentISBN = maxBy(isbnCounts, ([isbn, count]) => count)[0]; + return groupOfAargs.reverse().find((args) => args[0] === mostFrequentISBN); + }, + wait: 300, + }).asFunction(); + } + + start() { + Quagga.init({ + locator: { + halfSample: true, + }, + inputStream: { + name: 'Live', + type: 'LiveStream', + target: $('#interactive')[0], + constraints: { + // Vertical - This is *essential* for iPhone/iPad + aspectRatio: {ideal: 720/1280}, + }, + }, + decoder: { + readers: ['ean_reader'] + }, + }, async function(err) { + if (err) throw err; + + const track = Quagga.CameraAccess.getActiveTrack(); + if (track && typeof track.getCapabilities === 'function') { + const capabilities = track.getCapabilities(); + // Use a higher resolution + if (capabilities.width.max >= 1280 && capabilities.height.max >= 720) { + await track.applyConstraints({advanced: [{width: 1280, height: 720}]}); + } + } + + Quagga.start(); + + const quaggaVideo = /** @type {HTMLVideoElement} */($('#interactive video')[0]); + if (quaggaVideo.paused) { + quaggaVideo.setAttribute('controls', 'true'); + quaggaVideo.addEventListener('play', () => quaggaVideo.removeAttribute('controls')); + } + }); + + Quagga.onProcessed(this.handleQuaggaProcessed.bind(this)); + Quagga.onDetected(this.handleQuaggaDetected.bind(this)); + } + + handleQuaggaProcessed(result) { if (!result) return; const drawingCtx = Quagga.canvas.ctx.overlay; @@ -45,20 +101,35 @@ export function init() { if (result.codeResult && result.codeResult.code && isBarcodeISBN(result.codeResult.code)) { Quagga.ImageDebug.drawPath(result.line, {x: 'x', y: 'y'}, drawingCtx, RESULT_LINE_STYLE); } - }); + } - let lastResult = null; - Quagga.onDetected(result => { + handleQuaggaDetected(result) { const code = result.codeResult.code; - if (!isBarcodeISBN(code) || code === lastResult) return; - lastResult = code; + if (!isBarcodeISBN(code)) return; - const isbn = code; - const canvas = Quagga.canvas.dom.image; + this.submitISBNThrottled(code, Quagga.canvas.dom.image.toDataURL()); + } + + /** + * @param {string} isbn + * @param {string} tentativeCoverUrl + */ + submitISBN(isbn, tentativeCoverUrl) { + if (isbn === this.lastISBN) return; + + this.lastISBN = isbn; const card = LazyBookCard.fromISBN(isbn); - card.updateState({coverSrc: canvas.toDataURL()}); + card.updateState({coverSrc: tentativeCoverUrl}); $('#result-strip').prepend(card.render()); - }); + + if (this.returnTo) { + location = this.returnTo.replace('$$$', isbn); + } + } +} + +export function init() { + new OLBarcodeScanner().start(); } /** @@ -69,3 +140,44 @@ export function init() { function isBarcodeISBN(code) { return code.startsWith('97'); } + + +/** + * @template {(...args: any) => void} TFunc + */ +class ThrottleGrouping { + /** + * @param {object} param0 + * @param {TFunc} param0.func + * @param {function(Parameters[]): Parameters} param0.reducer + * @param {number} param0.wait + */ + constructor({func, reducer, wait=100}) { + this.func = func; + this.reducer = reducer; + this.wait = wait; + /** @type {Parameters[]} */ + this.curGroup = []; + this.timeout = null; + } + + submitGroup() { + this.timeout = null; + this.func(...this.reducer(this.curGroup)); + this.curGroup = []; + } + + /** + * @param {Parameters} args + */ + takeNext(...args) { + this.curGroup.push(args); + if (!this.timeout) { + this.timeout = setTimeout(this.submitGroup.bind(this), this.wait); + } + } + + asFunction() { + return this.takeNext.bind(this); + } +} diff --git a/openlibrary/plugins/openlibrary/tests/test_borrow_home.py b/openlibrary/plugins/openlibrary/tests/test_borrow_home.py deleted file mode 100644 index c6b568de48b..00000000000 --- a/openlibrary/plugins/openlibrary/tests/test_borrow_home.py +++ /dev/null @@ -1,79 +0,0 @@ -from .. import borrow_home - - -class Test_convert_works_to_editions: - def convert(self, site, works): - borrow_home.convert_works_to_editions(site, works) - return works - - def test_no_lending_edition(self, mock_site): - """if there is no lending edition, work should stay the same.""" - work = {"key": "/works/OL1W", "title": "foo"} - assert self.convert(mock_site, [work]) == [work] - - def test_lending_edition(self, mock_site): - """if there is no lending edition, work should stay the same.""" - mock_site.save( - { - "key": "/books/OL1M", - "ocaid": "foo2010bar", - "title": "bar", - "covers": [1234], - } - ) - - work = { - "key": "/works/OL1W", - "title": "foo", - "ia": "foofoo", - "lending_edition": "OL1M", - "cover_id": 1111, - } - assert self.convert(mock_site, [work]) == [ - { - "key": "/books/OL1M", - "title": "bar", - "lending_edition": "OL1M", - "ia": "foo2010bar", - "cover_id": 1234, - } - ] - - def test_edition_with_no_title(self, mock_site): - """When the editon has no title, work's title should be retained.""" - mock_site.save({"key": "/books/OL1M", "ocaid": "foofoo"}) - work = { - "key": "/works/OL1W", - "title": "foo", - "ia": "foofoo", - "lending_edition": "OL1M", - } - assert self.convert(mock_site, [work]) == [ - { - "key": "/books/OL1M", - "title": "foo", - "lending_edition": "OL1M", - "ia": "foofoo", - "cover_id": None, - } - ] - - def test_edition_with_no_cover(self, mock_site): - """When the editon has no cover, work's cover should *not* be retained.""" - mock_site.save({"key": "/books/OL1M", "ocaid": "foofoo"}) - work = { - "key": "/works/OL1W", - "title": "foo", - "ia": "foofoo", - "lending_edition": "OL1M", - "cover_id": 1234, - } - assert self.convert(mock_site, [work]) == [ - { - "key": "/books/OL1M", - "title": "foo", - "lending_edition": "OL1M", - "ia": "foofoo", - "cover_id": None, - } - ] diff --git a/openlibrary/plugins/upstream/addbook.py b/openlibrary/plugins/upstream/addbook.py index c2c7cb8eb21..e0827d9c9c8 100644 --- a/openlibrary/plugins/upstream/addbook.py +++ b/openlibrary/plugins/upstream/addbook.py @@ -7,6 +7,8 @@ import csv import datetime +from typing import Literal, overload, NoReturn + from infogami import config from infogami.core import code as core from infogami.core.db import ValidationException @@ -22,6 +24,7 @@ import logging from openlibrary.plugins.upstream import spamcheck, utils +from openlibrary.plugins.upstream.models import Author, Edition, Work from openlibrary.plugins.upstream.utils import render_template, fuzzy_find from openlibrary.plugins.upstream.account import as_admin @@ -84,11 +87,25 @@ def make_author(key, name): return w -def new_doc(type_, **data): +@overload +def new_doc(type_: Literal["/type/author"], **data) -> Author: + ... + + +@overload +def new_doc(type_: Literal["/type/edition"], **data) -> Edition: + ... + + +@overload +def new_doc(type_: Literal["/type/work"], **data) -> Work: + ... + + +def new_doc(type_: str, **data) -> Author | Edition | Work: """ Create an new OL doc item. :param str type_: object type e.g. /type/edition - :rtype: doc :return: the newly created document """ key = web.ctx.site.new_key(type_) @@ -114,13 +131,13 @@ def commit(self, **kw): if self.docs: web.ctx.site.save_many(self.docs, **kw) - def create_authors_from_form_data(self, authors, author_names, _test=False): + def create_authors_from_form_data( + self, authors: list[dict], author_names: list[str], _test: bool = False + ) -> bool: """ - Create any __new__ authors in the provided array. Updates the authors dicts _in place_ with the new key + Create any __new__ authors in the provided array. Updates the authors + dicts _in place_ with the new key. :param list[dict] authors: e.g. [{author: {key: '__new__'}}] - :param list[str] author_names: - :param bool _test: - :rtype: bool :return: Whether new author(s) were created """ created = False @@ -176,10 +193,9 @@ def GET(self): 'books/add', work=work, author=author, recaptcha=get_recaptcha() ) - def has_permission(self): + def has_permission(self) -> bool: """ Can a book be added? - :rtype: bool """ return web.ctx.site.can_write("/books/add") @@ -236,9 +252,12 @@ def POST(self): # no match return self.no_match(saveutil, i) - def find_matches(self, i): + def find_matches( + self, i: web.utils.Storage + ) -> None | Work | Edition | list[web.utils.Storage]: """ - Tries to find an edition, or work, or multiple work candidates that match the given input data. + Tries to find an edition, or work, or multiple work candidates that match the + given input data. Case#1: No match. None is returned. Case#2: Work match but not edition. Work is returned. @@ -246,8 +265,8 @@ def find_matches(self, i): Case#4: Multiple work match. List of works is returned. :param web.utils.Storage i: addbook user supplied formdata - :rtype: None or list or Work or Edition - :return: None or Work or Edition or list of Works that are likely matches. + :return: None or Work or Edition or list of Works (as Storage objects) that are + likely matches. """ i.publish_year = i.publish_date and self.extract_year(i.publish_date) @@ -298,12 +317,11 @@ def find_matches(self, i): else: return result.docs # Case 4 - def extract_year(self, value): + def extract_year(self, value: str) -> str: """ Extract just the 4 digit year from a date string. :param str value: A freeform string representing a publication date. - :rtype: str :return: a four digit year """ m = web.re_compile(r"(\d\d\d\d)").search(value) @@ -311,32 +329,27 @@ def extract_year(self, value): def try_edition_match( self, - work=None, - title=None, - author_key=None, - publisher=None, - publish_year=None, - id_name=None, - id_value=None, - ): + work: web.Storage = None, + title: str = None, + author_key: str = None, + publisher: str = None, + publish_year: str = None, + id_name: str = None, + id_value: str = None, + ) -> None | Edition | list[web.Storage]: """ Searches solr for potential edition matches. - :param web.Storage work: - :param str title: :param str author_key: e.g. /author/OL1234A - :param str publisher: :param str publish_year: yyyy :param str id_name: from list of values in mapping below - :param str id_value: - :rtype: None or Edition or list - :return: None, an Edition, or a list of Works + :return: None, an Edition, or a list of Works (as web.Storage objects) """ # insufficient data if not publisher and not publish_year and not id_value: - return + return None - q = {} + q: dict = {} work and q.setdefault('key', work.key.split("/")[-1]) title and q.setdefault('title', title) author_key and q.setdefault('author_key', author_key.split('/')[-1]) @@ -373,7 +386,7 @@ def try_edition_match( ["/books/" + key for key in work.edition_key] ) for e in editions: - d = {} + d: dict = {} if publisher: if not e.publishers or e.publishers[0] != publisher: continue @@ -383,22 +396,24 @@ def try_edition_match( ): continue if id_value and id_name in mapping: - if not id_name in e or id_value not in e[id_name]: + if id_name not in e or id_value not in e[id_name]: continue # return the first good likely matching Edition return e - def work_match(self, saveutil, work, i): + return None + + def work_match( + self, saveutil: DocSaveHelper, work: Work, i: web.utils.Storage + ) -> NoReturn: """ Action for when a work, but not edition, is matched. Saves a new edition of work, created form the formdata i. Redirects the user to the newly created edition page in edit mode to add more details. - :param DocSaveHelper saveutil: :param Work work: the matched work for this book :param web.utils.Storage i: user supplied book formdata - :rtype: None """ edition = self._make_edition(work, i) @@ -408,24 +423,19 @@ def work_match(self, saveutil, work, i): raise safe_seeother(edition.url("/edit?mode=add-book")) - def work_edition_match(self, edition): + def work_edition_match(self, edition: Edition) -> NoReturn: """ Action for when an exact work and edition match have been found. Redirect user to the found item's edit page to add any missing details. - :param Edition edition: """ raise safe_seeother(edition.url("/edit?mode=found")) - def no_match(self, saveutil, i): + def no_match(self, saveutil: DocSaveHelper, i: web.utils.Storage) -> NoReturn: """ Action to take when no matches are found. Creates and saves both a Work and Edition. Redirects the user to the work/edition edit page in `add-work` mode. - - :param DocSaveHelper saveutil: - :param web.utils.Storage i: - :rtype: None """ # Any new author has been created and added to # saveutil, and author_key added to i @@ -441,15 +451,9 @@ def no_match(self, saveutil, i): raise safe_seeother(edition.url("/edit?mode=add-work")) - def _make_edition(self, work, i): + def _make_edition(self, work: Work, i: web.utils.Storage) -> Edition: """ - Uses formdata 'i' to create (but not save) an edition - of 'work'. - - :param Work work: - :param web.utils.Storage i: - :rtype: Edition - :return: + Uses formdata 'i' to create (but not save) an edition of 'work'. """ edition = new_doc( "/type/edition", @@ -517,19 +521,17 @@ class SaveBookHelper: This does the required trimming and processing of input data before saving. """ - def __init__(self, work, edition): + def __init__(self, work: Work | None, edition: Edition | None): """ - :param openlibrary.plugins.upstream.models.Work|None work: None if editing an orphan edition - :param openlibrary.plugins.upstream.models.Edition|None edition: None if just editing work + :param Work|None work: None if editing an orphan edition + :param Edition|None edition: None if just editing work """ self.work = work self.edition = edition - def save(self, formdata): + def save(self, formdata: web.Storage) -> None: """ Update work and edition documents according to the specified formdata. - :param web.storage formdata: - :rtype: None """ comment = formdata.pop('_comment', '') @@ -567,6 +569,8 @@ def save(self, formdata): ) if not just_editing_work: + # Mypy misses that "not just_editing_work" means there is edition data. + assert self.edition # Handle orphaned editions new_work_key = (edition_data.get('works') or [{'key': None}])[0]['key'] if self.work is None and ( @@ -617,11 +621,7 @@ def save(self, formdata): saveutil.commit(comment=comment, action="edit-book") @staticmethod - def new_work(edition): - """ - :param openlibrary.plugins.upstream.models.Edition edition: - :rtype: openlibrary.plugins.upstream.models.Work - """ + def new_work(edition: Edition) -> Work: return new_doc( '/type/work', title=edition.get('title'), @@ -708,11 +708,10 @@ def process_edition(self, edition): self._prevent_ocaid_deletion(edition) return edition - def process_work(self, work): + def process_work(self, work: web.Storage) -> web.Storage: """ Process input data for work. :param web.storage work: form data work info - :rtype: web.storage """ def read_subject(subjects): @@ -789,12 +788,11 @@ def _prevent_ocaid_deletion(self, edition): raise ValidationException("Changing Internet Archive ID is not allowed.") @staticmethod - def use_work_edits(formdata): + def use_work_edits(formdata: web.Storage) -> bool: """ Check if the form data's work OLID matches the form data's edition's work OLID. If they don't, then we ignore the work edits. :param web.storage formdata: form data (parsed into a nested dict) - :rtype: bool """ if 'edition' not in formdata: # No edition data -> just editing work, so work data matters diff --git a/openlibrary/plugins/upstream/forms.py b/openlibrary/plugins/upstream/forms.py index bc4afb7b52b..f59f20a12e0 100644 --- a/openlibrary/plugins/upstream/forms.py +++ b/openlibrary/plugins/upstream/forms.py @@ -87,7 +87,9 @@ class RegisterForm(Form): ), Textbox( 'username', - description=_('Choose a screen name'), + description=_( + "Choose a screen name. Screen names are public and cannot be changed later." + ), klass='required', help=_("Letters and numbers only please, and at least 3 characters."), autocapitalize="off", diff --git a/openlibrary/plugins/upstream/merge_authors.py b/openlibrary/plugins/upstream/merge_authors.py index 261c2467da2..055b0bc9566 100644 --- a/openlibrary/plugins/upstream/merge_authors.py +++ b/openlibrary/plugins/upstream/merge_authors.py @@ -1,10 +1,11 @@ """Merge authors. """ import re - import json import web +from typing import Any + from infogami.infobase.client import ClientException from infogami.utils import delegate from infogami.utils.view import render_template, safeint @@ -20,12 +21,7 @@ class BasicRedirectEngine: to the newly identified canonical record. """ - def make_redirects(self, master, duplicates): - """ - :param str master: - :param list of str duplicates: - :rtype: list of dict - """ + def make_redirects(self, master: str, duplicates: list[str]) -> list[dict]: # Create the actual redirect objects docs_to_save = [make_redirect_doc(key, master) for key in duplicates] @@ -50,14 +46,9 @@ def find_all_references(self, keys): refs = {ref for key in keys for ref in self.find_references(key)} return list(refs) - def update_references(self, doc, master, duplicates): + def update_references(self, doc: Any, master: str, duplicates: list[str]) -> Any: """ Converts references to any of the duplicates in the given doc to the master. - - :param doc: - :param str master: - :param list of str duplicates: - :rtype: Any """ if isinstance(doc, dict): if list(doc) == ['key']: @@ -89,12 +80,11 @@ def merge(self, master, duplicates): docs = self.do_merge(master, duplicates) return self.save(docs, master, duplicates) - def do_merge(self, master, duplicates): + def do_merge(self, master: str, duplicates: list[str]) -> list: """ Performs the merge and returns the list of docs to save. :param str master: key of master doc :param list of str duplicates: keys of duplicates - :rtype: dict :return: Document to save """ docs_to_save = [] @@ -212,10 +202,9 @@ def name_eq(n1, n2): return space_squash_and_strip(n1) == space_squash_and_strip(n2) -def fix_table_of_contents(table_of_contents): +def fix_table_of_contents(table_of_contents: list[str | dict]) -> list: """ Some books have bad table_of_contents--convert them in to correct format. - :param typing.List[typing.Union[str, dict]] table_of_contents: """ def row(r): @@ -241,12 +230,7 @@ def row(r): return [row for row in map(row, table_of_contents) if any(row.values())] -def get_many(keys): - """ - :param list of str keys: - :rtype: list of dict - """ - +def get_many(keys: list[str]) -> list[dict]: def process(doc): # some books have bad table_of_contents. Fix them to avoid failure on save. if doc['type']['key'] == "/type/edition" and 'table_of_contents' in doc: @@ -333,9 +317,16 @@ def POST(self): if i.comment: data['comment'] = i.comment - process_merge_request('create-request', data) + result = process_merge_request('create-request', data) + mrid = result.get('id', None) + + username = user.get('key').split('/')[-1] - raise web.seeother('/search/authors') + redir_url = f'/merges?submitter={username}' + if mrid: + redir_url = f'{redir_url}#mrid-{mrid}' + + raise web.seeother(redir_url) else: # redirect to the master. The master will display a progressbar and call the merge_authors_json to trigger the merge. redir_url = ( diff --git a/openlibrary/plugins/upstream/models.py b/openlibrary/plugins/upstream/models.py index 918f2746940..e3d694a4c78 100644 --- a/openlibrary/plugins/upstream/models.py +++ b/openlibrary/plugins/upstream/models.py @@ -21,7 +21,7 @@ from openlibrary.plugins.upstream.utils import MultiDict, parse_toc, get_edition_config from openlibrary.plugins.upstream import account from openlibrary.plugins.upstream import borrow -from openlibrary.plugins.worksearch.code import works_by_author, sorted_work_editions +from openlibrary.plugins.worksearch.code import works_by_author from openlibrary.plugins.worksearch.search import get_solr from openlibrary.utils import dateutil @@ -62,38 +62,6 @@ def get_authors(self): authors = [a for a in authors if a and a.type.key == "/type/author"] return authors - def get_next(self): - """Next edition of work""" - if len(self.get('works', [])) != 1: - return - wkey = self.works[0].get_olid() - if not wkey: - return - editions = sorted_work_editions(wkey) - try: - i = editions.index(self.get_olid()) - except ValueError: - return - if i + 1 == len(editions): - return - return editions[i + 1] - - def get_prev(self): - """Previous edition of work""" - if len(self.get('works', [])) != 1: - return - wkey = self.works[0].get_olid() - if not wkey: - return - editions = sorted_work_editions(wkey) - try: - i = editions.index(self.get_olid()) - except ValueError: - return - if i == 0: - return - return editions[i - 1] - def get_covers(self): """ This methods excludes covers that are -1 or None, which are in the data @@ -539,6 +507,7 @@ def get_books(self, q=''): rows=i.rows, has_fulltext=i.mode == "ebooks", query=q, + facet=True, ) def get_work_count(self): @@ -702,13 +671,13 @@ def is_ascii(s): def get_related_books_subjects(self, filter_unicode=True): return self.filter_problematic_subjects(self.get_subjects(), filter_unicode) - def get_representative_edition(self): + def get_representative_edition(self) -> str | None: """When we have confidence we can direct patrons to the best edition of a work (for them), return qualifying edition key. Attempts to find best (most available) edition of work using archive.org work availability API. May be extended to support language - :rtype str: infogami edition key or url which resolves to an edition + :rtype str: infogami edition key or url which resolves to an edition or None """ work_id = self.key.replace('/works/', '') availability = lending.get_work_availability(work_id) @@ -716,13 +685,14 @@ def get_representative_edition(self): if 'openlibrary_edition' in availability[work_id]: return '/books/%s' % availability[work_id]['openlibrary_edition'] - def get_sorted_editions(self, ebooks_only=False, limit=None, keys=None): + return None + + def get_sorted_editions( + self, ebooks_only: bool = False, limit: int = None, keys: list[str] = None + ) -> list[Edition]: """ Get this work's editions sorted by publication year - :param bool ebooks_only: - :param int limit: :param list[str] keys: ensure keys included in fetched editions - :rtype: list[Edition] """ db_query = {"type": "/type/edition", "works": self.key} db_query['limit'] = limit or 10000 diff --git a/openlibrary/plugins/upstream/utils.py b/openlibrary/plugins/upstream/utils.py index 8ff446e44ec..537f2c9b1e1 100644 --- a/openlibrary/plugins/upstream/utils.py +++ b/openlibrary/plugins/upstream/utils.py @@ -1,5 +1,5 @@ import functools -from typing import List, Union, Tuple, Any +from typing import Any from collections.abc import Iterable import unicodedata @@ -16,7 +16,6 @@ import datetime import logging from html.parser import HTMLParser -from typing import Optional import requests @@ -124,7 +123,7 @@ def render_template(name, *a, **kw): return render[name](*a, **kw) -def kebab_case(upper_camel_case): +def kebab_case(upper_camel_case: str) -> str: """ :param str upper_camel_case: Text in upper camel case (e.g. "HelloWorld") :return: text in kebab case (e.g. 'hello-world') @@ -139,7 +138,7 @@ def kebab_case(upper_camel_case): @public -def render_component(name, attrs=None, json_encode=True): +def render_component(name: str, attrs: dict = None, json_encode: bool = True): """ :param str name: Name of the component (excluding extension) :param dict attrs: attributes to add to the component element @@ -515,7 +514,7 @@ def url_quote(text): @public -def urlencode(dict_or_list_of_tuples: Union[dict, list[tuple[str, Any]]]) -> str: +def urlencode(dict_or_list_of_tuples: dict | list[tuple[str, Any]]) -> str: """ You probably want to use this, if you're looking to urlencode parameters. This will encode things to utf8 that would otherwise cause urlencode to error. @@ -684,7 +683,7 @@ def normalize(s: str) -> str: continue -def get_language(lang_or_key: Union[Thing, str]) -> Optional[Thing]: +def get_language(lang_or_key: Thing | str) -> Thing | None: if isinstance(lang_or_key, str): return get_languages().get(lang_or_key) else: @@ -692,7 +691,7 @@ def get_language(lang_or_key: Union[Thing, str]) -> Optional[Thing]: @public -def get_language_name(lang_or_key: Union[Thing, str]): +def get_language_name(lang_or_key: Thing | str): if isinstance(lang_or_key, str): lang = get_language(lang_or_key) if not lang: @@ -705,7 +704,7 @@ def get_language_name(lang_or_key: Union[Thing, str]): @functools.cache -def convert_iso_to_marc(iso_639_1: str) -> Optional[str]: +def convert_iso_to_marc(iso_639_1: str) -> str | None: """ e.g. 'en' -> 'eng' """ @@ -1083,7 +1082,7 @@ def handle_endtag(self, tag): @public -def reformat_html(html_str: str, max_length: Optional[int] = None) -> str: +def reformat_html(html_str: str, max_length: int = None) -> str: """ Reformats an HTML string, removing all opening and closing tags. Adds a line break element between each set of text content. diff --git a/openlibrary/plugins/worksearch/code.py b/openlibrary/plugins/worksearch/code.py index a2e117a6923..ea44fd6ae84 100644 --- a/openlibrary/plugins/worksearch/code.py +++ b/openlibrary/plugins/worksearch/code.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from datetime import datetime import copy import json @@ -350,6 +351,10 @@ def ia_collection_s_transform(sf: luqum.tree.SearchField): def process_user_query(q_param: str) -> str: + if q_param == '*:*': + # This is a special solr syntax; don't process + return q_param + try: q_param = escape_unknown_fields( ( @@ -427,12 +432,20 @@ def build_q_from_params(param: dict[str, str]) -> str: return ' AND '.join(q_list) +solr_session = requests.Session() + + def execute_solr_query( solr_path: str, params: Union[dict, list[tuple[str, Any]]] ) -> Optional[Response]: - stats.begin("solr", url=f'{solr_path}?{urlencode(params)}') + url = solr_path + if params: + url += '&' if '?' in url else '?' + url += urlencode(params) + + stats.begin("solr", url=url) try: - response = requests.get(solr_path, params=params, timeout=10) + response = solr_session.get(url, timeout=10) response.raise_for_status() except requests.HTTPError: logger.exception("Failed solr query") @@ -442,23 +455,6 @@ def execute_solr_query( return response -def parse_json_from_solr_query( - solr_path: str, params: Union[dict, list[tuple[str, Any]]] -) -> Optional[dict]: - """ - Returns a json.loaded Python object or None - """ - response = execute_solr_query(solr_path, params) - if not response: - logger.error("Error parsing empty search engine response") - return None - try: - return response.json() - except JSONDecodeError: - logger.exception("Error parsing search engine response") - return None - - @public def has_solr_editions_enabled(): if 'pytest' in sys.modules: @@ -490,7 +486,9 @@ def run_solr_query( spellcheck_count=None, offset=None, fields: Union[str, list[str]] = None, - facet=True, + facet: Union[bool, Iterable[str]] = True, + allowed_filter_params=FACET_FIELDS, + extra_params: Optional[list[tuple[str, Any]]] = None, ): """ :param param: dict of query parameters @@ -511,7 +509,7 @@ def run_solr_query( ('start', offset), ('rows', rows), ('wt', param.get('wt', 'json')), - ] + ] + (extra_params or []) if spellcheck_count is None: spellcheck_count = default_spellcheck_count @@ -522,8 +520,19 @@ def run_solr_query( if facet: params.append(('facet', 'true')) - for facet in FACET_FIELDS: - params.append(('facet.field', facet)) + facet_fields = FACET_FIELDS if isinstance(facet, bool) else facet + for facet in facet_fields: + if isinstance(facet, str): + params.append(('facet.field', facet)) + elif isinstance(facet, dict): + params.append(('facet.field', facet['name'])) + if 'sort' in facet: + params.append((f'f.{facet["name"]}.facet.sort', facet['sort'])) + if 'limit' in facet: + params.append((f'f.{facet["name"]}.facet.limit', facet['limit'])) + else: + # Should never get here + raise ValueError(f'Invalid facet type: {facet}') if 'public_scan' in param: v = param.pop('public_scan').lower() @@ -548,7 +557,7 @@ def run_solr_query( else: del param['has_fulltext'] - for field in FACET_FIELDS: + for field in allowed_filter_params: if field == 'has_fulltext': continue if field == 'author_facet': @@ -652,7 +661,7 @@ def convert_work_query_to_edition_query(work_query: str) -> str: q_tree = luqum_parser(work_query) for node, parents in luqum_traverse(q_tree): - if isinstance(node, luqum.tree.SearchField): + if isinstance(node, luqum.tree.SearchField) and node.name != '*': new_name = convert_work_field_to_edition_field(node.name) if new_name: parent = parents[-1] if parents else None @@ -748,13 +757,57 @@ def convert_work_query_to_edition_query(work_query: str) -> str: params.append(('q', full_work_query)) if sort: - params.append(('sort', sort)) + params.append(('sort', process_sort(sort))) url = f'{solr_select_url}?{urlencode(params)}' response = execute_solr_query(solr_select_url, params) solr_result = response.json() if response else None - return (solr_result, url) + return SearchResponse.from_solr_result(solr_result, sort, url) + + +@dataclass +class SearchResponse: + facet_counts: dict[str, tuple[str, str, int]] + sort: str + docs: list + num_found: int + solr_select: str + raw_resp: dict = None + error: str = None + + @staticmethod + def from_solr_result( + solr_result: Optional[dict], + sort: str, + solr_select: str, + ) -> 'SearchResponse': + if not solr_result or 'error' in solr_result: + return SearchResponse( + facet_counts=None, + sort=sort, + docs=[], + num_found=None, + solr_select=solr_select, + error=(solr_result.get('error') if solr_result else None), + ) + else: + return SearchResponse( + facet_counts=( + dict( + process_facet_counts( + solr_result['facet_counts']['facet_fields'] + ) + ) + if 'facet_counts' in solr_result + else None + ), + sort=sort, + raw_resp=solr_result, + docs=solr_result['response']['docs'], + num_found=solr_result['response']['numFound'], + solr_select=solr_select, + ) def do_search( @@ -769,9 +822,7 @@ def do_search( :param sort: csv sort ordering :param spellcheck_count: Not really used; should probably drop """ - if sort: - sort = process_sort(sort) - (solr_result, solr_select) = run_solr_query( + return run_solr_query( param, rows, page, @@ -780,16 +831,6 @@ def do_search( fields=list(DEFAULT_SEARCH_FIELDS | {'editions'}), ) - if not solr_result or 'error' in solr_result: - return web.storage( - facet_counts=None, - docs=[], - is_advanced=bool(param.get('q')), - num_found=None, - solr_select=solr_select, - error=(solr_result.get('error') if solr_result else None), - ) - # TODO: Re-enable spellcheck; not working for a while though. # spellcheck = root.find("lst[@name='spellcheck']") # spell_map = {} @@ -801,19 +842,6 @@ def do_search( # continue # spell_map[a] = [i.text for i in e.find("arr[@name='suggestion']")] - return web.storage( - facet_counts=dict( - process_facet_counts(solr_result['facet_counts']['facet_fields']) - ), - resp=solr_result, - docs=solr_result['response']['docs'], - is_advanced=bool(param.get('q')), - num_found=solr_result['response']['numFound'], - solr_select=solr_select, - error=None, - # spellcheck=spell_map, - ) - def get_doc(doc: SolrDocument): """ @@ -866,41 +894,6 @@ def get_doc(doc: SolrDocument): ) -def work_object(w): # called by works_by_author - ia = w.get('ia', []) - obj = dict( - authors=[ - web.storage(key='/authors/' + k, name=n) - for k, n in zip(w['author_key'], w['author_name']) - ], - edition_count=w['edition_count'], - key=w['key'], - title=w['title'], - public_scan=w.get('public_scan_b', bool(ia)), - lending_edition=w.get('lending_edition_s', ''), - lending_identifier=w.get('lending_identifier_s', ''), - collections=set( - w['ia_collection_s'].split(';') if 'ia_collection_s' in w else [] - ), - url=w['key'] + '/' + urlsafe(w['title']), - cover_edition_key=w.get('cover_edition_key'), - first_publish_year=( - w['first_publish_year'] if 'first_publish_year' in w else None - ), - ia=w.get('ia', []), - cover_i=w.get('cover_i'), - id_project_gutenberg=w.get('id_project_gutenberg'), - id_librivox=w.get('id_librivox'), - id_standard_ebooks=w.get('id_standard_ebooks'), - id_openstax=w.get('id_openstax'), - ) - - for f in 'has_fulltext', 'subtitle': - if w.get(f): - obj[f] = w[f] - return web.storage(obj) - - class scan(delegate.page): """ Experimental EAN barcode scanner page to scan and add/view books by their barcodes. @@ -1008,150 +1001,50 @@ def GET(self): def works_by_author( - akey, sort='editions', page=1, rows=100, has_fulltext=False, query=None + akey: str, + sort='editions', + page=1, + rows=100, + facet=False, + has_fulltext=False, + query: str = None, ): - # called by merge_author_works - q = 'author_key:' + akey - if query: - q = query - - offset = rows * (page - 1) - params = [ - ('fq', 'author_key:' + akey), - ('fq', 'type:work'), - ('q', q), - ('start', offset), - ('rows', rows), - ( - 'fl', - ','.join( - [ - 'key', - 'author_name', - 'author_key', - 'title', - 'subtitle', - 'edition_count', - 'ia', - 'cover_edition_key', - 'has_fulltext', - 'language', - 'first_publish_year', - 'public_scan_b', - 'lending_edition_s', - 'lending_identifier_s', - 'ia_collection_s', - 'id_project_gutenberg', - 'id_librivox', - 'id_standard_ebooks', - 'id_openstax', - 'cover_i', - ] - ), - ), - ('wt', 'json'), - ('q.op', 'AND'), - ('facet', 'true'), - ('facet.mincount', 1), - ('f.author_facet.facet.sort', 'count'), - ('f.publish_year.facet.limit', -1), - ('facet.limit', 25), - ] - + param = {'q': query or '*:*'} if has_fulltext: - params.append(('fq', 'has_fulltext:true')) - - if sort == "editions": - params.append(('sort', 'edition_count desc')) - elif sort.startswith('old'): - params.append(('sort', 'first_publish_year asc')) - elif sort.startswith('new'): - params.append(('sort', 'first_publish_year desc')) - elif sort.startswith('title'): - params.append(('sort', 'title_sort asc')) - - facet_fields = [ - "author_facet", - "language", - "publish_year", - "publisher_facet", - "subject_facet", - "person_facet", - "place_facet", - "time_facet", - ] - for f in facet_fields: - params.append(("facet.field", f)) - - reply = parse_json_from_solr_query(solr_select_url, params) - if reply is None: - return web.storage( - num_found=0, - works=[], - years=[], - get_facet=[], - sort=sort, - ) - # TODO: Deep JSON structure defense - for now, let it blow up so easier to detect - facets = reply['facet_counts']['facet_fields'] - works = [work_object(w) for w in reply['response']['docs']] - - def get_facet(f, limit=None): - return list(web.group(facets[f][: limit * 2] if limit else facets[f], 2)) + param['has_fulltext'] = 'true' - return web.storage( - num_found=int(reply['response']['numFound']), - works=add_availability(works), - years=[(int(k), v) for k, v in get_facet('publish_year')], - get_facet=get_facet, + result = run_solr_query( + param=param, + page=page, + rows=rows, sort=sort, + facet=( + facet + and [ + "subject_facet", + "person_facet", + "place_facet", + "time_facet", + ] + ), + extra_params=[ + ('fq', f'author_key:{akey}'), + ('facet.limit', 25), + ], ) - -def sorted_work_editions(wkey, json_data=None): - """Setting json_data to a real value simulates getting SOLR data back, i.e. for testing (but ick!)""" - q = 'key:' + wkey - if json_data: - reply = json.loads(json_data) - else: - reply = parse_json_from_solr_query( - solr_select_url, - { - 'q.op': 'AND', - 'q': q, - 'rows': 10, - 'fl': 'edition_key', - 'qt': 'standard', - 'wt': 'json', - }, - ) - if reply is None or reply.get('response', {}).get('numFound', 0) == 0: - return [] - # TODO: Deep JSON structure defense - for now, let it blow up so easier to detect - return reply["response"]['docs'][0].get('edition_key', []) + result.docs = add_availability([get_doc(doc) for doc in result.docs]) + return result -def top_books_from_author(akey, rows=5, offset=0): - q = 'author_key:(' + akey + ')' - json_result = parse_json_from_solr_query( - solr_select_url, - { - 'q': q, - 'start': offset, - 'rows': rows, - 'fl': 'key,title,edition_count,first_publish_year', - 'sort': 'edition_count desc', - 'wt': 'json', - }, +def top_books_from_author(akey: str, rows=5) -> SearchResponse: + return run_solr_query( + {'q': f'author_key:{akey}'}, + fields=['key', 'title', 'edition_count', 'first_publish_year'], + sort='editions', + rows=rows, + facet=False, ) - if json_result is None: - return {'books': [], 'total': 0} - # TODO: Deep JSON structure defense - for now, let it blow up so easier to detect - response = json_result['response'] - return { - 'books': [web.storage(doc) for doc in response['docs']], - 'total': response['numFound'], - } class advancedsearch(delegate.page): @@ -1436,29 +1329,22 @@ def work_search( # Ensure we don't mutate the `query` passed in by reference query = copy.deepcopy(query) query['wt'] = 'json' - if sort: - sort = process_sort(sort) # deal with special /lists/ key queries query['q'], page, offset, limit = rewrite_list_query( query['q'], page, offset, limit ) - try: - (reply, solr_select) = run_solr_query( - query, - rows=limit, - page=page, - sort=sort, - offset=offset, - fields=fields, - facet=facet, - spellcheck_count=spellcheck_count, - ) - assert reply, "Received None response from run_solr_query" - response = reply['response'] - except (ValueError, OSError, AssertionError) as e: - logger.error("Error in processing search API.") - response = dict(start=0, numFound=0, docs=[], error=str(e)) + resp = run_solr_query( + query, + rows=limit, + page=page, + sort=sort, + offset=offset, + fields=fields, + facet=facet, + spellcheck_count=spellcheck_count, + ) + response = resp.raw_resp['response'] # backward compatibility response['num_found'] = response['numFound'] @@ -1527,19 +1413,9 @@ def GET(self): def setup(): - from openlibrary.plugins.worksearch import subjects - - # subjects module needs read_author_facet and solr_select_url. - # Importing this module to access them will result in circular import. - # Setting them like this to avoid circular-import. - subjects.read_author_facet = read_author_facet - if hasattr(config, 'plugin_worksearch'): - subjects.solr_select_url = solr_select_url + from openlibrary.plugins.worksearch import subjects, languages, publishers subjects.setup() - - from openlibrary.plugins.worksearch import languages, publishers - publishers.setup() languages.setup() diff --git a/openlibrary/plugins/worksearch/search.py b/openlibrary/plugins/worksearch/search.py index 985e4dea856..85bba1f2eb9 100644 --- a/openlibrary/plugins/worksearch/search.py +++ b/openlibrary/plugins/worksearch/search.py @@ -2,103 +2,13 @@ """ from openlibrary.utils.solr import Solr from infogami import config -from infogami.utils import stats -import web -import logging +_ACTIVE_SOLR: Solr | None = None -def get_solr(): - base_url = config.plugin_worksearch.get('solr_base_url') - return Solr(base_url) - - -def work_search(query, limit=20, offset=0, **kw): - """Search for works.""" - - kw.setdefault("doc_wrapper", work_wrapper) - kw.setdefault("fq", "type:work") - - fields = [ - "key", - "author_name", - "author_key", - "title", - "edition_count", - "ia", - "cover_edition_key", - "has_fulltext", - "subject", - "ia_collection_s", - "public_scan_b", - "lending_edition_s", - "lending_identifier_s", - ] - kw.setdefault("fields", fields) - - query = process_work_query(query) - solr = get_solr() - - stats.begin("solr", query=query, start=offset, rows=limit, kw=kw) - try: - result = solr.select(query, start=offset, rows=limit, **kw) - except Exception as e: - logging.getLogger("openlibrary").exception("Failed solr query") - return None - finally: - stats.end() - - return result - - -def process_work_query(query): - if "author" in query and isinstance(query["author"], dict): - author = query.pop("author") - query["author_key"] = author["key"] - ebook = query.pop("ebook", None) - if ebook == True or ebook == "true": - query["has_fulltext"] = "true" - - return query - - -def work_wrapper(w): - key = w['key'] - if not key.startswith("/works/"): - key += "/works/" - - d = web.storage(key=key, title=w["title"], edition_count=w["edition_count"]) - - if "cover_id" in w: - d.cover_id = w["cover_id"] - elif "cover_edition_key" in w: - book = web.ctx.site.get("/books/" + w["cover_edition_key"]) - cover = book and book.get_cover() - d.cover_id = cover and cover.id or None - d.cover_edition_key = w['cover_edition_key'] - else: - d.cover_id = None - d.subject = w.get('subject', []) - ia_collection = w['ia_collection_s'].split(';') if 'ia_collection_s' in w else [] - d.ia_collection = ia_collection - d.lendinglibrary = 'lendinglibrary' in ia_collection - d.printdisabled = 'printdisabled' in ia_collection - d.lending_edition = w.get('lending_edition_s', '') - d.lending_identifier = w.get('lending_identifier_s', '') - - # special care to handle missing author_key/author_name in the solr record - w.setdefault('author_key', []) - w.setdefault('author_name', []) - - d.authors = [ - web.storage(key='/authors/' + k, name=n) - for k, n in zip(w['author_key'], w['author_name']) - ] - - d.first_publish_year = ( - w['first_publish_year'][0] if 'first_publish_year' in w else None - ) - d.ia = w.get('ia', []) - d.public_scan = w.get('public_scan_b', bool(d.ia)) - d.has_fulltext = w.get('has_fulltext', "false") - return d +def get_solr(): + global _ACTIVE_SOLR + if not _ACTIVE_SOLR: + base_url = config.plugin_worksearch.get('solr_base_url') + _ACTIVE_SOLR = Solr(base_url) + return _ACTIVE_SOLR diff --git a/openlibrary/plugins/worksearch/subjects.py b/openlibrary/plugins/worksearch/subjects.py index 6b70d4b2aae..7689414d7f2 100644 --- a/openlibrary/plugins/worksearch/subjects.py +++ b/openlibrary/plugins/worksearch/subjects.py @@ -1,38 +1,21 @@ """Subject pages. """ - import web -import re -import requests import json -import logging -from collections import defaultdict import datetime -from infogami import config from infogami.plugins.api.code import jsonapi -from infogami.utils import delegate, stats -from infogami.utils.view import render, render_template, safeint +from infogami.utils import delegate +from infogami.utils.view import render_template, safeint from openlibrary.core.models import Subject from openlibrary.core.lending import add_availability -from openlibrary.plugins.worksearch.search import work_search +from openlibrary.solr.query_utils import query_dict_to_str from openlibrary.utils import str_to_key, finddict __all__ = ["SubjectEngine", "get_subject"] -# These two are available in .code module. Importing it here will result in a -# circular import. To avoid that, these values are set by the code.setup -# function. -read_author_facet = None -solr_select_url = None - -logger = logging.getLogger("openlibrary.worksearch") - -re_chars = re.compile("([%s])" % re.escape(r'+-!(){}[]^"~*?:\\')) -re_year = re.compile(r'\b(\d+)$') - SUBJECTS = [ web.storage( name="person", @@ -144,7 +127,7 @@ def GET(self, key): begin, end = i.published_in.split('-', 1) if safeint(begin, None) is not None and safeint(end, None) is not None: - filters['publish_year'] = (begin, end) # range + filters['publish_year'] = f'[{begin} TO {end}]' else: y = safeint(i.published_in, None) if y is not None: @@ -158,7 +141,7 @@ def GET(self, key): details=i.details.lower() == 'true', **filters, ) - if i.has_fulltext: + if i.has_fulltext == 'true': subject_results['ebook_count'] = subject_results['work_count'] return json.dumps(subject_results) @@ -170,7 +153,12 @@ def process_key(self, key): def get_subject( - key, details=False, offset=0, sort='editions', limit=DEFAULT_RESULTS, **filters + key: str, + details=False, + offset=0, + sort='editions', + limit=DEFAULT_RESULTS, + **filters, ): """Returns data related to a subject. @@ -235,15 +223,9 @@ def create_engine(): return Engine() return SubjectEngine() - sort_options = { - 'editions': 'edition_count desc', - 'new': 'first_publish_year desc', - } - sort_order = sort_options.get(sort) or sort_options['editions'] - engine = create_engine() subject_results = engine.get_subject( - key, details=details, offset=offset, sort=sort_order, limit=limit, **filters + key, details=details, offset=offset, sort=sort, limit=limit, **filters ) return subject_results @@ -255,59 +237,113 @@ def get_subject( details=False, offset=0, limit=DEFAULT_RESULTS, - sort='first_publish_year desc', + sort='new', **filters, ): - meta = self.get_meta(key) + # Circular imports are everywhere -_- + from openlibrary.plugins.worksearch.code import run_solr_query - q = self.make_query(key, filters) + meta = self.get_meta(key) subject_type = meta.name name = meta.path.replace("_", " ") - if details: - kw = self.query_optons_for_details() - else: - kw = {} - - result = work_search(q, offset=offset, limit=limit, sort=sort, **kw) - if not result: - return None - - for w in result.docs: - w.ia = w.ia and w.ia[0] or None - - # XXX-Anand: Oct 2013 - # Somewhere something is broken, work keys are coming as OL1234W/works/ - # Quick fix it solve that issue. - if w.key.endswith("/works/"): - w.key = "/works/" + w.key.replace("/works/", "") + unescaped_filters = {} + if 'publish_year' in filters: + # Don't want this escaped or used in fq for perf reasons + unescaped_filters['publish_year'] = filters.pop('publish_year') + result = run_solr_query( + { + 'q': query_dict_to_str( + {meta.facet_key: self.normalize_key(meta.path)}, + unescaped=unescaped_filters, + ), + **filters, + }, + offset=offset, + rows=limit, + sort=sort, + fields=[ + "key", + "author_name", + "author_key", + "title", + "edition_count", + "ia", + "cover_i", + "first_publish_year", + "cover_edition_key", + "has_fulltext", + "subject", + "ia_collection_s", + "public_scan_b", + "lending_edition_s", + "lending_identifier_s", + ], + facet=( + details + and [ + {"name": "author_facet", "sort": "count"}, + "language", + "publisher_facet", + {"name": "publish_year", "limit": -1}, + "subject_facet", + "person_facet", + "place_facet", + "time_facet", + "has_fulltext", + ] + ), + extra_params=[ + ('facet.mincount', 1), + ('facet.limit', 25), + ], + allowed_filter_params=[ + 'has_fulltext', + 'publish_year', + ], + ) subject = Subject( key=key, name=name, subject_type=subject_type, - work_count=result['num_found'], - works=add_availability(result['docs']), + work_count=result.num_found, + works=add_availability([self.work_wrapper(d) for d in result.docs]), ) if details: - subject.ebook_count = dict(result.facets["has_fulltext"]).get("true", 0) + result.facet_counts = { + facet_field: [ + self.facet_wrapper(facet_field, key, label, count) + for key, label, count in facet_counts + ] + for facet_field, facet_counts in result.facet_counts.items() + } + + subject.ebook_count = next( + ( + count + for key, count in result.facet_counts["has_fulltext"] + if key == "true" + ), + 0, + ) - subject.subjects = result.facets["subject_facet"] - subject.places = result.facets["place_facet"] - subject.people = result.facets["person_facet"] - subject.times = result.facets["time_facet"] + subject.subjects = result.facet_counts["subject_facet"] + subject.places = result.facet_counts["place_facet"] + subject.people = result.facet_counts["person_facet"] + subject.times = result.facet_counts["time_facet"] - subject.authors = result.facets["author_facet"] - subject.publishers = result.facets["publisher_facet"] - subject.languages = result.facets['language'] + subject.authors = result.facet_counts["author_key"] + subject.publishers = result.facet_counts["publisher_facet"] + subject.languages = result.facet_counts['language'] # Ignore bad dates when computing publishing_history # year < 1000 or year > current_year+1 are considered bad dates current_year = datetime.datetime.utcnow().year subject.publishing_history = [ [year, count] - for year, count in result.facets["publish_year"] + for year, count in result.facet_counts["publish_year"] if 1000 < year <= current_year + 1 ] @@ -335,31 +371,18 @@ def parse_key(self, key): return d.prefix, key[len(d.prefix) :] return None, None - def make_query(self, key, filters): - meta = self.get_meta(key) - - q = {meta.facet_key: self.normalize_key(meta.path)} - - if filters: - if filters.get("has_fulltext") == "true": - q['has_fulltext'] = "true" - if filters.get("publish_year"): - q['publish_year'] = filters['publish_year'] - return q - def normalize_key(self, key): return str_to_key(key).lower() - def facet_wrapper(self, facet, value, count): + def facet_wrapper(self, facet: str, value: str, label: str, count: int): if facet == "publish_year": return [int(value), count] elif facet == "publisher_facet": return web.storage( name=value, count=count, key="/publishers/" + value.replace(" ", "_") ) - elif facet == "author_facet": - author = read_author_facet(value) - return web.storage(name=author[1], key="/authors/" + author[0], count=count) + elif facet == "author_key": + return web.storage(name=label, key=f"/authors/{value}", count=count) elif facet in ["subject_facet", "person_facet", "place_facet", "time_facet"]: return web.storage( key=finddict(SUBJECTS, facet=facet).prefix @@ -372,25 +395,35 @@ def facet_wrapper(self, facet, value, count): else: return web.storage(name=value, count=count) - def query_optons_for_details(self): - """Additional query options to be added when details=True.""" - kw = {} - kw['facets'] = [ - {"name": "author_facet", "sort": "count"}, - "language", - "publisher_facet", - {"name": "publish_year", "limit": -1}, - "subject_facet", - "person_facet", - "place_facet", - "time_facet", - "has_fulltext", - "language", - ] - kw['facet.mincount'] = 1 - kw['facet.limit'] = 25 - kw['facet_wrapper'] = self.facet_wrapper - return kw + @staticmethod + def work_wrapper(w: dict) -> web.storage: + """ + Convert a solr document into the doc returned by the /subjects APIs. + These docs are weird :/ We should be using more standardized results + across our search APIs, but that would be a big breaking change. + """ + ia_collection = w.get('ia_collection_s', '').split(';') + return web.storage( + key=w['key'], + title=w["title"], + edition_count=w["edition_count"], + cover_id=w.get('cover_i'), + cover_edition_key=w.get('cover_edition_key'), + subject=w.get('subject', []), + ia_collection=ia_collection, + lendinglibrary='lendinglibrary' in ia_collection, + printdisabled='printdisabled' in ia_collection, + lending_edition=w.get('lending_edition_s', ''), + lending_identifier=w.get('lending_identifier_s', ''), + authors=[ + web.storage(key=f'/authors/{olid}', name=name) + for olid, name in zip(w.get('author_key', []), w.get('author_name', [])) + ], + first_publish_year=w.get('first_publish_year'), + ia=w.get('ia', [None])[0], + public_scan=w.get('public_scan_b', bool(w.get('ia'))), + has_fulltext=w.get('has_fulltext', False), + ) def setup(): diff --git a/openlibrary/plugins/worksearch/tests/test_worksearch.py b/openlibrary/plugins/worksearch/tests/test_worksearch.py index d89671ef6c4..93f3a626063 100644 --- a/openlibrary/plugins/worksearch/tests/test_worksearch.py +++ b/openlibrary/plugins/worksearch/tests/test_worksearch.py @@ -3,7 +3,6 @@ from openlibrary.plugins.worksearch.code import ( process_facet, process_user_query, - sorted_work_editions, escape_bracket, get_doc, escape_colon, @@ -32,24 +31,6 @@ def test_process_facet(): ] -def test_sorted_work_editions(): - json_data = '''{ -"responseHeader":{ -"status":0, -"QTime":1, -"params":{ -"fl":"edition_key", -"indent":"on", -"wt":"json", -"q":"key:OL100000W"}}, -"response":{"numFound":1,"start":0,"docs":[ -{ - "edition_key":["OL7536692M","OL7825368M","OL3026366M"]}] -}}''' - expect = ["OL7536692M", "OL7825368M", "OL3026366M"] - assert sorted_work_editions('OL100000W', json_data=json_data) == expect - - # {'Test name': ('query', fields[])} QUERY_PARSER_TESTS = { 'No fields': ('query here', 'query here'), diff --git a/openlibrary/solr/query_utils.py b/openlibrary/solr/query_utils.py index 393b881b443..da26909fdfa 100644 --- a/openlibrary/solr/query_utils.py +++ b/openlibrary/solr/query_utils.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional +from typing import Callable, Literal, Optional from luqum.parser import parser from luqum.tree import Item, SearchField, BaseOperation, Group, Word, Unary import re @@ -121,7 +121,7 @@ def fully_escape_query(query: str) -> str: """ escaped = query # Escape special characters - escaped = re.sub(r'[\[\]\(\)\{\}:"-+?~^/\\]', r'\\\g<0>', escaped) + escaped = re.sub(r'[\[\]\(\)\{\}:"-+?~^/\\,]', r'\\\g<0>', escaped) # Remove boolean operators by making them lowercase escaped = re.sub(r'AND|OR|NOT', lambda _1: _1.group(0).lower(), escaped) return escaped @@ -213,3 +213,34 @@ def find_next_word(item: Item) -> Optional[tuple[Word, Optional[BaseOperation]]] node.expr.head = '' return tree + + +def query_dict_to_str( + escaped: dict = None, + unescaped: dict = None, + op: Literal['AND', 'OR', ''] = '', +) -> str: + """ + Converts a query dict to a search query. + + >>> query_dict_to_str({'title': 'foo'}) + 'title:(foo)' + >>> query_dict_to_str({'title': 'foo bar', 'author': 'bar'}) + 'title:(foo bar) author:(bar)' + >>> query_dict_to_str({'title': 'foo bar', 'author': 'bar'}, op='OR') + 'title:(foo bar) OR author:(bar)' + >>> query_dict_to_str({'title': 'foo ? to escape'}) + 'title:(foo \\\\? to escape)' + >>> query_dict_to_str({'title': 'YES AND'}) + 'title:(YES and)' + """ + result = '' + if escaped: + result += f' {op} '.join( + f'{k}:({fully_escape_query(v)})' for k, v in escaped.items() + ) + if unescaped: + if result: + result += f' {op} ' + result += f' {op} '.join(f'{k}:{v}' for k, v in unescaped.items()) + return result diff --git a/openlibrary/templates/account/sidebar.html b/openlibrary/templates/account/sidebar.html index d9cac7d6b54..868223fc17c 100644 --- a/openlibrary/templates/account/sidebar.html +++ b/openlibrary/templates/account/sidebar.html @@ -11,10 +11,7 @@
diff --git a/openlibrary/templates/admin/people/view.html b/openlibrary/templates/admin/people/view.html index 2b4efba58cd..c29a398d78c 100644 --- a/openlibrary/templates/admin/people/view.html +++ b/openlibrary/templates/admin/people/view.html @@ -110,8 +110,8 @@

People / $person.username

Status: $person.status $if person.status in ['active', 'verified']: - - + +   

diff --git a/openlibrary/templates/barcodescanner.html b/openlibrary/templates/barcodescanner.html index bc6d47c8e9f..a9cdd1f6ac9 100644 --- a/openlibrary/templates/barcodescanner.html +++ b/openlibrary/templates/barcodescanner.html @@ -3,6 +3,7 @@ $var title: $_('Barcode Scanner (Beta)') $ ctx.setdefault('cssfile', 'barcodescanner') +$ ctx.setdefault("show_ol_shell", query_param('returnTo') is None)
diff --git a/openlibrary/templates/covers/add.html b/openlibrary/templates/covers/add.html index 155a274a5b1..19ab0387555 100644 --- a/openlibrary/templates/covers/add.html +++ b/openlibrary/templates/covers/add.html @@ -1,6 +1,7 @@ $def with (doc, data={}, status=None) $putctx('cssfile', 'form') +$putctx("show_ol_shell", False) $putctx('robots', 'noindex,nofollow') $if doc.type.key == "/type/author": diff --git a/openlibrary/templates/covers/manage.html b/openlibrary/templates/covers/manage.html index 6e24ffeb919..79eff299669 100644 --- a/openlibrary/templates/covers/manage.html +++ b/openlibrary/templates/covers/manage.html @@ -1,6 +1,7 @@ $def with (key, images) $putctx("cssfile", "form") +$putctx("show_ol_shell", False) $putctx('robots', 'noindex,nofollow')
diff --git a/openlibrary/templates/covers/saved.html b/openlibrary/templates/covers/saved.html index 1042545de99..6f662f2c393 100644 --- a/openlibrary/templates/covers/saved.html +++ b/openlibrary/templates/covers/saved.html @@ -1,6 +1,7 @@ $def with (image, showinfo=True) $putctx("cssfile", "form") +$putctx("show_ol_shell", False)
diff --git a/openlibrary/templates/lib/header_dropdown.html b/openlibrary/templates/lib/header_dropdown.html index 07fee238b48..ad989a7d532 100644 --- a/openlibrary/templates/lib/header_dropdown.html +++ b/openlibrary/templates/lib/header_dropdown.html @@ -1,9 +1,9 @@ -$def with (props) +$def with (props, track_prefix=None) $# @typedef {Object} DropdownLink $# @property {string} text of link $# @property {string|null} href of link for HTTP get request $# @property {string|null} post when set will be used instead of href and should use HTTP POST request. -$# @property {string|null} `track` for google analytics +$# @property {string|null} `track` event label for google analytics $# $# @param {Object} props template properties $# @param {string} props.name unique identifying name for dropdown and distinguishing it from other dropdowns @@ -16,7 +16,7 @@
$if singleton: $ link = props['links'][0] - $ tracker = link.get('track') and 'data-ol-link-track=%s' % link.get('track') or '' + $ tracker = 'data-ol-link-track=%s|%s' % (track_prefix, link['track']) if (track_prefix and link.get('track')) else '' $(props['label']) $else:
@@ -44,7 +44,7 @@ >
    $for link in props['links']: - $ tracker = link.get('track') and 'data-ol-link-track=%s' % link.get('track') or '' + $ tracker = 'data-ol-link-track=%s|%s' % (track_prefix, link['track']) if (track_prefix and link.get('track')) else '' $if 'subsection' in link:
  • $(link['subsection']) diff --git a/openlibrary/templates/lib/nav_head.html b/openlibrary/templates/lib/nav_head.html index d2e9d5f0def..865884b08c0 100644 --- a/openlibrary/templates/lib/nav_head.html +++ b/openlibrary/templates/lib/nav_head.html @@ -6,38 +6,38 @@ $if ctx.user: $ loginLinks = [ - $ { "href": ctx.user.key, "text": _("My Profile"), "track": "MainNav|MyProfile" }, - $ { "href": homepath() + "/account/loans", "text": _("My Loans"), "track": "MainNav|MyLoans" }, - $ { "href": "/account/books", "text": _("My Reading Log"), "track": "MainNav|MyReadingLog" }, - $ { "href": "/account/lists", "text": _("My Lists"), "track": "MainNav|MyLists" }, - $ { "href": "/account/books/already-read/stats", "text": _("My Reading Stats"), "track": "MainNav|MyReadingStats" }, - $ { "href": homepath() + "/account", "text": _("Settings"), "track": "MainNav|MySettings" }, - $ { "post": "/account/logout", "text": _("Log out"), "track": "MainNav|Logout" }, + $ { "href": ctx.user.key, "text": _("My Profile"), "track": "MyProfile" }, + $ { "href": homepath() + "/account/loans", "text": _("My Loans"), "track": "MyLoans" }, + $ { "href": "/account/books", "text": _("My Reading Log"), "track": "MyReadingLog" }, + $ { "href": "/account/lists", "text": _("My Lists"), "track": "MyLists" }, + $ { "href": "/account/books/already-read/stats", "text": _("My Reading Stats"), "track": "MyReadingStats" }, + $ { "href": homepath() + "/account", "text": _("Settings"), "track": "MySettings" }, + $ { "post": "/account/logout", "text": _("Log out"), "track": "Logout" }, $ ] $if is_privileged_user: - $ loginLinks.insert(1,{ "href": homepath() + "/merges", "text": _("Pending Requests"),"track": "MainNav|MyRequests" }) + $ loginLinks.insert(1,{ "href": homepath() + "/merges", "text": _("Pending Requests"),"track": "MyRequests" }) $else: $ loginLinks = [ $ { "loginClass": "login-links" } $ ] $ myBooksLinks = [ - $ { "href": homepath() + "/account/loans", "text": _("My Books"), "track": "MainNav|MyBooks" }, + $ { "href": homepath() + "/account/loans", "text": _("My Books"), "track": "MyBooks" }, $ ] $ browseLinks = [ - $ { "href": "/subjects", "text": _("Subjects"), "track": "MainNav|Subjects" }, - $ { "href": "/trending", "text": _("Trending"), "track": "MainNav|Trending" }, - $ { "href": "/explore", "text": _("Library Explorer"), "track": "MainNav|Explore" }, - $ { "href": "/lists", "text": _("Lists"), "track": "MainNav|Lists" }, - $ { "href": "/collections", "text": _("Collections"), "track": "MainNav|Collections" }, - $ { "href": "/k-12", "text": _("K-12 Student Library"), "track": "MainNav|K12Library" }, - $ { "href": "/random", "text": _("Random Book"), "track": "MainNav|RandomBook" }, - $ { "href": "/advancedsearch", "text": _("Advanced Search"), "track": "MainNav|AdvancedSearch" } + $ { "href": "/subjects", "text": _("Subjects"), "track": "Subjects" }, + $ { "href": "/trending", "text": _("Trending"), "track": "Trending" }, + $ { "href": "/explore", "text": _("Library Explorer"), "track": "Explore" }, + $ { "href": "/lists", "text": _("Lists"), "track": "Lists" }, + $ { "href": "/collections", "text": _("Collections"), "track": "Collections" }, + $ { "href": "/k-12", "text": _("K-12 Student Library"), "track": "K12Library" }, + $ { "href": "/random", "text": _("Random Book"), "track": "RandomBook" }, + $ { "href": "/advancedsearch", "text": _("Advanced Search"), "track": "AdvancedSearch" } $ ] $ moreLinks = [ - $ { "href": "/books/add", "text": _("Add a Book"), "track": "MainNav|AddBook" }, - $ { "href": "/recentchanges", "text": _("Recent Community Edits"), "track": "MainNav|RecentEdits" }, - $ { "href": "/developers", "text": _("Developer Center"), "track": "MainNav|Developers" }, - $ { "href": "/help", "text": _("Help & Support"), "track": "MainNav|Help" } + $ { "href": "/books/add", "text": _("Add a Book"), "track": "AddBook" }, + $ { "href": "/recentchanges", "text": _("Recent Community Edits"), "track": "RecentEdits" }, + $ { "href": "/developers", "text": _("Developer Center"), "track": "Developers" }, + $ { "href": "/help", "text": _("Help & Support"), "track": "Help" } $ ] $ hamburgerProps = { $ 'name': 'hamburger', @@ -77,8 +77,8 @@ $ 'links': myBooksLinks $ }
      -
    • $:render_template("lib/header_dropdown", myBooksProps )
      • -
    • $:render_template("lib/header_dropdown", browseProps )
      • +
    • $:render_template("lib/header_dropdown", myBooksProps, track_prefix="MainNav")
      • +
    • $:render_template("lib/header_dropdown", browseProps, track_prefix="MainNav")
    @@ -119,12 +119,12 @@
  • $_("Sign Up")
- $:render_template("lib/header_dropdown", hamburgerProps ) + $:render_template("lib/header_dropdown", hamburgerProps, track_prefix="Hamburger")
    -
  • $:render_template("lib/header_dropdown", myBooksProps )
    • -
  • $:render_template("lib/header_dropdown", browseProps )
    • +
  • $:render_template("lib/header_dropdown", myBooksProps, track_prefix="MainNav")
    • +
  • $:render_template("lib/header_dropdown", browseProps, track_prefix="MainNav")
diff --git a/openlibrary/templates/merge/authors.html b/openlibrary/templates/merge/authors.html index 5c77441d209..488163ecc0e 100644 --- a/openlibrary/templates/merge/authors.html +++ b/openlibrary/templates/merge/authors.html @@ -78,15 +78,15 @@

$_("Merge Authors")

$a.date
    - $for doc in top['books']: -
  • $doc.title $ungettext('1 edition', '%(count)d editions', doc.edition_count, count=doc.edition_count), + $for doc in top.docs: +
  • $doc['title'] $ungettext('1 edition', '%(count)d editions', doc['edition_count'], count=doc['edition_count']), $if doc.get('first_publish_year'): - $doc.first_publish_year + $doc['first_publish_year']
- $ungettext("1 work", "%(count)d works", top['total'], count=top['total']) + $ungettext("1 work", "%(count)d works", top.num_found, count=top.num_found)
@@ -94,7 +94,11 @@

$_("Merge Authors")

- + $if can_merge: + $ cta = _('Merge Authors') + $else: + $ cta = _('Request Merge') + $if mrid: $else: diff --git a/openlibrary/templates/merge_queue/merge_queue.html b/openlibrary/templates/merge_queue/merge_queue.html index afd16e93340..c3a905eec11 100644 --- a/openlibrary/templates/merge_queue/merge_queue.html +++ b/openlibrary/templates/merge_queue/merge_queue.html @@ -13,7 +13,7 @@ $if submitter: $ desc = _("Showing %(username)s's requests only.", username=submitter) $ link_text = _('Show all requests') - $ href = changequery(submitter=None) + $ href = changequery(submitter=None, page=None) $else: $ desc = _('Showing all requests.') $ link_text = _('Show my requests') if username else '' @@ -23,7 +23,14 @@

$_('Community Edit Requests')

- $desc $link_text + $if can_merge: + + $if reviewer: + $_("Showing requests reviewed by %(reviewer)s only.", reviewer=reviewer) $_("Remove reviewer filter") + $else: + $_("Show requests that I've reviewed") + + $desc $link_text
$ page = int(input(page=1).page) diff --git a/openlibrary/templates/search/inside.html b/openlibrary/templates/search/inside.html index 0caacba857d..5c2de8d702f 100644 --- a/openlibrary/templates/search/inside.html +++ b/openlibrary/templates/search/inside.html @@ -10,7 +10,7 @@

$_("Search Inside")

$:macros.SearchNavigation()
- +
$ num_found = 0 $if q: diff --git a/openlibrary/templates/search/subjects.html b/openlibrary/templates/search/subjects.html index afe4d02abcc..0adf57554b2 100644 --- a/openlibrary/templates/search/subjects.html +++ b/openlibrary/templates/search/subjects.html @@ -21,7 +21,7 @@

- +
diff --git a/openlibrary/templates/site/body.html b/openlibrary/templates/site/body.html index c70ae80fb39..9d3553b1a98 100644 --- a/openlibrary/templates/site/body.html +++ b/openlibrary/templates/site/body.html @@ -1,8 +1,7 @@ $def with (page) -$ cssfile = ctx.get('cssfile', 'user') $ bodyclass = ctx.get('bodyclass', []) -$ show_banners = cssfile != 'form' +$ show_ol_shell = ctx.get('show_ol_shell', True) $ bodyattrs = ctx.get('bodyattrs', []) $if ctx.path.startswith('/works/OL') or ctx.path.startswith('/authors/OL') or ctx.path.startswith('/books/OL') or ctx.path.startswith('/search'): @@ -21,12 +20,12 @@
$_("It looks like you're offline.")
$# on form pages e.g. manage-covers, add-cover we do not display the header $# this is consistent with version 1. - $if show_banners: + $if show_ol_shell: $:render_template("site/alert") $:render_template("lib/nav_head", None) $# don't render test-body-mobile for iframes -
- $if show_banners: +
+ $if show_ol_shell: $#print errors (hidden by default as styles are loaded via JS)
$for flash in get_flash_messages(): diff --git a/openlibrary/templates/site/footer.html b/openlibrary/templates/site/footer.html index ed0bc321633..3588751d135 100644 --- a/openlibrary/templates/site/footer.html +++ b/openlibrary/templates/site/footer.html @@ -1,6 +1,6 @@ $def with (page=None) -$ cssfile = ctx.get('cssfile', 'user') +$ show_ol_shell = ctx.get('show_ol_shell', True) $ stats = stats_summary() @@ -10,55 +10,57 @@ $else: $ total_time = 0 -
+$if show_ol_shell: +
-$if cssfile == 'user' or 'admin': - $if ("stats" in ctx.features) and query_param('debug'): - $:render_template("site/stats") - $:render_template("lib/nav_foot", page) +$if ("stats" in ctx.features) and query_param('debug'): + $:render_template("site/stats") - - + }); + } - - - - -
+ window.addEventListener('beforeinstallprompt', (e) => { + // Prevent the mini-infobar from appearing on mobile + e.preventDefault(); + }); + - $if any([path in request.canonical_url for path in ['/account/create', '/books/add', '/edit', '/books', '/contact']]): - - - + + + + +
- - - if (typeof archive_analytics !== 'undefined') { - archive_analytics.set_up_event_tracking(); + + + window.q = []; +} ); + +if (typeof archive_analytics !== 'undefined') { + archive_analytics.set_up_event_tracking(); +} + diff --git a/openlibrary/templates/type/author/view.html b/openlibrary/templates/type/author/view.html index 37b8d6dd7fd..d537995a500 100644 --- a/openlibrary/templates/type/author/view.html +++ b/openlibrary/templates/type/author/view.html @@ -22,8 +22,8 @@ $ books = page.get_books(q=query_param('q')) $ book_list_excerpt = '' -$if books and books.works: - $ book_titles_excerpt = [work.title for work in books.works[:8]] +$if books and books.docs: + $ book_titles_excerpt = [work.title for work in books.docs[:8]] $ book_list_excerpt = ', '.join(book_titles_excerpt) $ description = _("Author of %(book_titles)s", book_titles=book_list_excerpt) $putctx("description", description) @@ -142,7 +142,7 @@

    - $for doc in books.works: + $for doc in books.docs: $:macros.SearchResultsWork(doc)
@@ -160,16 +160,16 @@

$if subjects:
$label
- $for subject, count in subjects: + $for _, subject, count in subjects: $subject$cond(not loop.last, ",", "")
$if books.num_found > 0: - $:render_subjects(_("Subjects"), books.get_facet('subject_facet'), '') - $:render_subjects(_("Places"), books.get_facet('place_facet'), 'place:') - $:render_subjects(_("People"), books.get_facet('person_facet'), 'person:') - $:render_subjects(_("Time"), books.get_facet('time_facet'), 'time:') + $:render_subjects(_("Subjects"), books.facet_counts.get('subject_facet'), '') + $:render_subjects(_("Places"), books.facet_counts.get('place_facet'), 'place:') + $:render_subjects(_("People"), books.facet_counts.get('person_facet'), 'person:') + $:render_subjects(_("Time"), books.facet_counts.get('time_facet'), 'time:') $if "lists" in ctx.features: diff --git a/openlibrary/templates/type/list/embed.html b/openlibrary/templates/type/list/embed.html index 5600e547e6d..c67cc46ab54 100644 --- a/openlibrary/templates/type/list/embed.html +++ b/openlibrary/templates/type/list/embed.html @@ -3,6 +3,7 @@ $var title: $list.name $putctx('cssfile', 'form') +$putctx("show_ol_shell", False) $ page = safeint(query_param('page'), 1) - 1 $ page_size = 10 diff --git a/openlibrary/templates/work_search.html b/openlibrary/templates/work_search.html index 1a50ec1ab3b..6cd027f98a0 100644 --- a/openlibrary/templates/work_search.html +++ b/openlibrary/templates/work_search.html @@ -44,7 +44,7 @@
-

$_("Search")

+

$_("Search Books")

$ facet_map = ( diff --git a/openlibrary/utils/solr.py b/openlibrary/utils/solr.py index f5df1212938..2d23f8c332f 100644 --- a/openlibrary/utils/solr.py +++ b/openlibrary/utils/solr.py @@ -8,42 +8,20 @@ import requests import web -import urllib +from urllib.parse import urlencode, urlsplit logger = logging.getLogger("openlibrary.logger") -def urlencode(d, doseq=False): - """There is a bug in urllib when used with unicode data. - - >>> d = {"q": u"\u0C05"} - >>> urllib.parse.urlencode(d) - 'q=%E0%B0%85' - >>> urllib.parse.urlencode(d, doseq=True) - 'q=%3F' - - This function encodes all the unicode strings in utf-8 before passing them to urllib. - """ - - def utf8(d): - if isinstance(d, dict): - return {utf8(k): utf8(v) for k, v in d.items()} - elif isinstance(d, list): - return [utf8(v) for v in d] - else: - return web.safestr(d) - - return urllib.parse.urlencode(utf8(d), doseq=doseq) - - T = TypeVar('T') class Solr: def __init__(self, base_url): self.base_url = base_url - self.host = urllib.parse.urlsplit(self.base_url)[1] + self.host = urlsplit(self.base_url)[1] + self.session = requests.Session() def escape(self, query): r"""Escape special characters in the query string @@ -64,7 +42,7 @@ def get( ) -> Optional[T]: """Get a specific item from solr""" logger.info(f"solr /get: {key}, {fields}") - resp = requests.get( + resp = self.session.get( f"{self.base_url}/get", params={'id': key, **({'fl': ','.join(fields)} if fields else {})}, ).json() @@ -81,7 +59,7 @@ def get_many( if not keys: return [] logger.info(f"solr /get: {keys}, {fields}") - resp = requests.get( + resp = self.session.get( f"{self.base_url}/get", params={ 'ids': ','.join(keys), @@ -141,15 +119,13 @@ def select( if len(payload) < 500: url = url + "?" + payload logger.info("solr request: %s", url) - json_data = requests.get(url, timeout=10).json() + json_data = self.session.get(url, timeout=10).json() else: logger.info("solr request: %s ...", url) - if not isinstance(payload, bytes): - payload = payload.encode("utf-8") headers = { "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8" } - json_data = requests.post( + json_data = self.session.post( url, data=payload, headers=headers, timeout=10 ).json() return self._parse_solr_result( diff --git a/package-lock.json b/package-lock.json index bee08b9d6ae..872ff355594 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@babel/plugin-syntax-dynamic-import": "7.8.3", "@babel/preset-env": "7.14.9", "@babel/register": "7.15.3", + "@ericblade/quagga2": "^1.7.4", "@storybook/addon-actions": "6.3.8", "@storybook/addon-essentials": "6.3.8", "@storybook/addon-links": "6.3.8", @@ -49,9 +50,8 @@ "lucene-query-parser": "1.2.0", "npm-watch": "0.11.0", "promise-polyfill": "8.2.0", - "quagga": "0.12.1", "regenerator-runtime": "0.13.7", - "sinon": "13.0.1", + "sinon": "14.0.0", "slick-carousel": "1.6.0", "style-loader": "2.0.0", "stylelint": "13.13.1", @@ -3344,6 +3344,25 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/polyfill": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/polyfill/-/polyfill-7.12.1.tgz", + "integrity": "sha512-X0pi0V6gxLi6lFZpGmeNa4zxtwEmCs42isWLNjZZDE0Y8yVfgu0T2OAHlzBbdYlqbW/YXVvoBHpATEM+goCj8g==", + "deprecated": "🚨 This package has been deprecated in favor of separate inclusion of a polyfill and regenerator-runtime (when needed). See the @babel/polyfill docs (https://babeljs.io/docs/en/babel-polyfill) for more information.", + "dev": true, + "dependencies": { + "core-js": "^2.6.5", + "regenerator-runtime": "^0.13.4" + } + }, + "node_modules/@babel/polyfill/node_modules/core-js": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", + "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.", + "dev": true, + "hasInstallScript": true + }, "node_modules/@babel/preset-env": { "version": "7.14.9", "dev": true, @@ -4147,6 +4166,28 @@ "dev": true, "license": "MIT" }, + "node_modules/@ericblade/quagga2": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@ericblade/quagga2/-/quagga2-1.7.4.tgz", + "integrity": "sha512-7rBY0mPgV0iAIyrXUrijbitzglculdOizpj2IljvbhRoABvybZCJGXCxmnO7Vwuju7LclwRNZLPu+E0xmOFmSQ==", + "dev": true, + "dependencies": { + "@babel/polyfill": "^7.12.1", + "get-pixels": "^3.3.3", + "gl-mat2": "^1.0.1", + "gl-vec2": "^1.3.0", + "gl-vec3": "^1.1.3", + "lodash": "^4.17.21", + "ndarray": "^1.0.19", + "ndarray-linear-interpolate": "^1.0.0" + }, + "engines": { + "node": ">= 10.0" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, "node_modules/@gar/promisify": { "version": "1.1.2", "dev": true, @@ -11921,18 +11962,6 @@ "node": ">=4" } }, - "node_modules/@vue/cli-shared-utils/node_modules/har-validator": { - "version": "5.1.5", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^6.12.3", - "har-schema": "^2.0.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/@vue/cli-shared-utils/node_modules/parse-json": { "version": "5.2.0", "dev": true, @@ -11964,36 +11993,6 @@ "node": ">=8" } }, - "node_modules/@vue/cli-shared-utils/node_modules/request": { - "version": "2.88.2", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.3", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.5.0", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/@vue/cli-shared-utils/node_modules/semver": { "version": "6.3.0", "dev": true, @@ -12013,18 +12012,6 @@ "node": ">=8" } }, - "node_modules/@vue/cli-shared-utils/node_modules/tough-cookie": { - "version": "2.5.0", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "psl": "^1.1.28", - "punycode": "^2.1.1" - }, - "engines": { - "node": ">=0.8" - } - }, "node_modules/@vue/cli-shared-utils/node_modules/type-fest": { "version": "0.6.0", "dev": true, @@ -15130,6 +15117,16 @@ "node": ">=8" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "optional": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bl": { "version": "1.2.3", "dev": true, @@ -17976,8 +17973,9 @@ }, "node_modules/cwise-compiler": { "version": "1.1.3", + "resolved": "https://registry.npmjs.org/cwise-compiler/-/cwise-compiler-1.1.3.tgz", + "integrity": "sha512-WXlK/m+Di8DMMcCjcWr4i+XzcQra9eCdXIJrgh4TUgh0pIS/yJduLxS9JgefsHJ/YVLdgPtXm9r62W92MvanEQ==", "dev": true, - "license": "MIT", "dependencies": { "uniq": "^1.0.0" } @@ -18000,8 +17998,9 @@ }, "node_modules/data-uri-to-buffer": { "version": "0.0.3", - "dev": true, - "license": "MIT" + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-0.0.3.tgz", + "integrity": "sha512-Cp+jOa8QJef5nXS5hU7M1DWzXPEIoVR3kbV0dQuVGwROZg8bGf1DcCnkmajBTnvghTtSNMUdRrPjgaT6ZQucbw==", + "dev": true }, "node_modules/data-urls": { "version": "2.0.0", @@ -20346,6 +20345,13 @@ "node": ">=6" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "optional": true + }, "node_modules/filename-reserved-regex": { "version": "2.0.0", "dev": true, @@ -20770,6 +20776,20 @@ "dev": true, "license": "ISC" }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/fswin": { "version": "2.17.1227", "dev": true, @@ -21051,12 +21071,13 @@ } }, "node_modules/get-pixels": { - "version": "3.3.2", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/get-pixels/-/get-pixels-3.3.3.tgz", + "integrity": "sha512-5kyGBn90i9tSMUVHTqkgCHsoWoR+/lGbl4yC83Gefyr0HLIhgSWEx/2F/3YgsZ7UpYNuM6pDhDK7zebrUJ5nXg==", "dev": true, - "license": "MIT", "dependencies": { "data-uri-to-buffer": "0.0.3", - "jpeg-js": "^0.3.2", + "jpeg-js": "^0.4.1", "mime-types": "^2.0.1", "ndarray": "^1.0.13", "ndarray-pack": "^1.1.1", @@ -21206,18 +21227,21 @@ }, "node_modules/gl-mat2": { "version": "1.0.1", - "dev": true, - "license": "zlib" + "resolved": "https://registry.npmjs.org/gl-mat2/-/gl-mat2-1.0.1.tgz", + "integrity": "sha512-oHgZ3DalAo9qAhMZM9QigXosqotcUCsgxarwrinipaqfSHvacI79Dzs72gY+oT4Td1kDQKEsG0RyX6mb02VVHA==", + "dev": true }, "node_modules/gl-vec2": { "version": "1.3.0", - "dev": true, - "license": "zlib" + "resolved": "https://registry.npmjs.org/gl-vec2/-/gl-vec2-1.3.0.tgz", + "integrity": "sha512-YiqaAuNsheWmUV0Sa8k94kBB0D6RWjwZztyO+trEYS8KzJ6OQB/4686gdrf59wld4hHFIvaxynO3nRxpk1Ij/A==", + "dev": true }, "node_modules/gl-vec3": { "version": "1.1.3", - "dev": true, - "license": "zlib" + "resolved": "https://registry.npmjs.org/gl-vec3/-/gl-vec3-1.1.3.tgz", + "integrity": "sha512-jduKUqT0SGH02l8Yl+mV1yVsDfYgQAJyXGxkJQGyxPLHRiW25DwVIRPt6uvhrEMHftJfqhqKthRcyZqNEl9Xdw==", + "dev": true }, "node_modules/glob": { "version": "7.1.3", @@ -21751,45 +21775,27 @@ }, "node_modules/har-schema": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==", "dev": true, - "license": "ISC", "engines": { "node": ">=4" } }, "node_modules/har-validator": { - "version": "5.1.0", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "deprecated": "this library is no longer supported", "dev": true, - "license": "ISC", "dependencies": { - "ajv": "^5.3.0", + "ajv": "^6.12.3", "har-schema": "^2.0.0" }, "engines": { - "node": ">=4" - } - }, - "node_modules/har-validator/node_modules/ajv": { - "version": "5.5.2", - "dev": true, - "license": "MIT", - "dependencies": { - "co": "^4.6.0", - "fast-deep-equal": "^1.0.0", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.3.0" + "node": ">=6" } }, - "node_modules/har-validator/node_modules/fast-deep-equal": { - "version": "1.1.0", - "dev": true, - "license": "MIT" - }, - "node_modules/har-validator/node_modules/json-schema-traverse": { - "version": "0.3.1", - "dev": true, - "license": "MIT" - }, "node_modules/hard-rejection": { "version": "2.1.0", "dev": true, @@ -22980,8 +22986,9 @@ }, "node_modules/iota-array": { "version": "1.0.0", - "dev": true, - "license": "MIT" + "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz", + "integrity": "sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==", + "dev": true }, "node_modules/ip": { "version": "1.1.5", @@ -27076,9 +27083,10 @@ } }, "node_modules/jpeg-js": { - "version": "0.3.7", - "dev": true, - "license": "BSD-3-Clause" + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "dev": true }, "node_modules/jquery": { "version": "3.6.0", @@ -28654,8 +28662,9 @@ }, "node_modules/ndarray": { "version": "1.0.19", + "resolved": "https://registry.npmjs.org/ndarray/-/ndarray-1.0.19.tgz", + "integrity": "sha512-B4JHA4vdyZU30ELBw3g7/p9bZupyew5a7tX1Y/gGeF2hafrPaQZhgrGQfsvgfYbgdFZjYwuEcnaobeM/WMW+HQ==", "dev": true, - "license": "MIT", "dependencies": { "iota-array": "^1.0.0", "is-buffer": "^1.0.2" @@ -28663,13 +28672,15 @@ }, "node_modules/ndarray-linear-interpolate": { "version": "1.0.0", - "dev": true, - "license": "MIT" + "resolved": "https://registry.npmjs.org/ndarray-linear-interpolate/-/ndarray-linear-interpolate-1.0.0.tgz", + "integrity": "sha512-UN0f4+6XWsQzJ2pP5gVp+kKn5tJed6mA3K/L50uO619+7LKrjcSNdcerhpqxYaSkbxNJuEN76N05yBBJySnZDw==", + "dev": true }, "node_modules/ndarray-pack": { "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ndarray-pack/-/ndarray-pack-1.2.1.tgz", + "integrity": "sha512-51cECUJMT0rUZNQa09EoKsnFeDL4x2dHRT0VR5U2H5ZgEcm95ZDWcMA5JShroXjHOejmAD/fg8+H+OvUnVXz2g==", "dev": true, - "license": "MIT", "dependencies": { "cwise-compiler": "^1.1.2", "ndarray": "^1.0.13" @@ -28804,6 +28815,8 @@ }, "node_modules/node-bitmap": { "version": "0.0.1", + "resolved": "https://registry.npmjs.org/node-bitmap/-/node-bitmap-0.0.1.tgz", + "integrity": "sha512-Jx5lPaaLdIaOsj2mVLWMWulXF6GQVdyLvNSxmiYCvZ8Ma2hfKX0POoR2kgKOqz+oFsRreq0yYZjQ2wjE9VNzCA==", "dev": true, "engines": { "node": ">=v0.6.5" @@ -29664,8 +29677,9 @@ }, "node_modules/omggif": { "version": "1.0.10", - "dev": true, - "license": "MIT" + "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", + "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", + "dev": true }, "node_modules/on-finished": { "version": "2.3.0", @@ -30032,8 +30046,9 @@ }, "node_modules/parse-data-uri": { "version": "0.2.0", + "resolved": "https://registry.npmjs.org/parse-data-uri/-/parse-data-uri-0.2.0.tgz", + "integrity": "sha512-uOtts8NqDcaCt1rIsO3VFDRsAfgE4c6osG4d9z3l4dCBlxYFzni6Di/oNU270SDrjkfZuUvLZx1rxMyqh46Y9w==", "dev": true, - "license": "ISC", "dependencies": { "data-uri-to-buffer": "0.0.3" } @@ -30536,8 +30551,9 @@ }, "node_modules/pngjs": { "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", "dev": true, - "license": "MIT", "engines": { "node": ">=4.0.0" } @@ -33121,23 +33137,6 @@ "node": ">=0.6" } }, - "node_modules/quagga": { - "version": "0.12.1", - "dev": true, - "license": "MIT", - "dependencies": { - "get-pixels": "^3.2.3", - "gl-mat2": "^1.0.0", - "gl-vec2": "^1.0.0", - "gl-vec3": "^1.0.3", - "lodash": "^4.17.4", - "ndarray": "^1.0.18", - "ndarray-linear-interpolate": "^1.0.0" - }, - "engines": { - "node": ">= 4.0" - } - }, "node_modules/query-string": { "version": "5.1.1", "dev": true, @@ -34440,9 +34439,11 @@ } }, "node_modules/request": { - "version": "2.88.0", + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", "dev": true, - "license": "Apache-2.0", "dependencies": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -34451,7 +34452,7 @@ "extend": "~3.0.2", "forever-agent": "~0.6.1", "form-data": "~2.3.2", - "har-validator": "~5.1.0", + "har-validator": "~5.1.3", "http-signature": "~1.2.0", "is-typedarray": "~1.0.0", "isstream": "~0.1.2", @@ -34461,12 +34462,12 @@ "performance-now": "^2.1.0", "qs": "~6.5.2", "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", + "tough-cookie": "~2.5.0", "tunnel-agent": "^0.6.0", "uuid": "^3.3.2" }, "engines": { - "node": ">= 4" + "node": ">= 6" } }, "node_modules/require-directory": { @@ -35254,12 +35255,13 @@ "license": "MIT" }, "node_modules/sinon": { - "version": "13.0.1", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-14.0.0.tgz", + "integrity": "sha512-ugA6BFmE+WrJdh0owRZHToLd32Uw3Lxq6E6LtNRU+xTVBefx632h03Q7apXWRsRdZAJ41LB8aUfn2+O4jsDNMw==", "dev": true, - "license": "BSD-3-Clause", "dependencies": { "@sinonjs/commons": "^1.8.3", - "@sinonjs/fake-timers": "^9.0.0", + "@sinonjs/fake-timers": "^9.1.2", "@sinonjs/samsam": "^6.1.1", "diff": "^5.0.0", "nise": "^5.1.1", @@ -35271,33 +35273,37 @@ } }, "node_modules/sinon/node_modules/@sinonjs/fake-timers": { - "version": "9.1.0", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-9.1.2.tgz", + "integrity": "sha512-BPS4ynJW/o92PUR4wgriz2Ud5gpST5vz6GQfMixEDK0Z8ZCUv2M7SkBLykH56T++Xs+8ln9zTGbOvNGIe02/jw==", "dev": true, - "license": "BSD-3-Clause", "dependencies": { "@sinonjs/commons": "^1.7.0" } }, "node_modules/sinon/node_modules/diff": { - "version": "5.0.0", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", "dev": true, - "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } }, "node_modules/sinon/node_modules/has-flag": { "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true, - "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/sinon/node_modules/supports-color": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, - "license": "MIT", "dependencies": { "has-flag": "^4.0.0" }, @@ -38856,22 +38862,18 @@ } }, "node_modules/tough-cookie": { - "version": "2.4.3", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", "dev": true, - "license": "BSD-3-Clause", "dependencies": { - "psl": "^1.1.24", - "punycode": "^1.4.1" + "psl": "^1.1.28", + "punycode": "^2.1.1" }, "engines": { "node": ">=0.8" } }, - "node_modules/tough-cookie/node_modules/punycode": { - "version": "1.4.1", - "dev": true, - "license": "MIT" - }, "node_modules/tr46": { "version": "2.1.0", "dev": true, @@ -41185,6 +41187,25 @@ "fsevents": "^1.2.7" } }, + "node_modules/watchpack-chokidar2/node_modules/fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "deprecated": "fsevents 1 will break on node v14+ and could be using insecure binaries. Upgrade to fsevents 2.", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "dependencies": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + }, + "engines": { + "node": ">= 4.0" + } + }, "node_modules/watchpack-chokidar2/node_modules/is-binary-path": { "version": "1.0.1", "dev": true, @@ -41769,6 +41790,25 @@ "node": ">=6" } }, + "node_modules/webpack-dev-server/node_modules/fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "deprecated": "fsevents 1 will break on node v14+ and could be using insecure binaries. Upgrade to fsevents 2.", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "dependencies": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + }, + "engines": { + "node": ">= 4.0" + } + }, "node_modules/webpack-dev-server/node_modules/is-absolute-url": { "version": "3.0.3", "dev": true, @@ -45772,6 +45812,24 @@ "@babel/helper-plugin-utils": "^7.14.5" } }, + "@babel/polyfill": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/polyfill/-/polyfill-7.12.1.tgz", + "integrity": "sha512-X0pi0V6gxLi6lFZpGmeNa4zxtwEmCs42isWLNjZZDE0Y8yVfgu0T2OAHlzBbdYlqbW/YXVvoBHpATEM+goCj8g==", + "dev": true, + "requires": { + "core-js": "^2.6.5", + "regenerator-runtime": "^0.13.4" + }, + "dependencies": { + "core-js": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", + "dev": true + } + } + }, "@babel/preset-env": { "version": "7.14.9", "dev": true, @@ -46340,6 +46398,23 @@ "version": "0.2.5", "dev": true }, + "@ericblade/quagga2": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@ericblade/quagga2/-/quagga2-1.7.4.tgz", + "integrity": "sha512-7rBY0mPgV0iAIyrXUrijbitzglculdOizpj2IljvbhRoABvybZCJGXCxmnO7Vwuju7LclwRNZLPu+E0xmOFmSQ==", + "dev": true, + "requires": { + "@babel/polyfill": "^7.12.1", + "fsevents": "2.3.2", + "get-pixels": "^3.3.3", + "gl-mat2": "^1.0.1", + "gl-vec2": "^1.3.0", + "gl-vec3": "^1.1.3", + "lodash": "^4.17.21", + "ndarray": "^1.0.19", + "ndarray-linear-interpolate": "^1.0.0" + } + }, "@gar/promisify": { "version": "1.1.2", "dev": true @@ -51541,14 +51616,6 @@ "supports-color": "^5.3.0" } }, - "har-validator": { - "version": "5.1.5", - "dev": true, - "requires": { - "ajv": "^6.12.3", - "har-schema": "^2.0.0" - } - }, "parse-json": { "version": "5.2.0", "dev": true, @@ -51569,32 +51636,6 @@ "type-fest": "^0.6.0" } }, - "request": { - "version": "2.88.2", - "dev": true, - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.3", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.5.0", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - } - }, "semver": { "version": "6.3.0", "dev": true @@ -51606,14 +51647,6 @@ "ansi-regex": "^5.0.0" } }, - "tough-cookie": { - "version": "2.5.0", - "dev": true, - "requires": { - "psl": "^1.1.28", - "punycode": "^2.1.1" - } - }, "type-fest": { "version": "0.6.0", "dev": true @@ -53674,6 +53707,16 @@ "version": "2.1.0", "dev": true }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, "bl": { "version": "1.2.3", "dev": true, @@ -55640,6 +55683,8 @@ }, "cwise-compiler": { "version": "1.1.3", + "resolved": "https://registry.npmjs.org/cwise-compiler/-/cwise-compiler-1.1.3.tgz", + "integrity": "sha512-WXlK/m+Di8DMMcCjcWr4i+XzcQra9eCdXIJrgh4TUgh0pIS/yJduLxS9JgefsHJ/YVLdgPtXm9r62W92MvanEQ==", "dev": true, "requires": { "uniq": "^1.0.0" @@ -55658,6 +55703,8 @@ }, "data-uri-to-buffer": { "version": "0.0.3", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-0.0.3.tgz", + "integrity": "sha512-Cp+jOa8QJef5nXS5hU7M1DWzXPEIoVR3kbV0dQuVGwROZg8bGf1DcCnkmajBTnvghTtSNMUdRrPjgaT6ZQucbw==", "dev": true }, "data-urls": { @@ -57293,6 +57340,13 @@ "version": "8.1.0", "dev": true }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "optional": true + }, "filename-reserved-regex": { "version": "2.0.0", "dev": true @@ -57582,6 +57636,13 @@ "version": "1.0.0", "dev": true }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + }, "fswin": { "version": "2.17.1227", "dev": true @@ -57762,11 +57823,13 @@ "dev": true }, "get-pixels": { - "version": "3.3.2", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/get-pixels/-/get-pixels-3.3.3.tgz", + "integrity": "sha512-5kyGBn90i9tSMUVHTqkgCHsoWoR+/lGbl4yC83Gefyr0HLIhgSWEx/2F/3YgsZ7UpYNuM6pDhDK7zebrUJ5nXg==", "dev": true, "requires": { "data-uri-to-buffer": "0.0.3", - "jpeg-js": "^0.3.2", + "jpeg-js": "^0.4.1", "mime-types": "^2.0.1", "ndarray": "^1.0.13", "ndarray-pack": "^1.1.1", @@ -57878,14 +57941,20 @@ }, "gl-mat2": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gl-mat2/-/gl-mat2-1.0.1.tgz", + "integrity": "sha512-oHgZ3DalAo9qAhMZM9QigXosqotcUCsgxarwrinipaqfSHvacI79Dzs72gY+oT4Td1kDQKEsG0RyX6mb02VVHA==", "dev": true }, "gl-vec2": { "version": "1.3.0", + "resolved": "https://registry.npmjs.org/gl-vec2/-/gl-vec2-1.3.0.tgz", + "integrity": "sha512-YiqaAuNsheWmUV0Sa8k94kBB0D6RWjwZztyO+trEYS8KzJ6OQB/4686gdrf59wld4hHFIvaxynO3nRxpk1Ij/A==", "dev": true }, "gl-vec3": { "version": "1.1.3", + "resolved": "https://registry.npmjs.org/gl-vec3/-/gl-vec3-1.1.3.tgz", + "integrity": "sha512-jduKUqT0SGH02l8Yl+mV1yVsDfYgQAJyXGxkJQGyxPLHRiW25DwVIRPt6uvhrEMHftJfqhqKthRcyZqNEl9Xdw==", "dev": true }, "glob": { @@ -58259,34 +58328,18 @@ }, "har-schema": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==", "dev": true }, "har-validator": { - "version": "5.1.0", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", "dev": true, "requires": { - "ajv": "^5.3.0", + "ajv": "^6.12.3", "har-schema": "^2.0.0" - }, - "dependencies": { - "ajv": { - "version": "5.5.2", - "dev": true, - "requires": { - "co": "^4.6.0", - "fast-deep-equal": "^1.0.0", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.3.0" - } - }, - "fast-deep-equal": { - "version": "1.1.0", - "dev": true - }, - "json-schema-traverse": { - "version": "0.3.1", - "dev": true - } } }, "hard-rejection": { @@ -59101,6 +59154,8 @@ }, "iota-array": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz", + "integrity": "sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==", "dev": true }, "ip": { @@ -61720,7 +61775,9 @@ } }, "jpeg-js": { - "version": "0.3.7", + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", "dev": true }, "jquery": { @@ -62833,6 +62890,8 @@ }, "ndarray": { "version": "1.0.19", + "resolved": "https://registry.npmjs.org/ndarray/-/ndarray-1.0.19.tgz", + "integrity": "sha512-B4JHA4vdyZU30ELBw3g7/p9bZupyew5a7tX1Y/gGeF2hafrPaQZhgrGQfsvgfYbgdFZjYwuEcnaobeM/WMW+HQ==", "dev": true, "requires": { "iota-array": "^1.0.0", @@ -62841,10 +62900,14 @@ }, "ndarray-linear-interpolate": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ndarray-linear-interpolate/-/ndarray-linear-interpolate-1.0.0.tgz", + "integrity": "sha512-UN0f4+6XWsQzJ2pP5gVp+kKn5tJed6mA3K/L50uO619+7LKrjcSNdcerhpqxYaSkbxNJuEN76N05yBBJySnZDw==", "dev": true }, "ndarray-pack": { "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ndarray-pack/-/ndarray-pack-1.2.1.tgz", + "integrity": "sha512-51cECUJMT0rUZNQa09EoKsnFeDL4x2dHRT0VR5U2H5ZgEcm95ZDWcMA5JShroXjHOejmAD/fg8+H+OvUnVXz2g==", "dev": true, "requires": { "cwise-compiler": "^1.1.2", @@ -62956,6 +63019,8 @@ }, "node-bitmap": { "version": "0.0.1", + "resolved": "https://registry.npmjs.org/node-bitmap/-/node-bitmap-0.0.1.tgz", + "integrity": "sha512-Jx5lPaaLdIaOsj2mVLWMWulXF6GQVdyLvNSxmiYCvZ8Ma2hfKX0POoR2kgKOqz+oFsRreq0yYZjQ2wjE9VNzCA==", "dev": true }, "node-dir": { @@ -63513,6 +63578,8 @@ }, "omggif": { "version": "1.0.10", + "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", + "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", "dev": true }, "on-finished": { @@ -63758,6 +63825,8 @@ }, "parse-data-uri": { "version": "0.2.0", + "resolved": "https://registry.npmjs.org/parse-data-uri/-/parse-data-uri-0.2.0.tgz", + "integrity": "sha512-uOtts8NqDcaCt1rIsO3VFDRsAfgE4c6osG4d9z3l4dCBlxYFzni6Di/oNU270SDrjkfZuUvLZx1rxMyqh46Y9w==", "dev": true, "requires": { "data-uri-to-buffer": "0.0.3" @@ -64108,6 +64177,8 @@ }, "pngjs": { "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", "dev": true }, "pnp-webpack-plugin": { @@ -65874,19 +65945,6 @@ "version": "6.5.2", "dev": true }, - "quagga": { - "version": "0.12.1", - "dev": true, - "requires": { - "get-pixels": "^3.2.3", - "gl-mat2": "^1.0.0", - "gl-vec2": "^1.0.0", - "gl-vec3": "^1.0.3", - "lodash": "^4.17.4", - "ndarray": "^1.0.18", - "ndarray-linear-interpolate": "^1.0.0" - } - }, "query-string": { "version": "5.1.1", "dev": true, @@ -66744,7 +66802,9 @@ "dev": true }, "request": { - "version": "2.88.0", + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "dev": true, "requires": { "aws-sign2": "~0.7.0", @@ -66754,7 +66814,7 @@ "extend": "~3.0.2", "forever-agent": "~0.6.1", "form-data": "~2.3.2", - "har-validator": "~5.1.0", + "har-validator": "~5.1.3", "http-signature": "~1.2.0", "is-typedarray": "~1.0.0", "isstream": "~0.1.2", @@ -66764,7 +66824,7 @@ "performance-now": "^2.1.0", "qs": "~6.5.2", "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", + "tough-cookie": "~2.5.0", "tunnel-agent": "^0.6.0", "uuid": "^3.3.2" } @@ -67327,11 +67387,13 @@ } }, "sinon": { - "version": "13.0.1", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-14.0.0.tgz", + "integrity": "sha512-ugA6BFmE+WrJdh0owRZHToLd32Uw3Lxq6E6LtNRU+xTVBefx632h03Q7apXWRsRdZAJ41LB8aUfn2+O4jsDNMw==", "dev": true, "requires": { "@sinonjs/commons": "^1.8.3", - "@sinonjs/fake-timers": "^9.0.0", + "@sinonjs/fake-timers": "^9.1.2", "@sinonjs/samsam": "^6.1.1", "diff": "^5.0.0", "nise": "^5.1.1", @@ -67339,22 +67401,30 @@ }, "dependencies": { "@sinonjs/fake-timers": { - "version": "9.1.0", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-9.1.2.tgz", + "integrity": "sha512-BPS4ynJW/o92PUR4wgriz2Ud5gpST5vz6GQfMixEDK0Z8ZCUv2M7SkBLykH56T++Xs+8ln9zTGbOvNGIe02/jw==", "dev": true, "requires": { "@sinonjs/commons": "^1.7.0" } }, "diff": { - "version": "5.0.0", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", "dev": true }, "has-flag": { "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, "supports-color": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, "requires": { "has-flag": "^4.0.0" @@ -69652,17 +69722,13 @@ } }, "tough-cookie": { - "version": "2.4.3", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", "dev": true, "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - }, - "dependencies": { - "punycode": { - "version": "1.4.1", - "dev": true - } + "psl": "^1.1.28", + "punycode": "^2.1.1" } }, "tr46": { @@ -71184,6 +71250,17 @@ "upath": "^1.1.1" } }, + "fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "dev": true, + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } + }, "is-binary-path": { "version": "1.0.1", "dev": true, @@ -71600,6 +71677,17 @@ "locate-path": "^3.0.0" } }, + "fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "dev": true, + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } + }, "is-absolute-url": { "version": "3.0.3", "dev": true diff --git a/package.json b/package.json index f554501fb55..0fdd11ecf0a 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "@babel/plugin-syntax-dynamic-import": "7.8.3", "@babel/preset-env": "7.14.9", "@babel/register": "7.15.3", + "@ericblade/quagga2": "^1.7.4", "@storybook/addon-actions": "6.3.8", "@storybook/addon-essentials": "6.3.8", "@storybook/addon-links": "6.3.8", @@ -75,9 +76,8 @@ "lucene-query-parser": "1.2.0", "npm-watch": "0.11.0", "promise-polyfill": "8.2.0", - "quagga": "0.12.1", "regenerator-runtime": "0.13.7", - "sinon": "13.0.1", + "sinon": "14.0.0", "slick-carousel": "1.6.0", "style-loader": "2.0.0", "stylelint": "13.13.1", @@ -114,6 +114,5 @@ } }, "collectCoverage": true - }, - "dependencies": {} + } } diff --git a/scripts/deployment/are_servers_in_sync.sh b/scripts/deployment/are_servers_in_sync.sh index 725c7bc42c6..fe25b7fd5f6 100755 --- a/scripts/deployment/are_servers_in_sync.sh +++ b/scripts/deployment/are_servers_in_sync.sh @@ -7,7 +7,7 @@ for REPO_DIR in $REPO_DIRS; do echo $REPO_DIR for SERVER in $SERVERS; do - ssh $SERVER "cd $REPO_DIR; echo -ne $SERVER'\t'; sudo git rev-parse HEAD" + ssh $SERVER "cd $REPO_DIR; echo -ne $SERVER'\t'; sudo git status ; sudo git rev-parse HEAD" done echo "---" done diff --git a/scripts/deployment/pre_deploy.sh b/scripts/deployment/pre_deploy.sh new file mode 100755 index 00000000000..0fabd4d4dea --- /dev/null +++ b/scripts/deployment/pre_deploy.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +SERVERS="ol-home0 ol-covers0 ol-web1 ol-web2 ol-www0 ol-solr0" +REPO_DIRS="/opt/olsystem /opt/openlibrary /opt/openlibrary/vendor/infogami /opt/booklending_utils" + +for REPO_DIR in $REPO_DIRS; do + echo $REPO_DIR + + for SERVER in $SERVERS; do + ssh $SERVER "cd $REPO_DIR; echo -ne $SERVER'\t'; sudo git status + done + echo "---" +done diff --git a/scripts/partner_batch_imports.py b/scripts/partner_batch_imports.py index 76375662847..62ac733bd9e 100755 --- a/scripts/partner_batch_imports.py +++ b/scripts/partner_batch_imports.py @@ -51,11 +51,26 @@ EXCLUDED_INDEPENDENTLY_PUBLISHED_TITLES = { x.casefold() for x in ( + # Noisy classic re-prints 'annotated', 'annoté', + 'classic', + 'classics', + 'illustarted', # Some books have typos in their titles! 'illustrated', 'Illustrée', + 'original', + 'summary', + 'version', + # Not a book + 'calendar', + 'diary', + 'journal', + 'logbook', 'notebook', + 'notizbuch', + 'planner', + 'sketchbook', ) } diff --git a/scripts/update_stale_work_references.py b/scripts/update_stale_work_references.py new file mode 100644 index 00000000000..bf90d0b1a87 --- /dev/null +++ b/scripts/update_stale_work_references.py @@ -0,0 +1,25 @@ +""" +PYTHONPATH=. python ./scripts/update_stale_work_references.py /olsystem/etc/openlibrary.yml +""" + +import web +import infogami +from infogami import config # noqa: F401 +from openlibrary.config import load_config +from openlibrary.core.models import Work +from scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI +import datetime + + +def main(ol_config: str, start_offset=0, days=31): + load_config(ol_config) + infogami._setup() + cutoff_date = datetime.datetime.today() - datetime.timedelta(days=days) + Work.resolve_redirects_bulk( + start_offset=start_offset, + cutoff_date=cutoff_date, + ) + + +if __name__ == '__main__': + FnToCLI(main).run() diff --git a/static/css/components/merge-request-table.less b/static/css/components/merge-request-table.less index c82d4214b6d..189128e315b 100644 --- a/static/css/components/merge-request-table.less +++ b/static/css/components/merge-request-table.less @@ -2,7 +2,13 @@ padding: 0 10px 10px; .description { + display: flex; + flex-direction: column; margin-bottom: 10px; + + .reviewer-filter { + margin-bottom: 10px; + } } } diff --git a/static/css/page-barcodescanner.less b/static/css/page-barcodescanner.less index 80cc2ee023b..e7b208de55d 100644 --- a/static/css/page-barcodescanner.less +++ b/static/css/page-barcodescanner.less @@ -38,7 +38,11 @@ width: 100%; video, canvas { width: 100%; - height: 100vh; + // stylelint-disable declaration-block-no-duplicate-properties + // Fallback browsers that don't support dvh + height: calc(100vh - 60px); + height: 100dvh; // stylelint-disable-line unit-no-unknown + // stylelint-enable declaration-block-no-duplicate-properties object-fit: cover; } canvas { @@ -46,6 +50,10 @@ top: 0; left: 0; } + + video[controls] + canvas { + pointer-events: none; + } } #result-strip { font-family: @lucida_sans_serif-1; diff --git a/static/css/page-form.less b/static/css/page-form.less index 1eb92fd4217..03ae4565934 100644 --- a/static/css/page-form.less +++ b/static/css/page-form.less @@ -23,7 +23,6 @@ body { /* stylelint-disable selector-max-specificity */ div#revertNotice, div#revertLink, -footer, #bottom { display: none; } diff --git a/tests/unit/js/editionsEditPage.test.js b/tests/unit/js/editionsEditPage.test.js index 98d5412e8a4..8b964b80378 100644 --- a/tests/unit/js/editionsEditPage.test.js +++ b/tests/unit/js/editionsEditPage.test.js @@ -75,14 +75,14 @@ beforeEach(() => { // Per the test data used, and beforeEach(), the length always starts out at 5. describe('initIdentifierValidation', () => { // ISBN 10 - it('it does add a valid ISBN 10 ending in X', () => { + it('does add a valid ISBN 10 ending in X', () => { $('#select-id').val('isbn_10'); $('#id-value').val('0-8044-2957-X'); $('.repeat-add').trigger('click'); expect($('.repeat-item').length).toBe(6); }); - it('it does add a valid ISBN 10 NOT ending in X', () => { + it('does add a valid ISBN 10 NOT ending in X', () => { $('#select-id').val('isbn_10'); $('#id-value').val('0596520689'); $('.repeat-add').trigger('click'); @@ -96,7 +96,7 @@ describe('initIdentifierValidation', () => { expect($('.repeat-item').length).toBe(5); }); - it('it does NOT prompt to add a formally invalid ISBN 10', () => { + it('does NOT prompt to add a formally invalid ISBN 10', () => { $('#select-id').val('isbn_10'); $('#id-value').val('12345'); $('.repeat-add').trigger('click'); @@ -136,7 +136,7 @@ describe('initIdentifierValidation', () => { expect($('.repeat-item').length).toBe(6); }) - it('it does count identical stripped and unstripped ISBN 10s as the same ISBN', () => { + it('does count identical stripped and unstripped ISBN 10s as the same ISBN', () => { $('#select-id').val('isbn_10'); $('#id-value').val(' 144--93-55730 '); $('.repeat-add').trigger('click'); @@ -147,7 +147,7 @@ describe('initIdentifierValidation', () => { }); // ISBN 13 - it('it does add a valid ISBN 13', () => { + it('does add a valid ISBN 13', () => { $('#select-id').val('isbn_13'); $('#id-value').val('9781789801217'); $('.repeat-add').trigger('click'); @@ -161,7 +161,7 @@ describe('initIdentifierValidation', () => { expect($('.repeat-item').length).toBe(5); }); - it('it does NOT prompt to add a formally invalid ISBN 13', () => { + it('does NOT prompt to add a formally invalid ISBN 13', () => { $('#select-id').val('isbn_13'); $('#id-value').val('12345'); $('.repeat-add').trigger('click'); @@ -201,7 +201,7 @@ describe('initIdentifierValidation', () => { expect($('.repeat-item').length).toBe(6); }) - it('it does count identical stripped and unstripped ISBN 13s as the same ISBN', () => { + it('does count identical stripped and unstripped ISBN 13s as the same ISBN', () => { $('#select-id').val('isbn_13'); $('#id-value').val('-979-86 -64653403 '); $('.repeat-add').trigger('click');