Building my own ebook library (#5)

We will use the whoosh library to store and search our local database. Install the library with pip:

$ pip install whoosh

You can find the documentation here: Whoosh documentation.

Let’s create a file to encapsulate all our calls to the library.

#file: providers/local/backend.py

DB_LOCATION = 'providers/local/index'

from whoosh.fields import Schema, TEXT, DATETIME, NUMERIC, STORED
def get_schema():
    return Schema(  title=TEXT(stored=True, field_boost=2.0),
                    authors=TEXT(stored=True, field_boost=2.0),
                    notes=TEXT(stored=True),
                    acquired=DATETIME(stored=True),
                    rating=NUMERIC(stored=True),
                    cover=STORED)


from whoosh.index import create_in
import os.path
def create_index(schema, location_dir):
    if os.path.exists(location_dir):
        print('Index already exist')
    else:
        os.mkdir(location_dir)
        ix = create_in(location_dir, schema)
        print('Index created')


from whoosh.index import open_dir
def get_index(location_dir = DB_LOCATION):
    return open_dir(location_dir)


from datetime import datetime
def add_books(books, database = None):
    if not database:
        database = get_index()
    with database.writer() as writer:
        for book in books:
            acquired = book.get('acquired', '')
            acquired = acquired if acquired else datetime.utcnow()
            writer.add_document(
                title    = book.get('title', ''),
                authors  = book.get('authors', ''),
                notes    = book.get('notes', ''),
                acquired = acquired,
                rating   = book.get('rating', 0),
                cover    = book.get('cover', ''))


import re
from whoosh import qparser
from whoosh.qparser import MultifieldParser
def search(q):
    ix = get_index(DB_LOCATION)
    og = qparser.OrGroup.factory(0.9)
    parser = MultifieldParser(["title", "notes", "authors"], schema=ix.schema, group=og)
    query = parser.parse(q)
    with ix.searcher() as searcher:
        results = searcher.search(query)
        return [{
                    'title' : result['title'],
                    'authors' : result['authors'].split('-'),
                    'rating' : result.get('rating', -1),
                    'notes' : result.get('notes', ''),
                    'acquired' : result.get('acquired', ''),
                    'cover' : result.get('cover', '')
                } for result in results]

We need to create the library folder. Let write a quick script for that:

#file providers/local/setup.py
import backend

if __name__ == "__main__":
    print('Creating index at {}'.format(backend.DB_LOCATION))
    backend.create_index(backend.get_schema(), backend.DB_LOCATION)

and run the script from the root of your project:

python3 providers/local/setup.py

We won’t be using this script anymore, but it’s a good idea to keep it around for further reference.

Now that we have a local database, we can actually record the book’s metadata and show what we have in store.

Let’s start with presenting what we have. As previously, there are two things to do: fetch the results and present them using html. Our backend knows how to fetch, so there only remains to create an html_presenter.

We will use a template as we did for goodreads.

#file: providers/local/html_presenter.py
from io import StringIO
from template_loader import load_template

def format(results):
    content = StringIO()
    content.write('<table>')
    template = load_template('item.html', './providers/local/')
    for item_id, result in enumerate(results):
        content.write(template.substitute(
            cover = result.get('cover', ''),
            title = result.get('title', ''),
            authors = ' - '.join(result.get('authors', [])),
            rating = result.get('rating', ''),
            notes = result.get('notes', ''),
            acquired = result.get('acquired', ''),
            ))
    content.write('</table>')
    return content.getvalue()

and the html template in providers/local/item.html:

<!-- file: providers/local/item.html -->
<tr>
    <td style="background-color: lightblue;">
        <img src="$cover" height="200px"
             style="max-width:200px;
                    max-height:200px;
                    display: block;
                    margin-left: auto;
                    margin-right: auto;"> </td>
    <td>
        <ul>
            <li><strong>$title</strong></li>
            <li>$authors</li>
            <li>Rating: $rating | Acquired: $acquired</li>
            <li>Notes: $notes</li>
        </ul>
    </td>
</tr>

Finally we can complete our main.py:

#file: providers/local/main.py
import re
from local import backend
from local import html_presenter

def search(q):
    tokens = re.split('[^a-zA-Z0-9]', q)
    query = ' '.join(tokens)
    results = backend.search(query)
    html = html_presenter.format(results)
    return {
        'name': 'Local',
        'html': html
    }

def handle_get(server, path):
    pass

import server_helper
def handle_post(server, path):
    if path != '/download.html':
        return

    try:
        data = server_helper.parse_POST(server)
        title = data[b'title'][0].decode('utf8')
        rating = float(data.get(b'rating', b'-1')[0].decode('utf8'))
        authors = data.get(b'authors', b'')[0].decode('utf8')

        backend.add_books([{
            'title' : title,
            'authors' : authors,
            'rating' : rating
        }])

        response = '''
        <html>
            <h1>Success!</h1>
            <ul>
               <li> Title: {} </li>
               <li> Authors: {} </li>
               <li> Rating: {} </li>
            </ul>
        </html>
        '''.format(title, authors, rating)
        server.send_message(response)

    except Exception as e:
        response = '<pre>'
        import traceback
        response += str(traceback.format_exc())
        response += '</pre>'
        server.send_message(response)

If you save a few books and try a few searches, you’ll notice that we don’t save any images and that there’s no way to add notes to our records. Let’s add these functionalities now. We will start with the notes.

Instead of directly saving the book’s data into the database, we will add an intermediate endpoint to edit the details and add a custom note. Let’s call this endpoint /predownload.html.

We will create a file providers/local/endpoints.py to handle this logic. Move the code inside the try block of main.py > handle_post into a function endpoints.py > download and create a new function endpoints.py > predownload.

providers/local/main.py becomes:

#file: providers/local/main.py (continued)
import re
from local import backend, html_presenter, endpoints

# search(q) and handle_get(s, p) as previously

def handle_post(server, path):
    try:
        if path == '/download.html':
            endpoints.download(server, path)
        elif path == '/predownload.html':
            endpoints.predownload(server, path)
    except Exception as e:
        response = '<pre>'
        import traceback
        response += str(traceback.format_exc())
        response += '</pre>'
        server.send_message(response)

and providers/local/endpoints.py is:

#file: providers/local/endpoints.py
from local import backend
from local import html_presenter
import server_helper

def predownload(server, path):
    pass

def download(server, path):
    pass

To implement predownload, we simply try to extract as much info as we can from the POST data and write it into an html form.

#file:providers/local/endpoints.py (update)
from server_helper import postvar

def parse_post_variable(server):
    data = server_helper.parse_POST(server)
    return {
        'title'   : postvar(data, 'title'),
        'authors' : postvar(data, 'authors'),
        'rating'  : float(postvar(data, 'rating', '-1')),
        'notes'   : postvar(data, 'notes'),
        'acquired': postvar(data, 'acquired'),
        'cover'   : postvar(data, 'cover'),
    }

def predownload(server, path):
    data = parse_post_variable(server)
    response = html_presenter.predownload_form(data)
    server.send_message(response)

We also need to create the presenter function.

#file: providers/local/html_presenter.py (append)

def predownload_form(data):
    template = load_template('predownload.html', './providers/local/')
    return template.substitute(
            cover = data.get('cover', ''),
            title = data.get('title', ''),
            authors = data.get('authors', ''),
            rating = data.get('rating', ''),
            notes = data.get('notes', ''),
            acquired = data.get('acquired', ''),
    )

And to create the corresponding template.

<!-- file: providers/local/predownload.html -->
<html><head><meta charset="UTF-8"></head>
<style>
 label { display:inline-block; width: 60px; }
 img { max-width:200px; max-height:200px;
     display: block; margin-left: auto; margin-right: auto; }
 .img { background-color: lightblue; }
</style>

<table><tr>
    <td class="img"> <img src="$cover" height="200px"> </td>
    <td><form method="post" action="download.html">
        <label>Title</label>
        <input type="text" name="title" value="$title" size="100" placeholder="Title">
        <br/>
        <label>Authors</label>
        <input type="text" name="authors" value="$authors" size="100" placeholder="Authors">
        <br/>
        <label>Notes</label>
        <textarea name="notes" rows="4" cols="100">$notes</textarea>
        <br/>
        <label>Rating</label>
        <input type="text" name="rating"   value="$rating" placeholder="Rating">
        <br/>
        <label>Acquired</label>
        <input type="text" name="acquired" value="$acquired" placeholder="Acquired Date">
        <br/>
        <label>Cover</label>
        <input type="text" name="cover" value="$cover" placeholder="Cover URL">
        <br/>
        <input type="submit" value="Download">
    </form></td>
</tr></table></html>

Before we test this endpoint, we need to update the goodread’s download form. I copied the whole file below and indicated the line that changed with <!--new-->: the action field is modified from download.html to predownload.html and I added a hidden input for cover.

<!-- file: providers/goodreads/item.html -->
<tr><td style="background-color: lightblue;">
        <img src="$img" height="200px"
             style="max-width:200px;
                    max-height:200px;
                    display: block;
                    margin-left: auto;
                    margin-right: auto;"> </td>
    <td><ul>
             <li><strong>$title</strong></li>
             <li>$authors</li>
             <li>Rating: $rating</li>
<!--new-->   <li><form method="post" action="predownload.html">
                    <input type="hidden" name="title"   value="$title">
                    <input type="hidden" name="authors" value="$authors">
                    <input type="hidden" name="rating"  value="$rating">
<!--new-->          <input type="hidden" name="cover" value="$img">
                    <input type="submit" value="Download">
             </form></li>
    </ul></td></tr>

There remains to implement the download endpoint. Simply parse the data from the POST variable and use the add_book function.

#file: providers/local/endpoints.py (append)

def download(server, path):
    data = parse_post_variable(server)
    backend.add_books([data])
    response = html_presenter.downloaded(data)
    server.send_message(response)
#file: providers/local/html_presenter.py (append)

def downloaded(data):
    template = load_template('downloaded.html', './providers/local/')
    return template.substitute(
            cover = data.get('cover', ''),
            title = data.get('title', ''),
            authors = data.get('authors', ''),
            rating = data.get('rating', ''),
            notes = data.get('notes', ''),
            acquired = data.get('acquired', ''),
    )

and create the corresponding html template.

<!-- file: providers/local/downloaded.html -->
<html><head><meta charset="UTF-8"></head>
<style>
 img { max-width:200px; max-height:200px;
     display: block; float:left; }
 tr > :first-child { width:100px; text-align:right; border-right:1px solid black; }
</style>

<div>
<h1> Downloaded </h1>
<a href="index.html"> &larr; back </a>
</div>

<img src="$cover" height="200px">
<table>
    <tr><td> Title    </td><td> $title    </td></tr>
    <tr><td> Authors  </td><td> $authors  </td></tr>
    <tr><td> Notes    </td><td> $notes    </td></tr>
    <tr><td> Rating   </td><td> $rating   </td></tr>
    <tr><td> Acquired </td><td> $acquired </td></tr>
</table>
</html>

previous article