From: Radek Czajka Date: Thu, 19 Jan 2012 14:46:10 +0000 (+0100) Subject: Merge branch 'pretty' of github.com:fnp/wolnelektury into pretty X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/276656dcc680e1a35103d310dea046152ddc3785?hp=a91eb26eb1627fbb4ca7bd15ff850e0393ca817e Merge branch 'pretty' of github.com:fnp/wolnelektury into pretty Conflicts: wolnelektury/static/css/book_box.css wolnelektury/templates/catalogue/book_wide.html --- diff --git a/apps/catalogue/models.py b/apps/catalogue/models.py index 1f6210d20..ebecc36b4 100644 --- a/apps/catalogue/models.py +++ b/apps/catalogue/models.py @@ -748,7 +748,8 @@ class Book(models.Model): book.build_mobi() if not settings.NO_SEARCH_INDEX and search_index: - index_book.delay(book.id, book_info) + book.search_index() + #index_book.delay(book.id, book_info) book_descendants = list(book.children.all()) descendants_tags = set() diff --git a/apps/catalogue/tasks.py b/apps/catalogue/tasks.py index 86e84a511..e1ff9151a 100755 --- a/apps/catalogue/tasks.py +++ b/apps/catalogue/tasks.py @@ -5,6 +5,7 @@ from datetime import datetime from celery.task import task import catalogue.models +from traceback import print_exc @task def touch_tag(tag): @@ -18,4 +19,9 @@ def touch_tag(tag): @task def index_book(book_id, book_info=None): - return catalogue.models.Book.objects.get(id=book_id).search_index(book_info) + try: + return catalogue.models.Book.objects.get(id=book_id).search_index(book_info) + except Exception, e: + print "Exception during index: %s" % e + print_exc() + raise e diff --git a/apps/search/index.py b/apps/search/index.py index cc478ee53..307376de9 100644 --- a/apps/search/index.py +++ b/apps/search/index.py @@ -25,6 +25,7 @@ import re import errno from librarian import dcparser from librarian.parser import WLDocument +from lxml import etree import catalogue.models from multiprocessing.pool import ThreadPool from threading import current_thread @@ -212,7 +213,7 @@ class Index(BaseIndex): for tag in catalogue.models.Tag.objects.all(): doc = Document() - doc.add(NumericField("tag_id", Field.Store.YES, True).setIntValue(tag.id)) + doc.add(NumericField("tag_id", Field.Store.YES, True).setIntValue(int(tag.id))) doc.add(Field("tag_name", tag.name, Field.Store.NO, Field.Index.ANALYZED)) doc.add(Field("tag_name_pl", tag.name, Field.Store.NO, Field.Index.ANALYZED)) doc.add(Field("tag_category", tag.category, Field.Store.NO, Field.Index.NOT_ANALYZED)) @@ -223,9 +224,9 @@ class Index(BaseIndex): Create a lucene document referring book id. """ doc = Document() - doc.add(NumericField("book_id", Field.Store.YES, True).setIntValue(book.id)) + doc.add(NumericField("book_id", Field.Store.YES, True).setIntValue(int(book.id))) if book.parent is not None: - doc.add(NumericField("parent_id", Field.Store.YES, True).setIntValue(book.parent.id)) + doc.add(NumericField("parent_id", Field.Store.YES, True).setIntValue(int(book.parent.id))) return doc def remove_book(self, book): @@ -401,28 +402,28 @@ class Index(BaseIndex): if header.tag in self.skip_header_tags: continue + if header.tag is etree.Comment: + continue - content = u' '.join([t for t in header.itertext()]) - content = fix_format(content) - - doc = add_part(snippets, header_index=position, header_type=header.tag, content=content) - - self.index.addDocument(doc) + # section content + content = [] for start, end in walker(header): + # handle fragments and themes. if start is not None and start.tag == 'begin': fid = start.attrib['id'][1:] fragments[fid] = {'content': [], 'themes': [], 'start_section': position, 'start_header': header.tag} - fragments[fid]['content'].append(start.tail) + elif start is not None and start.tag == 'motyw': fid = start.attrib['id'][1:] if start.text is not None: fragments[fid]['themes'] += map(str.strip, map(give_me_utf8, start.text.split(','))) - fragments[fid]['content'].append(start.tail) + elif start is not None and start.tag == 'end': fid = start.attrib['id'][1:] if fid not in fragments: continue # a broken node, skip it + # import pdb; pdb.set_trace() frag = fragments[fid] if frag['themes'] == []: continue # empty themes list. @@ -442,12 +443,23 @@ class Index(BaseIndex): themes=frag['themes']) self.index.addDocument(doc) + + # Collect content. elif start is not None: for frag in fragments.values(): frag['content'].append(start.text) + content.append(start.text) elif end is not None: for frag in fragments.values(): frag['content'].append(end.tail) + content.append(end.tail) + + # in the end, add a section text. + doc = add_part(snippets, header_index=position, header_type=header.tag, + content=fix_format(u' '.join(filter(lambda s: s is not None, content)))) + + self.index.addDocument(doc) + finally: snippets.close() @@ -563,6 +575,8 @@ class SearchResult(object): fragment = stored.get("fragment_anchor") + if snippets: + snippets = snippets.replace("/\n", "\n") hit = (sec + (header_span,), fragment, scoreDocs.score, {'how_found': how_found, 'snippets': snippets and [snippets] or []}) self._hits.append(hit) @@ -630,6 +644,7 @@ class SearchResult(object): frag = catalogue.models.Fragment.objects.get(anchor=f[FRAGMENT]) m = {'score': f[SCORE], 'fragment': frag, + 'section_number': f[POSITION][POSITION_INDEX] + 1, 'themes': frag.tags.filter(category='theme') } m.update(f[OTHER]) @@ -884,7 +899,7 @@ class Search(IndexStore): self.chain_filters([only_in, self.term_filter(Term('is_book', 'true'))]), max_results) for found in top.scoreDocs: - books.append(SearchResult(self.searcher, found)) + books.append(SearchResult(self.searcher, found, how_found="search_perfect_book")) return books def search_book(self, searched, max_results=20, fuzzy=False, hint=None): @@ -910,7 +925,7 @@ class Search(IndexStore): self.chain_filters([only_in, self.term_filter(Term('is_book', 'true'))]), max_results) for found in top.scoreDocs: - books.append(SearchResult(self.searcher, found)) + books.append(SearchResult(self.searcher, found, how_found="search_book")) return books @@ -932,7 +947,7 @@ class Search(IndexStore): flt]), max_results) for found in top.scoreDocs: - books.append(SearchResult(self.searcher, found, snippets=self.get_snippets(found, q))) + books.append(SearchResult(self.searcher, found, snippets=self.get_snippets(found, q), how_found='search_perfect_parts')) return books @@ -964,7 +979,7 @@ class Search(IndexStore): topDocs = self.searcher.search(q, only_in, max_results) for found in topDocs.scoreDocs: - books.append(SearchResult(self.searcher, found)) + books.append(SearchResult(self.searcher, found, how_found='search_everywhere_themesXcontent')) print "* %s theme x content: %s" % (searched, books[-1]._hits) # query themes/content x author/title/tags @@ -983,7 +998,7 @@ class Search(IndexStore): topDocs = self.searcher.search(q, only_in, max_results) for found in topDocs.scoreDocs: - books.append(SearchResult(self.searcher, found)) + books.append(SearchResult(self.searcher, found, how_found='search_everywhere')) print "* %s scatter search: %s" % (searched, books[-1]._hits) return books diff --git a/apps/search/templatetags/__init__.py b/apps/search/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/search/templatetags/search_tags.py b/apps/search/templatetags/search_tags.py new file mode 100644 index 000000000..03e33c8d2 --- /dev/null +++ b/apps/search/templatetags/search_tags.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# +# import feedparser +# import datetime + +from django import template +from django.template import Node, Variable +from django.utils.encoding import smart_str +from django.core.urlresolvers import reverse +# from django.contrib.auth.forms import UserCreationForm, AuthenticationForm +# from django.db.models import Q +from django.conf import settings +# from django.utils.translation import ugettext as _ +from catalogue.templatetags.catalogue_tags import book_wide +from catalogue.models import Book +# from catalogue.forms import SearchForm +# from catalogue.utils import split_tags + + +register = template.Library() + + +@register.inclusion_tag('catalogue/book_searched.html') +def book_searched(result): + book = Book.objects.get(pk=result.book_id) + vals = book_wide(book) + + # snippets = [] + # for hit in result.hits: + # if hit['snippets']: + # snippets.append(hit['snippets']) + # elif hit['fragment']: + # snippets.append(hit['fragment'].short_text) + + # We don't need hits which lead to sections but do not have + # snippets. + vals['hits'] = filter(lambda h: 'fragment' in h or + h['snippets'], result.hits) + + for hit in vals['hits']: + hit['snippets'] = map(lambda s: s.replace("\n", "
").replace('---', '—'), hit['snippets']) + + return vals diff --git a/scripts/make-xml-zip.py b/scripts/make-xml-zip.py new file mode 100755 index 000000000..d8b3dde88 --- /dev/null +++ b/scripts/make-xml-zip.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. +# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. +# +import sys +sys.path.insert(0, '../apps') +sys.path.insert(0, '../lib') +sys.path.insert(0, '../lib/librarian') +sys.path.insert(0, '../wolnelektury') +sys.path.insert(0, '..') + +from django.core.management import setup_environ +from wolnelektury import settings +import sys +import zipfile + +setup_environ(settings) + +from catalogue.models import Book + + +if len(sys.argv) < 2: + print "Provide a zip name as first argument" + sys.exit(-1) + +zip = zipfile.ZipFile(sys.argv[1], 'w') +for book in Book.objects.all(): + zip.write(book.xml_file.path, "%s.xml" % book.slug) +zip.close() + diff --git a/wolnelektury/static/css/base.css b/wolnelektury/static/css/base.css index 5ce2f5a78..556d926c7 100755 --- a/wolnelektury/static/css/base.css +++ b/wolnelektury/static/css/base.css @@ -57,7 +57,7 @@ h2 { .mono { font-family: "Andale Mono", "Lucida Sans Typewriter", "Courier New"; - font-weight: bold; +/* font-weight: bold; */ } .accent1 { diff --git a/wolnelektury/static/css/book_box.css b/wolnelektury/static/css/book_box.css index b816fc01a..ca58a634e 100755 --- a/wolnelektury/static/css/book_box.css +++ b/wolnelektury/static/css/book_box.css @@ -18,14 +18,28 @@ .book-wide-box { width: 98.5em; + + /** This is a fullpage box, it must be aligned with the top menu. + This corresponds to a .1em margin below **/ margin-left: -0.1em; } +/* + * A mini-box wraps it's contents (image + label) in an + * other boxes have an inner box as a wrapper. + */ + +.book-box-inner { + /* min, so it can grow */ + min-height: 19.75em; + margin: .5em; +} + .book-mini-box a, .book-box-inner { display: block; color: black; border: 1px solid #ddd; - height: 20em; +/* height: 20em; */ padding: .8em 1em; margin: .1em; background: #fff; @@ -39,15 +53,17 @@ margin: .1em; overflow: hidden; } -.book-box-inner { - height: 19.75em; - margin: .5em; -} + .book-wide-box .book-box-inner { - height: 24.4em; + /* min, so it can grow */ + min-height: 24.4em; } +/*.book-wide-box.search-result .book-box-inner, .book-wide-box.search-result blockquote { + height: auto !important; +}*/ + .book-mini-box img, .book-box img, .book-wide-box img { width: 13.9em; height: 19.3em; @@ -212,10 +228,16 @@ ul.book-box-tools { vertical-align: center; } -.book-wide-box blockquote div { +.book-wide-box blockquote div.cite-text { padding: 0.888em; } +.book-wide-box blockquote p.cite-more { + display: inline; + font-size: 0.611em; + float: right; +} + ul.inline-items, ul.inline-items li { margin: 0; padding: 0; @@ -225,22 +247,20 @@ ul.inline-items li { display: inline-block; } -.book-wide-box #other-tools { +.book-wide-box .other-tools { float: left; width: 14.5em; margin: 6em 0 0 1.5em; } -.book-wide-box #other-download { +.book-wide-box .other-download { float: left; width: 22.5em; - margin: 6em 1.5em 0em 1.5em + margin: 6em 1.5em 0em 1.5em; } - - .star { font-size: 2.25em; margin-right: .5em; @@ -274,3 +294,7 @@ ul.inline-items li { display: none; } +.snippets .snippet-text { + font-size: 1.2em; + margin: 1.083em 0em; +} diff --git a/wolnelektury/static/css/master.css b/wolnelektury/static/css/master.css index 85f1a40d7..26c13cc71 100644 --- a/wolnelektury/static/css/master.css +++ b/wolnelektury/static/css/master.css @@ -147,6 +147,10 @@ hr { border-right: none; } +#lang-menu-items { + z-index: 1; +} + /* ======================== */ /* = Footer with sponsors = */ /* ======================== */ diff --git a/wolnelektury/static/js/search.js b/wolnelektury/static/js/search.js index 47b6665a6..5951b8b05 100644 --- a/wolnelektury/static/js/search.js +++ b/wolnelektury/static/js/search.js @@ -43,6 +43,10 @@ var __bind = function (self, fn) { $(function() { $("#search").search().labelify({labelledClass: "blur"}); + + $(".search-result .see-more-snippets").click(function() { + $(this).closest('.search-result').find('.snippets').removeClass('ui-helper-hidden'); + }); }); diff --git a/wolnelektury/templates/catalogue/book_searched.html b/wolnelektury/templates/catalogue/book_searched.html new file mode 100644 index 000000000..d719f30e2 --- /dev/null +++ b/wolnelektury/templates/catalogue/book_searched.html @@ -0,0 +1,34 @@ +{% extends "catalogue/book_wide.html" %} +{% load i18n %} + + +{% block box-class %}book-wide-box search-result{% endblock %} + +{% block quote %} +{% if hits.0.snippets %} +
{{hits.0.snippets.0|safe}}
+{% else %}{% if hits.0.fragment %} +
{{hits.0.fragment.short_text|safe}}
+{% endif %}{% endif %} + +{% if hits.1 %} +

{% trans "See more" %}

+{% endif %} +{% endblock %} + + +{% block box-append %} +
+ +{% for hit in hits %} + {% if hit.snippets %} +
{{hit.snippets.0|safe}}
+ {% else %} + {% if hit.fragment %} +
{{hit.fragment.short_text|safe}}
+ {% endif %} + {% endif %} +{% endfor %} + +
+{% endblock %} diff --git a/wolnelektury/templates/catalogue/book_wide.html b/wolnelektury/templates/catalogue/book_wide.html index a13f1a634..cd52ff182 100644 --- a/wolnelektury/templates/catalogue/book_wide.html +++ b/wolnelektury/templates/catalogue/book_wide.html @@ -6,12 +6,13 @@ {% block right-column %}
+ {% block quote %}
Ten, który walczy z potworami powinien zadbać, by sam nie stał się potworem. Gdy długo spoglądamy w otchłań, otchłań spogląda również w nas.
+ {% endblock %}
- -
+

{% trans "See" %}

    {% if extra_info.source_url %} @@ -28,7 +29,7 @@ {% endif %}
-
+

{% trans "Download" %}

  • diff --git a/wolnelektury/templates/catalogue/search_multiple_hits.html b/wolnelektury/templates/catalogue/search_multiple_hits.html index 7a3e580b8..5d222519f 100644 --- a/wolnelektury/templates/catalogue/search_multiple_hits.html +++ b/wolnelektury/templates/catalogue/search_multiple_hits.html @@ -1,75 +1,23 @@ {% extends "base.html" %} {% load i18n %} -{% load catalogue_tags pagination_tags %} +{% load search_tags pagination_tags %} {% block titleextra %}{% trans "Search" %}{% endblock %} {% block bodyid %}tagged-object-list{% endblock %} {% block body %} -

    {% trans "Search" %}

    - {% if did_you_mean %} {% trans "Dod you mean" %} {{did_you_mean|lower}}? {% endif %} + +
    -
      {% for result in results %} -
    1. -

      {{result.book.pretty_title}} (id: {{result.book_id}}, score: {{result.score}})

      -
        - {% for hit in result.hits %} -
      • - {% if hit.fragment %} - Idź do fragmentu -
        Tagi/Motywy: {% for tag in hit.themes %}{{tag.name}} {% endfor %}
        - {# snippets or short html? #} - {% if hit.snippets %} - {% for snip in hit.snippets %} - {{snip|safe}}
        - {% endfor %} - {% else %} - {{hit.fragment.short_text|safe}} - {% endif %} - - {% else %} - {# it's a section #} - {{hit.header_index}} - {% if hit.snippets %} - {% for snip in hit.snippets %} - {{snip|safe}}
        - {% endfor %} - {% else %} - [section matched but no snippets :-(] - {% endif %} - {% endif %} -
        • - {% endfor %} - -
      -
      2. - {% empty %} -

      No results.

      + {% book_searched result %} {% endfor %} -
    -{% comment %} -
    -

    {% trans "More than one result matching the criteria found." %}

    - -
    -{% endcomment %} {% endblock %}