From: Marcin Koziej Date: Thu, 12 Apr 2012 09:14:39 +0000 (+0200) Subject: Merge branch 'master' of github.com:fnp/wolnelektury X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/07a437ef67ffcad01b83255999d2d7744f994d7f?hp=82cef4d2af27243f4f4c33bafd9e1c2447d41895 Merge branch 'master' of github.com:fnp/wolnelektury --- diff --git a/apps/search/index.py b/apps/search/index.py index 4e71e2500..b689c763c 100644 --- a/apps/search/index.py +++ b/apps/search/index.py @@ -317,14 +317,19 @@ class Index(BaseIndex): doc.add(NumericField("parent_id", Field.Store.YES, True).setIntValue(int(book.parent.id))) return doc - def remove_book(self, book, remove_snippets=True): + def remove_book(self, book_or_id, remove_snippets=True): """Removes a book from search index. book - Book instance.""" - q = NumericRangeQuery.newIntRange("book_id", book.id, book.id, True, True) + if isinstance(book_or_id, catalogue.models.Book): + book_id = book_or_id.id + else: + book_id = book_or_id + + q = NumericRangeQuery.newIntRange("book_id", book_id, book_id, True, True) self.index.deleteDocuments(q) if remove_snippets: - snippets = Snippets(book.id) + snippets = Snippets(book_id) snippets.remove() def index_book(self, book, book_info=None, overwrite=True): @@ -339,7 +344,11 @@ class Index(BaseIndex): self.remove_book(book, remove_snippets=False) book_doc = self.create_book_doc(book) - meta_fields = self.extract_metadata(book, book_info) + meta_fields = self.extract_metadata(book, book_info, dc_only=['source_name', 'authors', 'title']) + # let's not index it - it's only used for extracting publish date + if 'source_name' in meta_fields: + del meta_fields['source_name'] + for f in meta_fields.values(): if isinstance(f, list) or isinstance(f, tuple): for elem in f: @@ -373,7 +382,7 @@ class Index(BaseIndex): published_date_re = re.compile("([0-9]+)[\]. ]*$") - def extract_metadata(self, book, book_info=None): + def extract_metadata(self, book, book_info=None, dc_only=None): """ Extract metadata from book and returns a map of fields keyed by fieldname """ @@ -388,6 +397,8 @@ class Index(BaseIndex): # validator, name for field in dcparser.BookInfo.FIELDS: + if dc_only and field.name not in dc_only: + continue if hasattr(book_info, field.name): if not getattr(book_info, field.name): continue @@ -1055,7 +1066,8 @@ class Search(IndexStore): return toks - def fuzziness(self, fuzzy): + @staticmethod + def fuzziness(fuzzy): """Helper method to sanitize fuzziness""" if not fuzzy: return None @@ -1092,7 +1104,8 @@ class Search(IndexStore): phrase.add(term) return phrase - def make_term_query(self, tokens, field='content', modal=BooleanClause.Occur.SHOULD, fuzzy=False): + @staticmethod + def make_term_query(tokens, field='content', modal=BooleanClause.Occur.SHOULD, fuzzy=False): """ Returns term queries joined by boolean query. modal - applies to boolean query diff --git a/apps/search/management/commands/optimizeindex.py b/apps/search/management/commands/optimizeindex.py index a8a4cf9dd..51bf95b4e 100644 --- a/apps/search/management/commands/optimizeindex.py +++ b/apps/search/management/commands/optimizeindex.py @@ -1,14 +1,38 @@ from django.core.management.base import BaseCommand -from search import Index +from search import Index, Search +from lucene import IndexReader, IndexSearcher, Term +from catalogue.models import Book + class Command(BaseCommand): help = 'Optimize Lucene search index' args = '' + def delete_old(self, index): + existing_ids = set([book.id for book in Book.objects.all()]) + + reader = IndexReader.open(index.index, False) + searcher = IndexSearcher(reader) + try: + num = searcher.docFreq(Term('is_book', 'true')) + docs = searcher.search(Search.make_term_query(['true'], 'is_book'), num) + for result in docs.scoreDocs: + stored = searcher.doc(result.doc) + book_id = int(stored.get('book_id')) + if not book_id in existing_ids: + print "book id %d doesn't exist." % book_id + index.remove_book(book_id) + finally: + searcher.close() + reader.close() + def handle(self, *args, **opts): index = Index() index.open() + + self.delete_old(index) + try: index.optimize() finally: diff --git a/apps/search/views.py b/apps/search/views.py index 09f217f23..fd5883ede 100644 --- a/apps/search/views.py +++ b/apps/search/views.py @@ -14,6 +14,7 @@ from catalogue.views import JSONResponse from search import Search, JVM, SearchResult from lucene import StringReader from suggest.forms import PublishingSuggestForm +from time import sleep import re import enchant @@ -50,8 +51,21 @@ def did_you_mean(query, tokens): return query + JVM.attachCurrentThread() -search = Search() +_search = None + + +def get_search(): + global _search + + while _search is False: + sleep(1) + + if _search is None: + _search = False + _search = Search() + return _search def hint(request): @@ -60,6 +74,7 @@ def hint(request): return JSONResponse([]) JVM.attachCurrentThread() + search = get_search() hint = search.hint() try: tags = request.GET.get('tags', '') @@ -117,6 +132,7 @@ def main(request): return render_to_response('catalogue/search_too_short.html', {'prefix': query}, context_instance=RequestContext(request)) + search = get_search() # hint.tags(tag_list) # if book: # hint.books(book) diff --git a/apps/wolnelektury_core/static/css/widget.css b/apps/wolnelektury_core/static/css/widget.css index 626698844..14aed95f2 100644 --- a/apps/wolnelektury_core/static/css/widget.css +++ b/apps/wolnelektury_core/static/css/widget.css @@ -1,54 +1,153 @@ + +/* + * RESET + * src: http://meyerweb.com/eric/tools/css/reset/ + */ + +#wl div, #wl span, #wl applet, #wl object, #wl iframe, +#wl #eh1, #wl h2, #wl h3, #wl h4, #wl h5, #wl h6, #wl p, #wl blockquote, #wl pre, +#wl a, #wl abbr, #wl acronym, #wl address, #wl big, #wl cite, #wl code, +#wl del, #wl dfn, #wl em, #wl img, #wl ins, #wl kbd, #wl q, #wl s, #wl samp, +#wl small, #wl strike, #wl strong, #wl sub, #wl sup, #wl tt, #wl var, +#wl b, #wl u, #wl i, #wl center, +#wl dl, #wl dt, #wl dd, #wl ol, #wl ul, #wl li, +#wl fieldset, #wl form, #wl label, #wl legend, +#wl table, #wl caption, #wl tbody, #wl tfoot, #wl thead, #wl tr, #wl th, #wl td, +#wl article, #wl aside, #wl canvas, #wl details, #wl embed, +#wl figure, #wl figcaption, #wl footer, #wl header, #wl hgroup, +#wl menu, #wl nav, #wl output, #wl ruby, #wl section, #wl summary, +#wl time, #wl mark, #wl audio, #wl video { + margin: 0; + padding: 0; + border: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; +} + #wl { -background-color: white; + line-height: 1; +} + +#wl ol, #wl ul { + list-style: none; +} +#wl blockquote, #wl q { + quotes: none; +} + +#wl blockquote:before, #wl blockquote:after, +#wl q:before, #wl q:after { + content: ''; + content: none; +} +#wl table { + border-collapse: collapse; + border-spacing: 0; } + +/* END RESET */ + +#wl { + background-color: white; +} + #wl a, a:visited, a:hover { -border: 0; + border: 0; } + #wl img { -border: 0; + border: 0; + width: 8.625em; + margin: 0.375em 0; } + + .ui-menu { -width: 160px; -font-size: small; -list-style-type: none; -padding: 0; -margin:0; -border-left: 1px solid #696969; -border-right: 1px solid #696969; -border-bottom: 1px solid #696969; + width: 29em; + font-size: small; + list-style-type: none; + padding: 0; + margin:0; + border: 1px solid #ddd; +} + +.ui-menu .search-hint-label { + display: inline-block; + width: 20em; + margin-right: 2em; + font-size: 1.1em; + line-height: 1.636em; + white-space: nowrap; + overflow: hidden; +} + +.ui-menu .search-hint-category { + width: 9em; +} + +.ui-corner-all { + /*width: 160px;*/ + cursor: pointer; + display:block; +} + +.ui-corner-all a { + text-decoration: none; + color: #0D7E85; + font-size: small; + padding: 5px; + } .ui-menu li { -font-size: small; -padding: 5px; -width: 150px } .ui-menu li:nth-child(odd) { -background-color: white; + background-color: white; } .ui-menu li:nth-child(even){ -background-color: #EDF1F5; + background-color: #EEE; +} + +.ui-menu li .ui-state-hover { + background-color: #fdf5ce; } .ui-menu li:hover{ -background-color: #053469; -color:white; + background-color: #053469; + color:white; } .ui-menu a:hover { -color:white; -text-decoration: none; + color:white; + text-decoration: none; } -.ui-corner-all { -width: 160px; -cursor: pointer; -display:block; + +#wl #id_qq { + font-family: Georgia; + font-size: 0.75em; + + background-color: #FFFFFF; + border: medium none; + border-radius: 0.38em 0.38em 0.38em 0.38em; + box-shadow: 0 0 0.5em #444444 inset; + color: #000000; + height: 2.54em; + width: 9.125em; + + line-height: 2.5em; + padding: 0 0 0 1em; + position: relative; } -.ui-corner-all a{ - color: black; - text-decoration: none; +#wl input[type=image] { + width: 1.5em; + height: 1.5625em; + top: 0.375em; + position: relative; + margin-left: 0.625em; } + diff --git a/apps/wolnelektury_core/static/img/search.png b/apps/wolnelektury_core/static/img/search.png index 6bfc84cb1..16b2a5077 100644 Binary files a/apps/wolnelektury_core/static/img/search.png and b/apps/wolnelektury_core/static/img/search.png differ diff --git a/apps/wolnelektury_core/static/js/search.js b/apps/wolnelektury_core/static/js/search.js index 6afc4a399..293b9e3fa 100644 --- a/apps/wolnelektury_core/static/js/search.js +++ b/apps/wolnelektury_core/static/js/search.js @@ -16,7 +16,9 @@ var __bind = function (self, fn) { focus: function() { return false; }, source: this.element.data('source'), }; - this.element.autocomplete(opts).data("autocomplete")._renderItem = __bind(this, this.render_item); + + this.element.autocomplete($.extend(opts, this.options)) + .data("autocomplete")._renderItem = __bind(this, this.render_item); }, enter: function(event, ui) { @@ -32,7 +34,7 @@ var __bind = function (self, fn) { .append(''+item.label+''+ ''+item.category+'') .appendTo(ul); - }, + }, destroy: function() { diff --git a/apps/wolnelektury_core/static/js/widget.js b/apps/wolnelektury_core/static/js/widget.js index de30eb714..550bf2084 100644 --- a/apps/wolnelektury_core/static/js/widget.js +++ b/apps/wolnelektury_core/static/js/widget.js @@ -8,15 +8,19 @@ var inputText = document.createElement('input'); var inputSubmit = document.createElement('input'); var body = document.getElementsByTagName('body') var stylesheet = document.createElement('link'); +var stylesheetJQUI = document.createElement('linl'); -var host = 'www.wolnelektury.pl'; +var host = 'localhost:8000'; //'www.wolnelektury.pl'; /* set attributes of created elements */ stylesheet.setAttribute('type', 'text/css'); stylesheet.setAttribute('rel', 'stylesheet'); stylesheet.setAttribute('href', 'http://'+host+'/static/css/widget.css'); +stylesheetJQUI.setAttribute('type', 'text/css'); +stylesheetJQUI.setAttribute('rel', 'stylesheet'); +stylesheetJQUI.setAttribute('href', 'http://'+host+'/static/css/ui-lightness/jquery-ui-1.8.16.custom.css'); linkLogo.setAttribute('href', 'http://'+host); -logo.setAttribute('src', 'http://'+host+'/static/img/logo.png'); +logo.setAttribute('src', 'http://'+host+'/static/img/logo-bez.png'); form.setAttribute('action', 'http://'+host+'/szukaj/'); form.setAttribute('method', 'get'); form.setAttribute('accept-charset', 'utf-8'); @@ -26,10 +30,11 @@ inputText.setAttribute('title', 'tytul, autor, motyw/temat, epoka, rodzaj, gatun inputText.setAttribute('value', ''); inputText.setAttribute('name', 'q'); inputText.setAttribute('id', 'id_qq'); -inputText.setAttribute('size', '13'); +inputText.setAttribute('data-source', 'http://'+host+'/szukaj/hint'); +/*inputText.setAttribute('size', '13');*/ inputSubmit.setAttribute('type', 'image'); inputSubmit.setAttribute('src', 'http://'+host+'/static/img/search.png'); -inputSubmit.setAttribute('style', 'position:relative; top:5px; margin-left:5px'); +/* inputSubmit.setAttribute('style', 'position:relative; top:5px; margin-left:5px');*/ /* import jquery and autocomplete */ var scriptJ = document.createElement('script'); @@ -39,6 +44,12 @@ scriptJ.setAttribute('src', 'http://'+host+'/static/js/jquery.js'); var scriptAutoComplete = document.createElement('script'); scriptAutoComplete.setAttribute('type', 'text/javascript'); scriptAutoComplete.setAttribute('src', 'http://'+host+'/static/js/jquery-ui-1.8.2.custom.min.js'); +scriptAutoComplete.setAttribute('id', 'wl-jquery-ui-script') + +var scriptSearch = document.createElement('script'); +scriptSearch.setAttribute('type', 'text/javascript'); +scriptSearch.setAttribute('src', 'http://'+host+'/static/js/search.js'); +scriptSearch.setAttribute('id', 'wl-search-script') var scriptInit = document.createElement('script'); scriptInit.setAttribute('type', 'text/javascript'); @@ -46,10 +57,12 @@ scriptInit.setAttribute('src', 'http://'+host+'/static/js/widgetInit.js'); body[0].appendChild(scriptJ); body[0].appendChild(scriptAutoComplete); +body[0].appendChild(scriptSearch); body[0].appendChild(scriptInit); /* append elements to widget */ widget.appendChild(stylesheet); +//widget.appendChild(stylesheetJQUI); widget.appendChild(linkLogo); linkLogo.appendChild(logo); widget.appendChild(form); @@ -57,6 +70,7 @@ form.appendChild(inputText); form.appendChild(inputSubmit); /* ...and a little make-up */ +/* widget.style.borderColor = "#84BF2A"; widget.style.borderWidth = "2px"; widget.style.borderStyle = "solid"; @@ -64,6 +78,7 @@ widget.style.width = "160px"; widget.style.padding = "10px"; widget.style.fontSize = "12px"; form.style.paddingTop = "10px"; +*/ /* resize - if needed */ if(widget.getAttribute('width') == '140'){ diff --git a/apps/wolnelektury_core/static/js/widgetInit.js b/apps/wolnelektury_core/static/js/widgetInit.js index ab57a3062..4cfa85a26 100644 --- a/apps/wolnelektury_core/static/js/widgetInit.js +++ b/apps/wolnelektury_core/static/js/widgetInit.js @@ -1,5 +1,20 @@ - $(function() { - $("#id_qq").autocomplete({ + +var wl_scripts_loaded = {}; + +function wl_load_search_if_ready(id) { + wl_scripts_loaded[id] = true; + if (wl_scripts_loaded['wl-search-script'] && + wl_scripts_loaded['wl-jquery-ui-script']) + { + var s = $('#id_qq'); + s.search({source: s.attr('data-source')}); + } +} + +$('#wl-search-script').ready(function(){wl_load_search_if_ready('wl-search-script');}); +$('#wl-jquery-ui-script').ready(function(){wl_load_search_if_ready('wl-jquery-ui-script');}); + +/*autocomplete({ source: function(request, response) { $.ajax({ url: "http://www.wolnelektury.pl/katalog/jtags/", @@ -27,3 +42,4 @@ } }); }); +*/ diff --git a/apps/wolnelektury_core/static/widget.html b/apps/wolnelektury_core/static/widget.html index 890184da4..fffc0167f 100644 --- a/apps/wolnelektury_core/static/widget.html +++ b/apps/wolnelektury_core/static/widget.html @@ -5,7 +5,7 @@ - + diff --git a/wolnelektury/settings/__init__.py b/wolnelektury/settings/__init__.py index f2314a9ad..15126b9a4 100644 --- a/wolnelektury/settings/__init__.py +++ b/wolnelektury/settings/__init__.py @@ -63,6 +63,7 @@ INSTALLED_APPS_OUR = [ 'picture', 'social', 'waiter', + 'search', ] INSTALLED_APPS_CONTRIB = [