b5cc0baa337173c2a6def9f13ecec426cf4d474c
[wolnelektury.git] / src / search / views.py
1 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
3 #
4 from django.conf import settings
5 from django.http.response import HttpResponseRedirect
6 from django.shortcuts import render
7 from django.views.decorators import cache
8 from django.http import HttpResponse, JsonResponse
9
10 from catalogue.models import Book, Tag
11 from pdcounter.models import Author
12 from picture.models import Picture
13 from search.index import Search, SearchResult, PictureResult
14 from .forms import SearchFilters
15 from suggest.forms import PublishingSuggestForm
16 import re
17 import json
18
19 from wolnelektury.utils import re_escape
20
21
22 def match_word_re(word):
23     if 'sqlite' in settings.DATABASES['default']['ENGINE']:
24         return r"\b%s\b" % word
25     elif 'mysql' in settings.DATABASES['default']['ENGINE']:
26         return "[[:<:]]%s[[:>:]]" % word
27
28
29 query_syntax_chars = re.compile(r"[\\/*:(){}?.[\]+]")
30
31
32 def remove_query_syntax_chars(query, replace=' '):
33     return query_syntax_chars.sub(replace, query)
34
35
36 def did_you_mean(query, tokens):
37     return query
38     # change = {}
39     # for t in tokens:
40     #     authors = Tag.objects.filter(category='author', name__iregex=match_word_re(t))
41     #     if len(authors) > 0:
42     #         continue
43
44     #     if False:
45     #         if not dictionary.check(t):
46     #             try:
47     #                 change_to = dictionary.suggest(t)[0].lower()
48     #                 if change_to != t.lower():
49     #                     change[t] = change_to
50     #             except IndexError:
51     #                 pass
52
53     # if change == {}:
54     #     return None
55
56     # for frm, to in change.items():
57     #     query = query.replace(frm, to)
58
59     # return query
60
61
62 @cache.never_cache
63 def hint(request, mozhint=False, param='term'):
64     prefix = request.GET.get(param, '')
65     if len(prefix) < 2:
66         return JsonResponse([], safe=False)
67
68     prefix = re_escape(' '.join(remove_query_syntax_chars(prefix).split()))
69
70     try:
71         limit = int(request.GET.get('max', ''))
72     except ValueError:
73         limit = 20
74     else:
75         if limit < 1:
76             limit = 20
77
78     authors = Tag.objects.filter(
79         category='author', name_pl__iregex='\m' + prefix).only('name', 'id', 'slug', 'category')
80     data = [
81         {
82             'label': author.name,
83             'id': author.id,
84             'url': author.get_absolute_url(),
85         }
86         for author in authors[:limit]
87     ]
88     if len(data) < limit:
89         for b in Book.objects.filter(findable=True, title__iregex='\m' + prefix)[:limit-len(data)]:
90             author_str = b.author_unicode()
91             translator = b.translator()
92             if translator:
93                 author_str += ' (tłum. ' + translator + ')'
94             data.append(
95                 {
96                     'label': b.title,
97                     'author': author_str,
98                     'id': b.id,
99                     'url': b.get_absolute_url()
100                 }
101             )
102
103     if mozhint:
104         data = [
105             prefix,
106             [
107                 item['label']
108                 for item in data
109             ]
110         ]
111
112     callback = request.GET.get('callback', None)
113     if callback:
114         return HttpResponse("%s(%s);" % (callback, json.dumps(data)),
115                             content_type="application/json; charset=utf-8")
116     else:
117         return JsonResponse(data, safe=False)
118
119
120
121 @cache.never_cache
122 def search(request):
123     filters = SearchFilters(request.GET)
124     ctx = {
125         'title': 'Wynik wyszukiwania',
126         'query': request.GET.get('q', ''),
127         'filters': filters,
128     }
129     if filters.is_valid():
130         ctx['results'] = filters.results()
131         for k, v in ctx['results'].items():
132             if v:
133                 ctx['hasresults'] = True
134                 break
135     return render(request, 'search/results.html', ctx)
136
137
138 @cache.never_cache
139 def main(request):
140     if request.EXPERIMENTS['layout'].value:
141         return search(request)
142
143     query = request.GET.get('q', '')
144
145     format = request.GET.get('format')
146     lang = request.GET.get('lang')
147     epoch = request.GET.get('epoch')
148     kind = request.GET.get('kind')
149     genre = request.GET.get('genre')
150
151     if len(query) < 2:
152         return render(
153             request, 'catalogue/search_too_short.html',
154             {'prefix': query})
155     elif len(query) > 256:
156         return render(
157             request, 'catalogue/search_too_long.html',
158             {'prefix': query})
159
160     query = prepare_query(query)
161     if not (format or lang or epoch or kind or genre):
162         pd_authors = search_pd_authors(query)
163     else:
164         pd_authors = []
165     if not format or format != 'obraz':
166         books = search_books(
167             query,
168             lang=lang,
169             only_audio=format=='audio',
170             only_synchro=format=='synchro',
171             epoch=epoch,
172             kind=kind,
173             genre=genre
174         )
175     else:
176         books = []
177     if (not format or format == 'obraz') and not lang:
178         pictures = search_pictures(
179             query,
180             epoch=epoch,
181             kind=kind,
182             genre=genre
183         )
184     else:
185         pictures = []
186     
187     suggestion = ''
188
189     if not (books or pictures or pd_authors):
190         form = PublishingSuggestForm(initial={"books": query + ", "})
191         return render(
192             request,
193             'catalogue/search_no_hits.html',
194             {
195                 'form': form,
196                 'did_you_mean': suggestion
197             })
198
199     if not (books or pictures) and len(pd_authors) == 1:
200         return HttpResponseRedirect(pd_authors[0].get_absolute_url())
201
202     return render(
203         request,
204         'catalogue/search_multiple_hits.html',
205         {
206             'pd_authors': pd_authors,
207             'books': books,
208             'pictures': pictures,
209             'did_you_mean': suggestion,
210             'set': {
211                 'lang': lang,
212                 'format': format,
213                 'epoch': epoch,
214                 'kind': kind,
215                 'genre': genre,
216             },
217             'tags': {
218                 'epoch': Tag.objects.filter(category='epoch', for_books=True),
219                 'genre': Tag.objects.filter(category='genre', for_books=True),
220                 'kind': Tag.objects.filter(category='kind', for_books=True),
221             },
222         })
223
224 def search_books(query, lang=None, only_audio=False, only_synchro=False, epoch=None, kind=None, genre=None):
225     search = Search()
226     results_parts = []
227     search_fields = []
228     words = query.split()
229     fieldsets = (
230         (['authors', 'authors_nonstem'], True),
231         (['title', 'title_nonstem'], True),
232         (['metadata', 'metadata_nonstem'], True),
233         (['text', 'text_nonstem', 'themes_pl', 'themes_pl_nonstem'], False),
234     )
235     for fields, is_book in fieldsets:
236         search_fields += fields
237         results_parts.append(search.search_words(words, search_fields, required=fields, book=is_book))
238     results = []
239     ids_results = {}
240     for results_part in results_parts:
241         for result in sorted(SearchResult.aggregate(results_part), reverse=True):
242             book_id = result.book_id
243             if book_id in ids_results:
244                 ids_results[book_id].merge(result)
245             else:
246                 results.append(result)
247                 ids_results[book_id] = result
248     descendant_ids = set(
249         Book.objects.filter(id__in=ids_results, ancestor__in=ids_results).values_list('id', flat=True))
250     results = [result for result in results if result.book_id not in descendant_ids]
251     for result in results:
252         search.get_snippets(result, query, num=3)
253
254     def ensure_exists(r):
255         try:
256             if not r.book:
257                 return False
258         except Book.DoesNotExist:
259             return False
260
261         if lang and r.book.language != lang:
262             return False
263         if only_audio and not r.book.has_mp3_file():
264             return False
265         if only_synchro and not r.book.has_daisy_file():
266             return False
267         if epoch and not r.book.tags.filter(category='epoch', slug=epoch).exists():
268             return False
269         if kind and not r.book.tags.filter(category='kind', slug=kind).exists():
270             return False
271         if genre and not r.book.tags.filter(category='genre', slug=genre).exists():
272             return False
273
274         return True
275
276     results = [r for r in results if ensure_exists(r)]
277     return results
278
279
280 def search_pictures(query, epoch=None, kind=None, genre=None):
281     search = Search()
282     results_parts = []
283     search_fields = []
284     words = query.split()
285     fieldsets = (
286         (['authors', 'authors_nonstem'], True),
287         (['title', 'title_nonstem'], True),
288         (['metadata', 'metadata_nonstem'], True),
289         (['themes_pl', 'themes_pl_nonstem'], False),
290     )
291     for fields, is_book in fieldsets:
292         search_fields += fields
293         results_parts.append(search.search_words(words, search_fields, required=fields, book=is_book, picture=True))
294     results = []
295     ids_results = {}
296     for results_part in results_parts:
297         for result in sorted(PictureResult.aggregate(results_part), reverse=True):
298             picture_id = result.picture_id
299             if picture_id in ids_results:
300                 ids_results[picture_id].merge(result)
301             else:
302                 results.append(result)
303                 ids_results[picture_id] = result
304
305     def ensure_exists(r):
306         try:
307             if not r.picture:
308                 return False
309         except Picture.DoesNotExist:
310             return False
311
312         if epoch and not r.picture.tags.filter(category='epoch', slug=epoch).exists():
313             return False
314         if kind and not r.picture.tags.filter(category='kind', slug=kind).exists():
315             return False
316         if genre and not r.picture.tags.filter(category='genre', slug=genre).exists():
317             return False
318
319         return True
320
321     results = [r for r in results if ensure_exists(r)]
322     return results
323
324
325 def search_pd_authors(query):
326     pd_authors = Author.objects.filter(name__icontains=query)
327     existing_slugs = Tag.objects.filter(
328         category='author', slug__in=list(pd_authors.values_list('slug', flat=True))) \
329         .values_list('slug', flat=True)
330     pd_authors = pd_authors.exclude(slug__in=existing_slugs)
331     return pd_authors
332
333
334 def prepare_query(query):
335     query = ' '.join(query.split())
336     # filter out private use characters
337     import unicodedata
338     query = ''.join(ch for ch in query if unicodedata.category(ch) != 'Co')
339     query = remove_query_syntax_chars(query)
340
341     words = query.split()
342     if len(words) > 10:
343         query = ' '.join(words[:10])
344     return query