add submodule librarian
[edumed.git] / catalogue / management / commands / importlessons.py
1 # -*- coding: utf-8 -*-
2 # This file is part of EduMed, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 import os
6 import sys
7 import time
8 from optparse import make_option
9 from django.conf import settings
10 from django.core.management.base import BaseCommand
11 from django.core.management.color import color_style
12 from django.core.files import File
13
14 from librarian import IOFile
15 from catalogue.models import Lesson, Section
16
17 #from search import Index
18
19
20 class Command(BaseCommand):
21     option_list = BaseCommand.option_list + (
22         make_option('-q', '--quiet', action='store_false', dest='verbose', default=True,
23             help='Verbosity level; 0=minimal output, 1=normal output, 2=all output'),
24         make_option('-a', '--attachments', dest='attachments', metavar="PATH", default='materialy',
25             help='Attachments dir path.'),
26         make_option('--ignore-incomplete', action='store_true', dest='ignore_incomplete', default=False,
27             help='Attachments dir path.'),
28     )
29     help = 'Imports lessons from the specified directories.'
30     args = 'directory [directory ...]'
31
32     def import_book(self, file_path, options, attachments, ignore_incomplete=False):
33         verbose = options.get('verbose')
34         iofile = IOFile.from_filename(os.path.join(self.curdir, file_path))
35         iofile.attachments = attachments
36         return Lesson.publish(iofile, ignore_incomplete)
37
38     @staticmethod
39     def all_attachments(path):
40         files = {}
41
42         def read_dir(path):
43             for name in os.listdir(path):
44                 fullname = os.path.join(path, name)
45                 if os.path.isdir(fullname):
46                     read_dir(fullname)
47                 else:
48                     f = IOFile.from_filename(fullname)
49                     files[name.decode('utf-8')] = f
50                     files.setdefault(name.replace(" ", "").decode('utf-8'), f)
51
52         read_dir(path)
53         return files
54
55
56     def handle(self, *directories, **options):
57         from django.db import connection, transaction
58
59         levels = set()
60         self.style = color_style()
61         
62         verbose = options.get('verbose')
63         self.curdir = os.path.abspath(os.curdir)
64
65
66         # Start transaction management.
67         # SQLite will choke on generating thumbnails 
68         use_transaction = not connection.features.autocommits_when_autocommit_is_off
69         if use_transaction:
70             transaction.commit_unless_managed()
71             transaction.enter_transaction_management()
72             transaction.managed(True)
73         else:
74             print 'WARNING: Not using transaction management.'
75
76         files_imported = 0
77         files_skipped = 0
78
79         for dir_name in directories:
80             abs_dir = os.path.join(self.curdir, dir_name)
81             if not os.path.isdir(abs_dir):
82                 print self.style.ERROR("%s: Not a directory. Skipping." % abs_dir)
83             else:
84                 att_dir = os.path.join(abs_dir, options['attachments'])
85                 attachments = self.all_attachments(att_dir)
86
87                 # files queue
88                 files = sorted(os.listdir(abs_dir))
89                 postponed = {}
90                 ignore_incomplete = set()
91                 while files:
92                     file_name = files.pop(0)
93                     file_path = os.path.join(abs_dir, file_name)
94                     file_base, ext = os.path.splitext(file_path)
95
96                     # Skip files that are not XML files
97                     if not ext == '.xml':
98                         continue
99
100                     if verbose > 0:
101                         print "Parsing '%s'" % file_path
102                     else:
103                         sys.stdout.write('.')
104                         sys.stdout.flush()
105
106                     # Import book files
107                     try:
108                         lesson = self.import_book(file_path, options, attachments,
109                                     ignore_incomplete=file_name in ignore_incomplete)
110                     except Section.IncompleteError, e:
111                         if file_name not in postponed or postponed[file_name] < files_imported:
112                             # Push it back into the queue, maybe the missing lessons will show up.
113                             if verbose > 0:
114                                 print self.style.NOTICE('Waiting for missing lessons.')
115                             files.append(file_name)
116                             postponed[file_name] = files_imported
117                         elif options['ignore_incomplete'] and file_name not in ignore_incomplete:
118                             files.append(file_name)
119                             ignore_incomplete.add(file_name)
120                             postponed[file_name] = files_imported
121                         else:
122                             # We're in a loop, nothing's being imported - some lesson is really missing.
123                             raise e
124                     except BaseException, e:
125                         import traceback
126                         traceback.print_exc()
127                         files_skipped += 1
128                     else:
129                         files_imported += 1
130                         if use_transaction:
131                             transaction.commit()
132                         if hasattr(lesson, 'level'):
133                             levels.add(lesson.level)
134                     finally:
135                         if verbose > 0:
136                             print
137
138
139         if levels:
140             print "Rebuilding level packages:"
141             for level in levels:
142                 print level.name
143                 level.build_packages()
144
145         # Print results
146         print
147         print "Results: %d files imported, %d skipped, %d total." % (
148             files_imported, files_skipped, files_imported + files_skipped)
149         print
150
151         if use_transaction:
152             transaction.commit()
153             transaction.leave_transaction_management()