src/documents/management/commands/import_wl.py

   1 # This file is part of FNP-Redakcja, licensed under GNU Affero GPLv3 or later.
   2 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
   3 #
   4 from collections import defaultdict
   5 import json
   6 from urllib.request import urlopen
   7
   8 from django.core.management.base import BaseCommand
   9 from django.core.management.color import color_style
  10 from django.db import transaction
  11 from librarian.dcparser import BookInfo
  12 from librarian import ParseError, ValidationError
  13
  14 from documents.models import Book
  15
  16
  17 WL_API = 'http://www.wolnelektury.pl/api/books/'
  18
  19
  20 class Command(BaseCommand):
  21     help = 'Imports XML files from WL.'
  22
  23     def add_arguments(self, parser):
  24         parser.add_argument('-q', '--quiet', action='store_false', dest='verbose', default=True,
  25             help='Less output')
  26
  27     def handle(self, *args, **options):
  28
  29         self.style = color_style()
  30
  31         verbose = options.get('verbose')
  32
  33         # Start transaction management.
  34         transaction.enter_transaction_management()
  35
  36         if verbose:
  37             print('Reading currently managed files (skipping hidden ones).')
  38         slugs = defaultdict(list)
  39         for b in Book.objects.exclude(slug__startswith='.').all():
  40             if verbose:
  41                 print(b.slug)
  42             text = b.materialize().encode('utf-8')
  43             try:
  44                 info = BookInfo.from_bytes(text)
  45             except (ParseError, ValidationError):
  46                 pass
  47             else:
  48                 slugs[info.slug].append(b)
  49
  50         book_count = 0
  51         commit_args = {
  52             "author_name": 'Platforma',
  53             "description": 'Automatycznie zaimportowane z Wolnych Lektur',
  54             "publishable": True,
  55         }
  56
  57         if verbose:
  58             print('Opening books list')
  59         for book in json.load(urlopen(WL_API)):
  60             book_detail = json.load(urlopen(book['href']))
  61             xml_text = urlopen(book_detail['xml']).read()
  62             info = BookInfo.from_bytes(xml_text)
  63             previous_books = slugs.get(info.slug)
  64             if previous_books:
  65                 if len(previous_books) > 1:
  66                     print(self.style.ERROR("There is more than one book "
  67                         "with slug %s:") % info.slug)
  68                 previous_book = previous_books[0]
  69                 comm = previous_book.slug
  70             else:
  71                 previous_book = None
  72                 comm = '*'
  73             print(book_count, info.slug , '-->', comm)
  74             Book.import_xml_text(xml_text, title=info.title[:255],
  75                 slug=info.slug[:128], previous_book=previous_book,
  76                 commit_args=commit_args)
  77             book_count += 1
  78
  79         # Print results
  80         print()
  81         print("Results:")
  82         print("Imported %d books from WL:" % (
  83                 book_count, ))
  84         print()
  85
  86
  87         transaction.commit()
  88         transaction.leave_transaction_management()
  89