New export management commands

mep/accounts/management/commands/export_locations.py

@@ -0,0 +1,87 @@
+"""
+Manage command to export location data for use by others.
+
+Generates a CSV and JSON file including details on which member
+of the library lived where (if known) during what time period 
+(if known). The table includes summary details and coordinates 
+for associated addresses.
+"""
+
+from django.db.models import Prefetch
+from mep.common.management.export import BaseExport
+from mep.common.utils import absolutize_url
+from mep.accounts.models import Address
+
+
+class Command(BaseExport):
+    """Export member data."""
+
+    help = __doc__
+
+    model = Address
+
+    csv_fields = [
+        "member_id",  # member slug
+        "member_uri",
+        "care_of_person_id",  # person slug
+        "street_address",
+        "postal_code",
+        "city",
+        "arrondissement",
+        "country",
+        "start_date",
+        "end_date",
+        "longitude",
+        "latitude",
+    ]
+
+    # def get_queryset(self):
+    #     """
+    #     custom filter needed to return person-address combos,
+    #     so we can pass a one object per row to `get_object_data`
+    #     """
+    #     addresses = Address.objects.prefetch_related(
+    #         Prefetch("account"),
+    #         Prefetch("person"),
+    #         Prefetch("location"),
+    #     )
+    #     res = []
+    #     for addr in addresses.all():
+    #         persons = [addr.person] if addr.person else addr.account.persons.all()
+    #         for person in persons:
+    #             res.append((person, addr))
+    #     return res
+
+    def get_base_filename(self):
+        """set the filename to 'locations.csv'"""
+        return "locations"
+
+    def get_object_data(self, obj):
+        """
+        Generate dictionary of data to export for a single
+        :class:`~mep.people.models.Person`
+        """
+        addr = obj
+        loc = addr.location
+        persons = addr.account.persons.all()
+
+        # required properties
+        return dict(
+            # Member
+            member_id=[person.slug for person in persons],
+            member_uri=[
+                absolutize_url(person.get_absolute_url()) for person in persons
+            ],
+            # Address data
+            start_date=addr.partial_start_date,
+            end_date=addr.partial_end_date,
+            care_of_person_id=addr.care_of_person.slug if addr.care_of_person else None,
+            # Location data
+            street_address=loc.street_address,
+            city=loc.city,
+            postal_code=loc.postal_code,
+            latitude=float(loc.latitude) if loc.latitude is not None else None,
+            longitude=float(loc.longitude) if loc.longitude is not None else None,
+            country=loc.country.name if loc.country else None,
+            arrondissement=loc.arrondissement(),
+        )

mep/accounts/tests/test_accounts_commands.py

@@ -22,8 +22,9 @@
     export_events,
     import_figgy_cards,
     report_timegaps,
+    export_locations,
 )
-from mep.accounts.models import Account, Borrow, Event
+from mep.accounts.models import Account, Borrow, Event, Address, Location
 from mep.books.models import Creator, CreatorType
 from mep.common.management.export import StreamArray
 from mep.common.utils import absolutize_url
@@ -618,3 +619,48 @@ def test_command_line(self):
             call_command("export_events", "-d", tempdir.name, "-m", 2, stdout=stdout)
             # 2 objects (once each)
             assert mock_get_obj_data.call_count == 2
+
+
+class TestExportLocations(TestCase):
+    fixtures = ["sample_people"]
+
+    def setUp(self):
+        self.cmd = export_locations.Command()
+        self.cmd.stdout = StringIO()
+
+    def test_get_queryset(self):
+        # queryset should only include library members
+        member = Person.objects.get(pk=189)  # francisque gay, member
+        location = Location.objects.get(pk=213)
+        address = Address.objects.get(pk=236)
+        qs = self.cmd.get_queryset()
+        people, addresses = zip(*qs)
+        assert member in set(people)
+        assert address in set(addresses)
+        assert address.location == location
+        assert member in set(address.account.persons.all())
+
+    def test_get_object_data(self):
+        # fetch some example people from fixture & call get_object_data
+        gay = Person.objects.get(name="Francisque Gay")
+        address = Address.objects.get(pk=236)
+        gay_data = self.cmd.get_object_data((gay, address))
+
+        # check some basic data
+
+        # slug is 'gay' in sample_people, 'gay-francisque' in db
+        assert gay_data["member_id"] == "gay"
+        assert gay_data["member_uri"] == "https://example.com/members/gay/"
+
+        # check addresses & coordinates
+        assert "3 Rue Garancière" == gay_data["street_address"]
+        assert "Paris" == gay_data["city"]
+        assert "France" == gay_data["country"]
+        assert 48.85101 == gay_data["latitude"]
+        assert 2.33590 == gay_data["longitude"]
+        assert "75006" == gay_data["postal_code"]
+        assert 6 == gay_data["arrondissement"]
+        assert gay_data["start_date"] == "1919-01-01"
+        assert gay_data["start_date"] == "1919-01-01"
+        assert gay_data["end_date"] == "1930-01-01"
+        assert gay_data["care_of_person_id"] == "hemingway"

mep/books/fixtures/sample_works.json

@@ -9,6 +9,7 @@
 			"sort_title": "Exit Eliza",
 			"year": 1912,
 			"slug": "exit-eliza",
+			"category_id":1,
 			"work_format": 1,
 			"uri": "http://www.worldcat.org/oclc/2777112",
 			"updated_at": "2020-03-27T13:26:15Z"
@@ -24,6 +25,7 @@
 			"sort_title": "Grimm Fairy Tales",
 			"year": null,
 			"uri": "",
+			"category_id":1,
 			"slug": "grimm-fairy-tales",
 			"work_format": 1,
 			"updated_at": "2020-03-27T13:26:15Z"
@@ -38,6 +40,7 @@
 			"title": "The Kreutzer sonata",
 			"sort_title": "Kreutzer sonata",
 			"year": null,
+			"category_id":1,
 			"work_format": 1,
 			"uri": "http://worldcat.org/entity/work/id/4918580916",
 			"slug": "kreutzer-sonata",
@@ -52,6 +55,7 @@
 			"mep_id": null,
 			"title": "Chronicle of my Life",
 			"sort_title": "Chronicle of my Life",
+			"category_id":2,
 			"year": null,
 			"uri": "",
 			"slug": "chronicle-life",
@@ -68,6 +72,7 @@
 			"mep_id": null,
 			"title": "Murder on the Blue Train",
 			"sort_title": "Murder on the Blue Train",
+			"category_id":1,
 			"year": null,
 			"uri": "",
 			"slug": "murder-blue-train",
@@ -88,6 +93,7 @@
 			"edition_uri": "http://www.worldcat.org/oclc/243873605",
 			"ebook_url": "http://example.com",
 			"work_format": 2,
+			"category_id":2,
 			"slug": "dial",
 			"updated_at": "2020-03-27T13:26:15Z"
 		}
@@ -269,5 +275,19 @@
 			"notes": "",
 			"uri": "http://schema.org/Periodical"
 		}
+	},
+	{
+		"model": "books.genre",
+		"pk": 1,
+		"fields": {
+			"name": "Fiction"
+		}
+	},
+	{
+		"model": "books.genre",
+		"pk": 2,
+		"fields": {
+			"name": "Nonfiction"
+		}
 	}
 ]

mep/books/management/commands/export_books.py

@@ -8,9 +8,7 @@
 """
 
 from collections import OrderedDict
-
 from django.db.models import F
-
 from mep.books.models import CreatorType, Work
 from mep.common.management.export import BaseExport
 from mep.common.utils import absolutize_url
@@ -36,11 +34,14 @@ class Command(BaseExport):
     # query the database at load time (but maybe only a problem for tests)
 
     csv_fields = (
-        ["uri", "title"]
+        # including "id" to store slug for exports,
+        # given not all exported entities have a URI
+        ["id", "uri", "title"]
         + [creator.lower() for creator in creator_types]
         + [
             "year",
             "format",
+            "genre_category",
             "uncertain",
             "ebook_url",
             "volumes_issues",
@@ -77,17 +78,23 @@ def get_object_data(self, work):
         # required properties
         data = OrderedDict(
             [
+                ("id", work.slug),
                 ("uri", absolutize_url(work.get_absolute_url())),
                 ("title", work.title),
             ]
         )
         data.update(self.creator_info(work))
         if work.year:
             data["year"] = work.year
+
         # format is not currently set for all items
         if work.work_format:
             data["format"] = work.work_format.name
 
+        # genre category
+        if work.category:
+            data["genre_category"] = work.category.name
+
         data["uncertain"] = work.is_uncertain
 
         if work.ebook_url:
@@ -110,7 +117,6 @@ def get_object_data(self, work):
 
         # date last modified
         data["updated"] = work.updated_at.isoformat()
-
         return data
 
     def creator_info(self, work):

mep/books/tests/test_books_commands.py

@@ -406,6 +406,7 @@ def test_get_object_data(self):
         assert data["uri"] == absolutize_url(exit_e.get_absolute_url())
         assert data["title"] == exit_e.title
         assert data["year"] == exit_e.year
+        assert data["genre_category"] == exit_e.category.name
         assert data["format"] == exit_e.work_format.name
         assert not data["uncertain"]  # not marked uncertain
         assert "work_uri" not in data

mep/common/management/export.py

@@ -68,7 +68,8 @@ def add_arguments(self, parser):
         parser.add_argument(
             "-d",
             "--directory",
-            help="Specify the directory where files should be generated",
+            help="Specify the directory where files should be generated. "
+            "The directory will be created if it does not already exist.",
         )
         parser.add_argument(
             "-m",
@@ -94,6 +95,8 @@ def handle(self, *args, **kwargs):
         # get stream array / generator of data for export
         data = self.get_data(kwargs.get("max"))
         self.stdout.write("Exporting JSON and CSV")
+        # ensure directory exists (useful to allow command line user to specify dated dir)
+        os.makedirs(os.path.dirname(base_filename), exist_ok=True)
         # open and initialize CSV file
         with open("{}.csv".format(base_filename), "w") as csvfile:
             # write utf-8 byte order mark at the beginning of the file
@@ -146,7 +149,9 @@ def get_data(self, maximum=None):
         # grab the first N if maximum is specified
         if maximum:
             objects = objects[:maximum]
-        total = objects.count()
+        total = len(
+            objects
+        )  # fewer assumptions, allows other (multi model/class) objects
         return StreamArray((self.get_object_data(obj) for obj in objects), total)
 
     def get_object_data(self, obj):

mep/people/fixtures/sample_people.json

@@ -17,6 +17,15 @@
             "code": "FR"
         }
     },
+    {
+        "model": "people.country",
+        "pk": 3,
+        "fields": {
+            "name": "Ancient Greece",
+            "geonames_id": "http://sws.geonames.org/8354443/",
+            "code": "GR"
+        }
+    },
     {
         "model": "people.person",
         "pk": 189,
@@ -78,7 +87,9 @@
             "title": "",
             "profession": null,
             "slug": "aeschylus",
-            "nationalities": []
+            "nationalities": [
+                3
+            ]
         }
     },
     {
@@ -143,9 +154,9 @@
             "location": 213,
             "account": 4852,
             "person": null,
-            "start_date": null,
-            "end_date": null,
-            "care_of_person": null
+            "start_date": "1919-01-01",
+            "end_date": "1930-01-01",
+            "care_of_person": 224
         }
     },
     {

mep/people/management/commands/export_creators.py

@@ -0,0 +1,40 @@
+"""
+Manage command to export creator data for use by others.
+
+Generates a CSV and JSON file including details on every creator
+(author, translated, editor, etc) in the database, with details
+on creator nationality, gender, and other information.
+"""
+
+from mep.people.models import Person
+from mep.people.management.commands.export_members import Command as ExportMemberCommand
+
+
+class Command(ExportMemberCommand):
+    """Export creator data."""
+
+    csv_fields = [
+        "id",  # no URI for authors so using slug as ID
+        "name",
+        "sort_name",
+        "title",
+        "gender",
+        "is_organization",
+        "birth_year",
+        "death_year",
+        "viaf_url",
+        "wikipedia_url",
+        # related country
+        "nationalities",
+        # generic
+        "notes",
+        "updated",
+    ]
+
+    def get_queryset(self):
+        """filter to creators"""
+        return Person.objects.filter(creator__isnull=False).distinct()
+
+    def get_base_filename(self):
+        """set the filename to "creators.csv" since it's a subset of people"""
+        return "creators"