diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..a8c2a37 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,71 @@ +name: "Tests" + +on: + pull_request: + # Heroku's automatic deploys wait for these checks on the merged commit + push: + branches: + - "develop" + - "main" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 + + services: + postgres: + # matches docker-compose.yml + image: postgres:16 + env: + POSTGRES_DB: lasfera + POSTGRES_USER: lasfera + POSTGRES_PASSWORD: password + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U lasfera" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + env: + DJANGO_READ_DOT_ENV_FILE: "false" + DB_HOST: localhost + DB_PORT: "5432" + DB_NAME: lasfera + DB_USER: lasfera + DB_PASS: password + + steps: + - uses: actions/checkout@v4 + + # psycopg2 is built from source and needs the PostgreSQL client headers + - name: Install system packages + run: sudo apt-get update && sudo apt-get install -y libpq-dev + + # Poetry has to be installed before setup-python can cache its virtualenv + - name: Install Poetry + run: pipx install poetry==2.2.1 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: poetry + + - name: Install dependencies + run: poetry install --no-interaction --no-root + + - name: Check for missing migrations + run: poetry run python manage.py makemigrations --check --dry-run + + - name: Run tests + run: poetry run python manage.py test --noinput diff --git a/config/context_processors.py b/config/context_processors.py index 090ede4..727e5a8 100644 --- a/config/context_processors.py +++ b/config/context_processors.py @@ -4,3 +4,8 @@ def carto(request): """Expose the CARTO API key to templates for the gazetteer basemaps.""" return {"CARTO_API_KEY": settings.CARTO_API_KEY} + + +def test_site_banner(request): + """Whether to show the test site banner, on the site and in the admin.""" + return {"TEST_SITE_BANNER": settings.TEST_SITE_BANNER} diff --git a/config/settings.py b/config/settings.py index 4a499af..9b9815e 100644 --- a/config/settings.py +++ b/config/settings.py @@ -40,6 +40,9 @@ # CARTO API key for the gazetteer basemaps (client-exposed) CARTO_API_KEY = env("CARTO_API_KEY", default="") +# Show a banner on every page saying this is a test site whose changes won't +# carry over to the live site; turn on for staging +TEST_SITE_BANNER = env.bool("TEST_SITE_BANNER", default=False) # Application definition @@ -121,6 +124,7 @@ "django.contrib.messages.context_processors.messages", "django.template.context_processors.request", "config.context_processors.carto", + "config.context_processors.test_site_banner", ], }, }, diff --git a/config/tests.py b/config/tests.py new file mode 100644 index 0000000..ff5ed06 --- /dev/null +++ b/config/tests.py @@ -0,0 +1,15 @@ +from django.test import TestCase, override_settings +from django.urls import reverse + +BANNER = "Anything you add or change here may be removed" + + +class TestSiteBannerTest(TestCase): + def test_hidden_by_default(self): + self.assertNotContains(self.client.get(reverse("dmca")), BANNER) + self.assertNotContains(self.client.get(reverse("admin:login")), BANNER) + + @override_settings(TEST_SITE_BANNER=True) + def test_shown_on_site_and_admin(self): + self.assertContains(self.client.get(reverse("dmca")), BANNER) + self.assertContains(self.client.get(reverse("admin:login")), BANNER) diff --git a/manuscript/admin.py b/manuscript/admin.py index ab11857..02de4ed 100644 --- a/manuscript/admin.py +++ b/manuscript/admin.py @@ -1,12 +1,20 @@ +import json + from django import forms -from django.contrib import admin +from django.contrib import admin, messages +from django.contrib.admin.utils import unquote from django.contrib.contenttypes.admin import GenericTabularInline -from django.db.models import Count -from django.urls import reverse -from django.utils.html import format_html +from django.core.exceptions import PermissionDenied, ValidationError +from django.db.models import Count, Value +from django.db.models.functions import Concat +from django.shortcuts import redirect +from django.template.response import TemplateResponse +from django.urls import path, reverse +from django.utils.html import format_html, format_html_join from import_export.admin import ImportExportModelAdmin from common.admin import StripDivMixin +from manuscript.canvas_linking import clean_canvas_rows, link_canvases from manuscript.models import ( AuthorityFile, Codex, @@ -59,10 +67,75 @@ class StanzaInline(admin.StackedInline): ) -class FolioInline(admin.StackedInline): +class FolioAdminForm(forms.ModelForm): + class Meta: + model = Folio + fields = "__all__" + + def clean(self): + cleaned_data = super().clean() + if "canvas_id" in self.changed_data: + # the recorded canvas details describe the previously linked + # canvas, so drop them when the canvas ID is changed by hand + self.instance.canvas_index = None + self.instance.canvas_label = None + self.instance.image_id = None + return cleaned_data + + +def linked_image_link(folio, thumbnail=False): + """Link to a folio's row on the canvas linking page, labelled with the + linked image's position and manifest label""" + url = reverse( + "admin:manuscript_singlemanuscript_canvases", args=[folio.manuscript_id] + ) + if not folio.canvas_id: + return format_html('Not linked', url) + position = ( + f"Image {folio.canvas_index + 1}" if folio.canvas_index is not None else "Image" + ) + label = f" · {folio.canvas_label}" if folio.canvas_label else "" + image = "" + if thumbnail and folio.image_id: + # image_id is usually a IIIF image service; drop the preview if not + image = format_html( + '', + folio.image_id, + ) + # the canvas linking page scrolls to and highlights this folio's row + return format_html( + '{}{}{}', url, folio.pk, image, position, label + ) + + +class FolioInline(admin.TabularInline): + """Compact list of a manuscript's folios for editing folio numbers and line + ranges. Other details are edited on each folio's own page, and images are + linked on the Link IIIF canvases page. Folios aren't added or deleted here: + deleting a folio also deletes its toponym aliases.""" + model = Folio classes = ("collapse",) - extra = 1 + extra = 0 + can_delete = False + show_change_link = True + fields = ( + "folio_number", + "line_code_range_start", + "line_code_range_end", + "linked_image", + ) + readonly_fields = ("linked_image",) + ordering = ("canvas_index", "folio_number") + + def has_add_permission(self, request, obj=None): + return False + + @admin.display(description="Linked image") + def linked_image(self, folio): + return linked_image_link(folio) class DetailInline(admin.StackedInline): @@ -268,10 +341,121 @@ class SingleManuscriptAdmin(ImportExportModelAdmin): "library", ] + change_form_template = "admin/manuscript/singlemanuscript/change_form.html" + @admin.display(boolean=True, description="IIIF Available") def has_iiif_url(self, obj): return bool(obj.iiif_url) + def get_urls(self): + return [ + path( + "/canvases/", + self.admin_site.admin_view(self.link_canvases_view), + name="manuscript_singlemanuscript_canvases", + ), + *super().get_urls(), + ] + + def link_canvases_view(self, request, object_id): + """Page for linking the manuscript's folios to its IIIF canvases by hand""" + manuscript = self.get_object(request, unquote(object_id)) + if manuscript is None: + return self._get_obj_does_not_exist_redirect(request, self.opts, object_id) + if not request.user.has_perm("manuscript.change_folio"): + raise PermissionDenied + + change_url = reverse( + "admin:manuscript_singlemanuscript_change", args=[manuscript.pk] + ) + if not manuscript.iiif_url: + messages.warning(request, "This manuscript has no IIIF manifest URL.") + return redirect(change_url) + + can_create = request.user.has_perm("manuscript.add_folio") + submitted = None + if request.method == "POST": + try: + raw_rows = json.loads(request.POST.get("rows", "")) + except ValueError: + raw_rows = None + try: + rows = clean_canvas_rows(raw_rows) + counts, created = link_canvases(manuscript, rows, can_create=can_create) + except ValidationError as e: + for message in e.messages: + messages.error(request, message) + # keep what was entered so it can be corrected + if isinstance(raw_rows, list): + submitted = { + str(row.get("id")): str(row.get("folio") or "") + for row in raw_rows + if isinstance(row, dict) + } + else: + if counts: + messages.success( + request, + f"Saved: linked {counts['linked']}, unlinked " + f"{counts['unlinked']}, and created {counts['created']} " + f"new folio record{'' if counts['created'] == 1 else 's'}.", + ) + if created: + messages.warning( + request, + format_html( + "Add a line range to the new folio records so " + "the text can turn to them: {}", + format_html_join( + ", ", + '{}', + ( + ( + reverse( + "admin:manuscript_folio_change", + args=[folio.pk], + ), + folio.folio_number, + ) + for folio in created + ), + ), + ), + ) + else: + messages.info(request, "No changes to save.") + return redirect(request.path) + + folios = manuscript.folio_set.annotate( + toponym_count=Count("locationalias", distinct=True) + ) + context = { + **self.admin_site.each_context(request), + "opts": self.opts, + "title": f"Link IIIF canvases: {manuscript}", + "manuscript": manuscript, + "change_url": change_url, + "can_create": can_create, + "folios": [ + { + "id": folio.pk, + "number": folio.folio_number or "", + "canvas_id": folio.canvas_id, + "line_start": folio.line_code_range_start, + "line_end": folio.line_code_range_end, + "toponyms": folio.toponym_count, + "admin_url": reverse( + "admin:manuscript_folio_change", args=[folio.pk] + ), + } + for folio in folios + ], + "submitted": submitted, + } + return TemplateResponse( + request, "admin/manuscript/singlemanuscript/link_canvases.html", context + ) + class Media: js = ("js/text_annotations.js",) css = {"all": ("css/text_annotator.css",)} @@ -283,20 +467,46 @@ def get_queryset(self, request): @admin.register(Folio) class FolioAdmin(ImportExportModelAdmin): resource_class = FolioResource + form = FolioAdminForm - list_display = ["manuscript", "folio_number", "line_range_display", "stanza_count"] - list_filter = ["manuscript"] + list_display = [ + "folio_display", + "folio_number", + "canvas_label", + "line_range_display", + "stanza_count", + ] + list_select_related = ["manuscript"] + ordering = ["manuscript__siglum", "folio_number"] + list_filter = ["manuscript", ("canvas_id", admin.EmptyFieldListFilter)] search_fields = [ "folio_number", "manuscript__siglum", "line_code_range_start", "line_code_range_end", + "canvas_label", ] - readonly_fields = ["stanza_list", "line_range_display", "stanza_count"] + readonly_fields = [ + "stanza_list", + "line_range_display", + "stanza_count", + "linked_image", + "canvas_index", + "canvas_label", + "image_id", + "toponym_list", + ] fieldsets = ( ("Folio Information", {"fields": ("manuscript", "folio_number")}), + ( + "IIIF Canvas", + { + # the linked image shows the canvas position, counting from 1 + "fields": ("linked_image", "canvas_id", "canvas_label", "image_id"), + }, + ), ( "Line Range", { @@ -313,14 +523,70 @@ class FolioAdmin(ImportExportModelAdmin): "fields": ("stanza_count", "stanza_list"), }, ), + ("Toponyms", {"fields": ("toponym_list",)}), + ("Notes", {"fields": ("folio_includes_map", "folio_notes")}), ) + @admin.display(description="Linked image") + def linked_image(self, obj): + if not obj.pk or not obj.manuscript_id: + return "-" + return linked_image_link(obj, thumbnail=True) + + @admin.display(description="Toponym aliases on this folio") + def toponym_list(self, obj): + if not obj.pk: + return "-" + aliases = obj.locationalias_set.select_related("location").order_by( + "placename_alias", "id" + ) + if not aliases: + return "None" + rows = [] + for alias in aliases: + if alias.location: + location = alias.location + name = location.name or location.placename_modern or "" + toponym = format_html( + '{} {}', + reverse("admin:manuscript_location_change", args=[location.pk]), + name or location.placename_id, + f"({location.placename_id})" if name else "", + ) + else: + toponym = "-" + rows.append( + ( + reverse("admin:manuscript_locationalias_change", args=[alias.pk]), + alias.placename_alias or f"Alias {alias.pk}", + toponym, + ) + ) + return format_html( + '
' + '' + "{}
AliasToponym
", + format_html_join( + "", '{}{}', rows + ), + ) + def stanza_count(self, obj): return obj.stanzas.count() stanza_count.short_description = "Number of Stanzas" + @admin.display( + description="Folio", + ordering=Concat("manuscript__siglum", Value(" "), "folio_number"), + ) + def folio_display(self, obj): + manuscript = obj.manuscript or "No manuscript" + return f"{manuscript} · {obj.folio_number or '(no folio number)'}" + def line_range_display(self, obj): + if not obj.line_code_range_start: + return "-" end = obj.line_code_range_end or "End" return f"{obj.line_code_range_start} → {end}" diff --git a/manuscript/canvas_linking.py b/manuscript/canvas_linking.py new file mode 100644 index 0000000..92b8176 --- /dev/null +++ b/manuscript/canvas_linking.py @@ -0,0 +1,169 @@ +"""Linking a manuscript's folios to the canvases of its IIIF manifest by hand""" + +from collections import Counter, defaultdict + +from django.core.exceptions import ValidationError +from django.db import transaction + +from manuscript.models import Folio +from manuscript.utils import normalize_folio_label + +FOLIO_NUMBER_MAX_LENGTH = Folio._meta.get_field("folio_number").max_length +CANVAS_ID_MAX_LENGTH = Folio._meta.get_field("canvas_id").max_length + + +def clean_canvas_rows(rows): + """Validate the rows submitted from the canvas linking page. Each row is a + canvas from the manifest: id, index, label, image_id, and the folio + number entered for it (blank for none).""" + if not isinstance(rows, list): + raise ValidationError("The submitted images could not be read.") + + cleaned = [] + for row in rows: + try: + canvas_id = str(row["id"]).strip() + index = int(row["index"]) + label = str(row.get("label") or "").strip()[:255] + image_id = str(row.get("image_id") or "").strip() or None + folio_number = str(row.get("folio") or "").strip() + except (KeyError, TypeError, ValueError): + raise ValidationError("The submitted images could not be read.") + if not canvas_id or len(canvas_id) > CANVAS_ID_MAX_LENGTH or index < 0: + raise ValidationError("The submitted images could not be read.") + if image_id and len(image_id) > CANVAS_ID_MAX_LENGTH: + image_id = None + if len(folio_number) > FOLIO_NUMBER_MAX_LENGTH: + raise ValidationError( + f"The folio number entered for image {index + 1} is too long." + ) + cleaned.append( + { + "id": canvas_id, + "index": index, + "label": label, + "image_id": image_id, + "folio": folio_number, + } + ) + + duplicate_ids = [ + canvas_id + for canvas_id, count in Counter(row["id"] for row in cleaned).items() + if count > 1 + ] + if duplicate_ids: + raise ValidationError("The manifest lists the same image more than once.") + return cleaned + + +def link_canvases(manuscript, rows, can_create=True): + """Link the manuscript's folios to canvases as entered on the canvas linking + page, where rows are cleaned by clean_canvas_rows. + + A folio number is matched to an existing folio of the manuscript (ignoring + differences like "f. 12r" and "12r"), or a new folio is created. Folios are + never renamed or deleted: a folio whose canvas is cleared or given to + another folio is only unlinked, so its toponyms and other data stay intact. + + Returns a Counter of linked, unlinked, and created folios, and a list of + the folios created. Raises ValidationError, without saving anything, if + the entries conflict.""" + errors = [] + + # folio number entered -> row, checking no folio is entered twice + rows_by_folio = {} + positions_by_folio = defaultdict(list) + for row in rows: + if row["folio"]: + key = normalize_folio_label(row["folio"]) or row["folio"].lower() + rows_by_folio[key] = row + positions_by_folio[key].append(row) + for key, entered in positions_by_folio.items(): + if len(entered) > 1: + positions = ", ".join(str(row["index"] + 1) for row in entered) + errors.append( + f'Folio "{entered[0]["folio"]}" is entered for more than one ' + f"image ({positions}). Each folio can be linked to one image." + ) + + folios = list(manuscript.folio_set.all()) + folios_by_key = defaultdict(list) + for folio in folios: + key = ( + normalize_folio_label(folio.folio_number) + or (folio.folio_number or "").lower() + ) + if key: + folios_by_key[key].append(folio) + + targets = {} # folio number key -> existing folio, or None to create + for key, row in rows_by_folio.items(): + matches = folios_by_key.get(key, []) + if len(matches) > 1: + numbers = ", ".join(sorted(f'"{f.folio_number}"' for f in matches)) + errors.append( + f'Folio "{row["folio"]}" matches several folio records in this ' + f"manuscript ({numbers}). Merge or renumber those folio records " + "first." + ) + elif matches: + targets[key] = matches[0] + elif not can_create: + errors.append( + f'There is no folio record "{row["folio"]}" in this manuscript, ' + "and you don't have permission to add folio records." + ) + else: + targets[key] = None + + if errors: + raise ValidationError(errors) + + counts = Counter() + created = [] + submitted_ids = {row["id"] for row in rows} + linked_folio_ids = { + folio.pk: rows_by_folio[key]["id"] + for key, folio in targets.items() + if folio is not None + } + with transaction.atomic(): + # unlink first, so moving a folio to another canvas can't collide + # with the one-folio-per-canvas constraint + for folio in folios: + if folio.canvas_id not in submitted_ids: + continue + if linked_folio_ids.get(folio.pk) == folio.canvas_id: + continue + if folio.pk not in linked_folio_ids: + counts["unlinked"] += 1 + folio.canvas_id = None + folio.canvas_index = None + folio.canvas_label = None + folio.image_id = None + folio.save( + update_fields=["canvas_id", "canvas_index", "canvas_label", "image_id"] + ) + + for key, folio in targets.items(): + row = rows_by_folio[key] + if folio is None: + folio = Folio(manuscript=manuscript, folio_number=row["folio"]) + created.append(folio) + counts["created"] += 1 + elif ( + folio.canvas_id == row["id"] + and folio.canvas_index == row["index"] + and folio.canvas_label == row["label"] + and folio.image_id == row["image_id"] + ): + continue + if folio.canvas_id != row["id"]: + counts["linked"] += 1 + folio.canvas_id = row["id"] + folio.canvas_index = row["index"] + folio.canvas_label = row["label"] + folio.image_id = row["image_id"] + folio.save() + return counts, created diff --git a/manuscript/management/commands/link_folio_canvases.py b/manuscript/management/commands/link_folio_canvases.py new file mode 100644 index 0000000..2f8ad0f --- /dev/null +++ b/manuscript/management/commands/link_folio_canvases.py @@ -0,0 +1,215 @@ +import re +from collections import Counter, defaultdict + +from django.core.management.base import BaseCommand +from django.db import transaction + +from manuscript.models import SingleManuscript +from manuscript.utils import fetch_manifest, get_canvases, normalize_folio_label + +# folio numbers copied from the viewer's page menu: " :