[#40] Bulk PDF export

- backend changes

Makefile

@@ -28,6 +28,16 @@ ingest:
 	docker-compose exec backend python manage.py loaddata \
 		/app/right_tree/api/data/fixtures/plants.json
 
+ingest_linz:
+	docker-compose up -d postgres
+	docker-compose exec -T postgres pg_restore -U righttree -d righttree -n linz -Fc < linz.dump
+	docker-compose exec -T postgres psql -U righttree -d righttree -f - < create_indices.sql
+
+migrate:
+	docker-compose up -d backend postgres
+	docker-compose exec backend python manage.py makemigrations --noinput
+	docker-compose exec backend python manage.py migrate --noinput
+
 createsuperuser:
 	docker-compose up -d backend
 	docker-compose exec backend python manage.py createsuperuser

backend/.gitignore

@@ -1,19 +1,9 @@
 *.pyc
-*.sqlite3
 __pycache__
 
-right_tree/api/data/resources/*.cpg
+right_tree/api/data/resources
-right_tree/api/data/resources/*.dbf
+!right_tree/api/data/resources/plant_data.xlsx
-right_tree/api/data/resources/*.pdf
-right_tree/api/data/resources/*.prj
-right_tree/api/data/resources/*.sbn
-right_tree/api/data/resources/*.sbx
-right_tree/api/data/resources/*.shp
-right_tree/api/data/resources/*.shx
-right_tree/api/data/resources/*.txt
-right_tree/api/data/resources/*.xml
-right_tree/api/data/resources/*.zip
 
 right_tree/api/data/fixtures/plants.json
+right_tree/media
 right_tree/staticfiles
-right_tree/api/data/generated_resources/*

backend/Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.11-slim-bullseye
 WORKDIR /app
 
 RUN apt update \
-    && apt install -y --no-install-recommends gdal-bin \
+    && apt install -y --no-install-recommends gdal-bin wkhtmltopdf \
     && rm -rf /var/lib/apt/lists/* \
     && apt clean
 
@@ -12,3 +12,5 @@ COPY ./requirements.txt /app/requirements.txt
 RUN pip install -U --no-cache-dir -r requirements.txt
 
 COPY . /app
+
+ENV DJANGO_SETTINGS_MODULE="right_tree.settings"

backend/requirements.txt

@@ -8,3 +8,4 @@ gunicorn==20.1.0
 pandas==1.5.3
 pdfkit==1.0.0
 PyPDF2==1.28.6
+celery[redis]==5.2.7

backend/right_tree/api/admin.py

@@ -1,5 +1,10 @@
-from django.contrib import admin
+from django.contrib import admin, messages
-import right_tree.api.models as models
+from django.http import HttpResponseRedirect, FileResponse
+from django.utils.text import slugify
+from django.utils import timezone
+
+from right_tree.api import models
+from right_tree.api.resource_generation_utils import storage
 
 
 class ZoneAdmin(admin.ModelAdmin):
@@ -7,6 +12,71 @@ class ZoneAdmin(admin.ModelAdmin):
     search_fields = ['name', 'habitat__name', 'variant', 'refined_variant', 'id']
 
 
+class QuestionnaireAdmin(admin.ModelAdmin):
+    list_display = ['address_display', 'location_display', 'soil_variant', 'ecological_district', 'habitat', 'zone']
+    actions = ['export']
+
+    @admin.display(description="Address")
+    def address_display(self, obj):
+        return obj.address.full_address
+
+    @admin.display(description="Location")
+    def location_display(self, obj):
+        return f"({obj.location.x}, {obj.location.y})"
+
+    @admin.action(description="Export planting guides for selected questionnaires")
+    def export(self, request, queryset):
+        export = models.Export.objects.create(creation_date=timezone.now())
+        export.questionnaires.set(queryset)
+        export.export()
+
+        return HttpResponseRedirect('/admin/api/export')
+
+
+class ExportAdmin(admin.ModelAdmin):
+    list_display = ['creation_date', 'completion_date', 'completion_display']
+    actions = ['download']
+
+    @admin.display(description="Completion")
+    def completion_display(self, obj):
+        return f"{obj.completion:.1%}"
+
+    @admin.action(description="Download completed exports")
+    def download(self, request, queryset):
+        if queryset.count() > 1:
+            self.message_user(
+                request,
+                'Cannot download more than one export at a time.',
+                messages.ERROR,
+            )
+            return
+
+        export = queryset.first()
+
+        if not export.complete:
+            self.message_user(
+                request,
+                'Cannot download. Export is incomplete.',
+                messages.ERROR,
+            )
+            return
+
+        filename = f"export_{export.pk}/export.zip"
+        filepath = storage.path(filename)
+
+        if storage.exists(filename):
+            return FileResponse(
+                open(filepath, 'rb'),
+                filename=f"export_{slugify(export.creation_date)}.zip",
+            )
+
+        self.message_user(
+            request,
+            'Cannot download. Export is corrupt.',
+            messages.ERROR,
+        )
+
+
 admin.site.register(models.Plant)
 admin.site.register(models.SoilOrder)
 admin.site.register(models.SoilLayer)
@@ -18,3 +88,5 @@ admin.site.register(models.HabitatImage)
 admin.site.register(models.Habitat)
 admin.site.register(models.Zone, ZoneAdmin)
 admin.site.register(models.ChristchurchRegion)
+admin.site.register(models.Questionnaire, QuestionnaireAdmin)
+admin.site.register(models.Export, ExportAdmin)

backend/right_tree/api/apps.py

@@ -3,3 +3,6 @@ from django.apps import AppConfig
 
 class ApiConfig(AppConfig):
     name = 'right_tree.api'
+
+    def ready(self):
+        import right_tree.api.signals

backend/right_tree/api/celery.py

@@ -0,0 +1,5 @@
+from celery import Celery
+
+app = Celery('righttree')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks()

backend/right_tree/api/migrations/0014_address_export_questionnaire.py

@@ -0,0 +1,49 @@
+# Generated by Django 3.2.17 on 2023-02-21 01:38
+
+import django.contrib.gis.db.models.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0013_zone_tooltiptext'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Address',
+            fields=[
+                ('ogc_fid', models.AutoField(primary_key=True, serialize=False)),
+                ('address_number', models.IntegerField()),
+                ('suburb_locality_ascii', models.CharField(max_length=255)),
+                ('town_city_ascii', models.CharField(max_length=255)),
+                ('full_road_name_ascii', models.CharField(max_length=255)),
+                ('wkb_geometry', django.contrib.gis.db.models.fields.PointField(srid=4326)),
+                ('full_address', models.CharField(max_length=500)),
+            ],
+            options={
+                'db_table': 'linz"."nz_street_address',
+                'managed': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Questionnaire',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('location', django.contrib.gis.db.models.fields.PointField(srid=4326)),
+                ('soil_variant', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.soilvariant')),
+                ('zone', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.zone')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Export',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('creation_date', models.DateTimeField()),
+                ('completion_date', models.DateTimeField(null=True)),
+                ('questionnaires', models.ManyToManyField(to='api.Questionnaire')),
+            ],
+        ),
+    ]

backend/right_tree/api/models.py

@@ -1,4 +1,12 @@
+from functools import cached_property
+from pathlib import Path
+
+from django.db.models.functions import Lower
 from django.contrib.gis.db import models
+from django.contrib.gis.db.models.functions import Distance
+from django.contrib.postgres.indexes import OpClass
+from django.utils.text import slugify
+
 
 
 class SoilOrder(models.Model):
@@ -162,3 +170,111 @@ class Plant(models.Model):
 
     class Meta:
         ordering = ['stage', 'name', 'commonname', 'synonym']
+
+
+class Address(models.Model):
+    ogc_fid = models.AutoField(primary_key=True)
+
+    # fields for queries
+    address_number = models.IntegerField()  # ignore the 'A' in '32A' for now
+    suburb_locality_ascii = models.CharField(max_length=255)
+    town_city_ascii = models.CharField(max_length=255)
+    full_road_name_ascii = models.CharField(max_length=255)
+    wkb_geometry = models.PointField()
+
+    # use only for displaying to user
+    full_address = models.CharField(max_length=500)
+
+    class Meta:
+        db_table = 'linz\".\"nz_street_address'
+        indexes = [
+            models.Index(OpClass(Lower('full_road_name_ascii'), name='text_pattern_ops'), name='full_road_name_lower_idx'),
+            models.Index(OpClass(Lower('suburb_locality_ascii'), name='text_pattern_ops'), name='suburb_locality_lower_idx'),
+            models.Index(OpClass(Lower('town_city_ascii'), name='text_pattern_ops'), name='town_city_lower_idx'),
+            models.Index(fields=['address_number'], name='address_number_idx'),
+        ]
+
+        # effectively read-only
+        # https://docs.djangoproject.com/en/3.2/ref/models/options/#managed
+        managed = False
+
+
+class Questionnaire(models.Model):
+    location = models.PointField()
+    soil_variant = models.ForeignKey(SoilVariant, on_delete=models.CASCADE)
+    zone = models.ForeignKey(Zone, on_delete=models.CASCADE)
+
+    @property
+    def habitat(self):
+        return self.zone.habitat
+
+    @property
+    def ecological_district_layer(self):
+        return EcologicalDistrictLayer.objects.filter(geom__intersects=self.location).first()
+
+    @property
+    def ecological_district(self):
+        return self.ecological_district_layer.ecologic_2
+
+    @cached_property
+    def address(self):
+        return \
+            Address.objects.filter(
+                wkb_geometry__dwithin=(self.location, 1e-3),  # order of ~100 metres in degrees for NZ
+            ).annotate(
+                distance=Distance('wkb_geometry', self.location),
+            ).order_by(
+                'distance',
+            ).first()
+
+    @property
+    def plants(self):
+        return self.zone.plant_set.all() & self.soil_variant.plant_set.all()
+
+    @cached_property
+    def slug(self):
+        return "_".join(
+            map(slugify, [
+                self.address.full_address if self.address else str(self.location),
+                self.zone.name,
+                self.zone.variant,
+                self.zone.refined_variant,
+                self.soil_variant.name,
+                str(self.zone.pk),  # need this for uniqueness
+                str(self.soil_variant.pk),  # # need this for uniqueness
+            ])
+        )
+
+
+class Export(models.Model):
+    creation_date = models.DateTimeField()
+    completion_date = models.DateTimeField(null=True)
+    questionnaires = models.ManyToManyField(Questionnaire)
+
+    @property
+    def complete(self):
+        return self.completion_date is not None
+
+    @property
+    def completion(self):
+        if self.complete:
+            return 1.0
+
+        from .resource_generation_utils import storage
+
+        _, files = storage.listdir(f"export_{self.pk}")
+
+        # halved as there are two files (csv, pdf) per questionnaire
+        return 0.5 * len(files) / self.questionnaires.count()
+
+    def export(self):
+        from .tasks import generate_pdf
+        from .resource_generation_utils import storage
+
+        parent = Path(storage.path(f"export_{self.pk}"))
+
+        if not parent.exists():
+            parent.mkdir()
+
+        for q in self.questionnaires.all():
+            generate_pdf.delay(q.pk, self.pk)

backend/right_tree/api/resource_generation_utils.py

@@ -1,8 +1,7 @@
 import csv
-from os import write
+from io import StringIO
-from pathlib import Path
+from os.path import splitext
 
-import right_tree.api.data
 from .filters import *
 from .wms_utils import get_address_from_coordinates, get_point_from_coordinates
 
@@ -10,6 +9,12 @@ import pdfkit
 import pandas as pd
 from PyPDF2 import PdfFileMerger
 
+from django.core.files.storage import get_storage_class
+
+
+Storage = get_storage_class()
+storage = Storage()
+
 
 CSV_FILENAME = 'plants.csv'
 PLANTING_GUIDE_PDF_FILENAME = 'planting_guide.pdf'
@@ -17,17 +22,11 @@ HEADER_FIELDS = ['Names', 'Growth Form / Max Height (m) / Spacing (m) / Forest P
                  'Tolerances (Water / Drought / Frost / Salinity)', 'Ecosystem Services', 'Carbon Sequestration Rate', 'Planting Stage']
 
 
-def get_plant_resource_filepath(filename=CSV_FILENAME, resource_dir='generated_resources'):
+def get_location_filters(params):
-    """ Retrives the filepath for the plant csv file.
-    """
-    return Path(right_tree.api.data.__file__).resolve().parent / resource_dir / filename
-
-
-def get_location_filters(request):
     """ Retrives the selected location data from the request.
     """
     filter_rows = [['LOCATION FILTERS:', ' ']]
-    coordinates = request.query_params.get('coordinates')
+    coordinates = params.get('coordinates')
 
     if coordinates is not None:
         eco_district_layer = ecological_district_coordinate_filter(
@@ -45,12 +44,12 @@ def get_location_filters(request):
     return filter_rows
 
 
-def get_soil_filters(request):
+def get_soil_filters(params):
-    """ Retrives the selected soil type data from the request.
+    """ Retrives the selected soil type data from the request params.
     """
     filter_rows = [['SOIL FILTERS:', ' ']]
-    soil_variant = request.query_params.get('soilVariant')
+    soil_variant = params.get('soilVariant')
-    coordinates = request.query_params.get('coordinates')
+    coordinates = params.get('coordinates')
 
     if soil_variant is not None and coordinates is not None:
         soil_order_obj = soil_order_coordinate_filter(coordinates).first()
@@ -63,13 +62,13 @@ def get_soil_filters(request):
     return filter_rows
 
 
-def get_site_filters(request):
+def get_site_filters(params):
-    """ Retrives the selected site data from the request.
+    """ Retrives the selected site data from the request params
     """
     filter_rows = [['SITE FILTERS:', ' ']]
 
-    habitat = request.query_params.get('habitat')
+    habitat = params.get('habitat')
-    zone = request.query_params.get('zone')
+    zone = params.get('zone')
     if zone is not None and habitat is not None:
         habitat_json = json.loads(habitat)
         zone_json = json.loads(zone)
@@ -83,10 +82,10 @@ def get_site_filters(request):
     return filter_rows
 
 
-def get_additional_region_info(request):
+def get_additional_region_info(params):
     """ If the location coordinates fall within the CHCH or Auckland regions then return a description of where to find more information.
     """
-    coordinates = request.query_params.get('coordinates')
+    coordinates = params.get('coordinates')
     if coordinates is not None:
         if is_in_christchurch(coordinates):
             return [["Your location falls within the ecosystem type covered by the Christchurch Council ecosystem maps - further information can be obtained from Ōtautahi/Christchurch ecosystems map link to: https://ccc.govt.nz/environment/land/ecosystem-map", " "], [' ', ' ']]
@@ -95,41 +94,73 @@ def get_additional_region_info(request):
 
     return []
 
-def get_filter_values(request):
+def get_filter_values(params):
-    """ Retrives all selected values/filters from the request.
+    """ Retrives all selected values/filters from the request parameters
     """
     filter_rows = []
 
     # Add all the location filters
-    filter_rows += get_location_filters(request)
+    filter_rows += get_location_filters(params)
     filter_rows.append([' ', ' '])
 
     # Add the soil filters
-    filter_rows += get_soil_filters(request)
+    filter_rows += get_soil_filters(params)
     filter_rows.append([' ', ' '])
 
     # Add the project site filters
-    filter_rows += get_site_filters(request)
+    filter_rows += get_site_filters(params)
     filter_rows.append([' ', ' '])
 
-    filter_rows += get_additional_region_info(request)
+    filter_rows += get_additional_region_info(params)
 
     return filter_rows
 
 
-def write_csv_filter_info(request, writer):
+def generate_csv(data: list[list[str]], output_filename: str) -> str:
-    """ Retrieves and writes filter information to a CSV file given a writer.
+    with storage.open(output_filename, 'w') as f:
-    """
+        csv.writer(f).writerows(data)
-    for filter_row in get_filter_values(request):
+
-        writer.writerow(filter_row)
+    return storage.path(output_filename)
 
 
-def write_csv_plant_info(plant_data, writer):
+def generate_pdf(data: list[list[str]], output_filename: str):
-    """ Writes plant list information to a CSV file given a writer.
+    """ Generates a pdf from a csv given data and a csv generation method.
+        Requires an html file to be generated as an intermediate step.
     """
-    writer.writerow(HEADER_FIELDS)
+
-    for plant in plant_data:
+    name, _ = splitext(output_filename)
-        plant_data_row = [
+    csv_filepath = generate_csv(data, f"{name}.csv")
+    pdf_filepath = storage.path(output_filename)
+
+    with StringIO() as html_buf:
+        # Convert csv to html
+        pd.read_csv(csv_filepath).to_html(html_buf)  # reading from buffer causes segfault :/
+        html_buf.seek(0)
+
+        # Convert html to pdf
+        pdfkit.from_file(html_buf, pdf_filepath)
+
+    return pdf_filepath
+
+
+def merge_pdfs(pdfs: list[str], output_filename):
+    """Merge a list of PDF filenames"""
+
+    output_filepath = storage.path(output_filename)
+    merger = PdfFileMerger()
+
+    for pdf in pdfs:
+        merger.append(pdf)
+
+    merger.write(output_filepath)
+    merger.close()
+
+    return output_filepath
+
+
+def serialize_plants_queryset(plants_queryset):
+    return [HEADER_FIELDS] + [
+        [
             plant.display_name,
             plant.display_growth_form,
             plant.moisture_preferences,
@@ -137,69 +168,20 @@ def write_csv_plant_info(plant_data, writer):
             plant.ecosystem_services,
             plant.carbon_sequestration,
             plant.stage
+        ] for plant in plants_queryset
     ]
-        writer.writerow(plant_data_row)
 
 
-def create_plant_csv_file(request, plant_data):
+def create_planting_guide_pdf(filter_data, plant_data, output_filename):
-    """ Constructs a csv file that contains selected filter values and the resulting plant list.
-    """
-    with open(get_plant_resource_filepath(), 'w', encoding='UTF8') as f:
-        writer = csv.writer(f)
-
-        # Write filter/selected values
-        write_csv_filter_info(request, writer)
-
-        # Write the plant data
-        write_csv_plant_info(plant_data, writer)
-
-
-def generate_pdf(data_name, data, csv_generation_function):
-    """ Generates a pdf from a csv given data and a csv generation method. 
-        Requires an html file to be generated as an intermediate step.
-    """
-
-    # Define filepaths - some are required as intermediate files
-    csv_filepath = get_plant_resource_filepath(f'{data_name}.csv')
-    html_filepath = get_plant_resource_filepath(f'{data_name}.html')
-    pdf_filepath = get_plant_resource_filepath(f'{data_name}.pdf')
-
-    # Create an initial csv file with the data
-    with open(csv_filepath, 'w', encoding='UTF8') as f:
-        writer = csv.writer(f)
-        csv_generation_function(data, writer)
-
-    # Convert the csv to and html file then to a pdf
-    CSV = pd.read_csv(csv_filepath)
-    CSV.to_html(html_filepath)
-    pdfkit.from_file(str(html_filepath), str(pdf_filepath))
-
-
-def create_planting_guide_pdf(request, plant_data):
     """ Creates a planting guide pdf document with a pre-generated planting guide with
         filter and plant list tabular informtation appended.
     """
-
+    # TODO: space values appear as NaN... this should be fixed
-    # Define the names of the resource files as used in multiple places
+    return merge_pdfs(
-    filter_file_prefix = "filters"
+        [
-    plant_list_file_prefix = "plant_list"
+            storage.path(PLANTING_GUIDE_PDF_FILENAME),
-
+            generate_pdf(filter_data, f"{output_filename}.filters"),
-    # Generate the filters and plant list pdf files
+            generate_pdf(plant_data, f"{output_filename}.plants"),
-    generate_pdf(filter_file_prefix, request, write_csv_filter_info) # TODO: space values appear as NaN... this should be fixed
+        ],
-    generate_pdf(plant_list_file_prefix, plant_data, write_csv_plant_info)
+        output_filename,
-
+    )
-    # Create a list of pdfs that need to be merged
-    planting_guide_pdf_filepath = get_plant_resource_filepath(PLANTING_GUIDE_PDF_FILENAME, 'resources')
-    filter_pdf_filepath = get_plant_resource_filepath(f'{filter_file_prefix}.pdf')
-    plant_list_pdf_filepath = get_plant_resource_filepath(f'{plant_list_file_prefix}.pdf')
-    pdfs = [planting_guide_pdf_filepath, filter_pdf_filepath, plant_list_pdf_filepath]
-
-    # Merge pdfs
-    merger = PdfFileMerger()
-    for pdf in pdfs:
-        merger.append(str(pdf))
-
-    # Create the final output pdf with all merged documents
-    output_pdf_filepath = get_plant_resource_filepath(PLANTING_GUIDE_PDF_FILENAME)
-    merger.write(str(output_pdf_filepath))
-    merger.close()

backend/right_tree/api/serializers.py

@@ -1,4 +1,4 @@
-from rest_framework import serializers
+from rest_framework import serializers, exceptions
 from right_tree.api.models import *
 
 
@@ -95,3 +95,18 @@ class PlantSerializer(serializers.HyperlinkedModelSerializer):
 
 class AddressSerializer(serializers.Serializer):
     full_address = serializers.CharField(max_length=500)
+
+
+class QuestionnaireSerializer(serializers.ModelSerializer):
+
+    soil_variant = serializers.CharField(max_length=10)
+
+    class Meta:
+        model = Questionnaire
+        fields = '__all__'
+
+    def validate_soil_variant(self, value):
+        try:
+            return SoilVariant.objects.get(name__startswith=value)
+        except SoilVariant.DoesNotExist as e:
+            raise exceptions.ValidationError(e)

backend/right_tree/api/signals.py

@@ -0,0 +1,12 @@
+from shutil import rmtree
+
+from django.db.models.signals import post_delete
+from django.dispatch import receiver
+
+from .models import Export
+from .resource_generation_utils import storage
+
+
+@receiver(post_delete, sender=Export)
+def delete_export(sender, instance, *args, **kwargs):
+    rmtree(storage.path(f"export_{instance.pk}"))

backend/right_tree/api/tasks.py

@@ -0,0 +1,53 @@
+import json
+import logging
+import pathlib
+import zipfile
+
+from celery import shared_task
+
+from django.utils import timezone
+
+from .models import Questionnaire, Export
+from .resource_generation_utils import create_planting_guide_pdf, get_filter_values, serialize_plants_queryset, storage
+
+
+@shared_task
+def generate_pdf(questionnaire_id, export_id):
+    q = Questionnaire.objects.get(pk=questionnaire_id)
+    e = Export.objects.get(pk=export_id)
+    z = q.zone
+    filename = f"export_{e.pk}/{q.slug}.pdf"
+
+    try:
+        create_planting_guide_pdf(
+            get_filter_values({  # awful hack to reuse some logic we already have
+                'coordinates': {'lat': q.location.y, 'lng': q.location.x},
+                'soilVariant': q.soil_variant.name[0],
+                'habitat': json.dumps({'name': z.habitat.name}),
+                'zone': json.dumps({'name': z.name, 'variant': z.variant, 'refined_variant': z.refined_variant}),
+            }),
+            serialize_plants_queryset(q.plants),
+            filename,
+        )
+    except Exception as e:
+        logging.warning(e)
+    else:
+        if not storage.exists(filename):
+            raise FileNotFoundError(f"There was an error creating file: {filename}")
+    finally:
+        if e.completion >= 1:
+            generate_zip.delay(export_id)
+
+
+@shared_task
+def generate_zip(export_id):
+    export = Export.objects.get(pk=export_id)
+    zfilepath = storage.path(f"export_{export_id}/export.zip")
+
+    with zipfile.ZipFile(zfilepath, 'w', zipfile.ZIP_DEFLATED) as zf:
+        for q in export.questionnaires.all():
+            fpath = pathlib.Path(storage.path(f"export_{export_id}/{q.slug}.pdf"))
+            zf.write(fpath, fpath.name)
+
+    export.completion_date = timezone.now()
+    export.save()

backend/right_tree/api/views.py

@@ -1,15 +1,15 @@
-from django.http import HttpResponseBadRequest, HttpResponse, FileResponse
+from django.http import HttpResponseBadRequest, FileResponse
 from django.shortcuts import get_object_or_404
-from rest_framework import viewsets
+from django.utils import timezone
+from django.utils.text import slugify
+from rest_framework import viewsets, permissions
 from rest_framework.response import Response
-from wsgiref.util import FileWrapper
-
-from right_tree.api.models import Habitat, HabitatImage, Plant, EcologicalDistrictLayer, SoilOrder, Zone
-from right_tree.api.serializers import HabitatImageSerializer, HabitatSerializer, PlantSerializer, SoilOrderSerializer, EcologicalDistrictLayerSerializer, AddressSerializer, ZoneSerializer
 
+from .models import Habitat, HabitatImage, Plant, EcologicalDistrictLayer, SoilOrder, Zone, Questionnaire
+from .serializers import HabitatImageSerializer, HabitatSerializer, PlantSerializer, SoilOrderSerializer, EcologicalDistrictLayerSerializer, AddressSerializer, ZoneSerializer, QuestionnaireSerializer
 from .filters import *
 from .wms_utils import get_address_from_coordinates, search_address
-from .resource_generation_utils import create_plant_csv_file, get_plant_resource_filepath, create_planting_guide_pdf, PLANTING_GUIDE_PDF_FILENAME
+from .resource_generation_utils import generate_csv, get_filter_values, serialize_plants_queryset, create_planting_guide_pdf, PLANTING_GUIDE_PDF_FILENAME, CSV_FILENAME, storage
 
 
 class PlantViewSet(viewsets.ModelViewSet):
@@ -123,12 +123,16 @@ class CSVDownloadView(viewsets.ViewSet):
 
     def list(self, request, *args, **kwargs):
         filtered_plants = get_filtered_plants(request)
-        create_plant_csv_file(request, filtered_plants)
+        plant_data = serialize_plants_queryset(filtered_plants)
+        filename = f"plants_{slugify(timezone.now())}.csv"
 
-        csv_file = open(get_plant_resource_filepath(), 'rb')
+        generate_csv(plant_data, filename)
-        response = HttpResponse(csv_file, content_type='text/csv')
+
-        response['Content-Disposition'] = 'attachment; filename="plants.csv"'
+        return FileResponse(
-        return response
+            storage.open(filename, 'rb'),
+            filename='plants.csv',
+            content_type='text/csv',
+        )
 
 
 class PDFDownloadView(viewsets.ViewSet):
@@ -136,10 +140,22 @@ class PDFDownloadView(viewsets.ViewSet):
     """
 
     def list(self, request, *args, **kwargs):
+        filter_data = get_filter_values(request.query_params)
         filtered_plants = get_filtered_plants(request)
+        plant_data = serialize_plants_queryset(filtered_plants)
+        filename = f"planting_guide_{slugify(timezone.now())}.pdf"
 
-        create_planting_guide_pdf(request, filtered_plants)
+        create_planting_guide_pdf(filter_data, plant_data, filename)
-        pdf_file = open(get_plant_resource_filepath(PLANTING_GUIDE_PDF_FILENAME), 'rb')
 
-        response = HttpResponse(FileWrapper(pdf_file), content_type='application/pdf')
+        return FileResponse(
-        return response
+            storage.open(filename, 'rb'),
+            filename=PLANTING_GUIDE_PDF_FILENAME,
+            content_type='application/pdf',
+        )
+
+
+class QuestionnaireViewSet(viewsets.ModelViewSet):
+    serializer_class = QuestionnaireSerializer
+    queryset = Questionnaire.objects.all()
+    http_method_names = ("post",)
+    permission_classes = [permissions.AllowAny]

backend/right_tree/api/wms_utils.py

@@ -5,8 +5,11 @@ import requests
 from urllib.parse import urlencode
 from unicodedata import normalize
 
+from django.db.models import Q
 from django.contrib.gis.geos import Point, GEOSGeometry
-from django.conf import settings
+
+from .models import Address
+
 
 LINZ_API_KEY = os.getenv("LINZ_API_KEY")
 LINZ_WFS_ENDPOINT = f"https://data.linz.govt.nz/services;key={LINZ_API_KEY}/wfs"
@@ -24,10 +27,14 @@ class WFSError(Exception):
 
 def get_point_from_coordinates(coordinates):
     """Given a coordinates json string, returns the coordinates as a Point object"""
+    if isinstance(coordinates, Point):
+        return coordinates
+    elif isinstance(coordinates, str):
         coordinates_json = json.loads(coordinates)
-    pnt = Point(coordinates_json["lng"],
+    elif isinstance(coordinates, dict):
-                coordinates_json["lat"], srid=4326)
+        coordinates_json = coordinates
-    return pnt
+
+    return Point(coordinates_json["lng"], coordinates_json["lat"], srid=4326)
 
 
 def wfs_getfeature(endpoint, **kwargs):
@@ -94,7 +101,6 @@ def get_address_from_coordinates(coordinates):
 
 
 def search_address(address):
-
     # normalize accent characters etc.
     address = normalize(
         "NFKD",
@@ -104,31 +110,22 @@ def search_address(address):
     nums = search_num.findall(address)
     strings = search_str.findall(address)
 
-    num_terms = [f"address_number = {n}" for n in nums]
+    num_filter = Q()
-    string_terms = []
+    str_filter = Q()
+
+    for n in nums:
+        num_filter |= Q(address_number=n)
 
     for s in strings:
-        string_terms += [
+        str_filter |= (
-            f"full_road_name_ascii ILIKE '{s}%'",
+            Q(full_road_name_ascii__istartswith=s) |
-            f"town_city_ascii ILIKE '{s}%'",
+            Q(town_city_ascii__istartswith=s) |
-            f"suburb_locality_ascii ILIKE '{s}%'",
+            Q(suburb_locality_ascii__istartswith=s)
-        ]
-
-    num_expr = " OR ".join(f"({term})" for term in num_terms)
-    string_expr = " OR ".join(f"({term})" for term in string_terms)
-
-    cql_filter = f"({num_expr}) AND ({string_expr})"
-
-    resp = linz_wfs_getfeature(
-        typeNames=PROPERTY_INFO_LAYER,
-        cql_filter=cql_filter,
-        count=10,
         )
-    features = resp.get("features", [])
 
     return [
         {
-            'coordinates': feature['geometry']['coordinates'],
+            'coordinates': (addr.wkb_geometry.x, addr.wkb_geometry.y),
-            'address': feature['properties']['full_address'],
+            'address': addr.full_address,
-        } for feature in features
+        } for addr in Address.objects.filter(num_filter & str_filter).distinct()[:10]
     ]

backend/right_tree/settings.py

@@ -81,15 +81,21 @@ WSGI_APPLICATION = 'right_tree.wsgi.application'
 
 # Database
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
-
+db = {
-DATABASES = {
-    'default': {
     'ENGINE': 'django.contrib.gis.db.backends.postgis',
     'NAME': os.getenv("DATABASE_NAME", "righttree"),
     'USER': os.getenv("DATABASE_USER", "righttree"),
     'PASSWORD': os.getenv("DATABASE_PASSWORD", "righttree"),
     'HOST': os.getenv("DATABASE_HOST", "postgres"),
     'PORT': int(os.getenv("DATABASE_PORT", 5432)),
+}
+DATABASES = {
+    'default': db,
+    'linz': {
+        **db,
+        'OPTIONS': {
+            'options': '-c search_path=linz'
+        },
     }
 }
 
@@ -118,7 +124,7 @@ AUTH_PASSWORD_VALIDATORS = [
 
 LANGUAGE_CODE = 'en-us'
 
-TIME_ZONE = 'UTC'
+TIME_ZONE = 'Pacific/Auckland'
 
 USE_I18N = True
 
@@ -131,8 +137,9 @@ USE_TZ = True
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 
 STATIC_URL = '/staticfiles/'
-PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_DIR = Path(os.path.abspath(__file__)).parent
-STATIC_ROOT = os.path.join(PROJECT_DIR, 'staticfiles')
+STATIC_ROOT = PROJECT_DIR / 'staticfiles'
+MEDIA_ROOT = PROJECT_DIR / 'media'
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
@@ -142,3 +149,7 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 CORS_ALLOW_HEADERS = [
     'access-control-allow-origin'
 ]
+
+# Celery configuration
+CELERY_TIMEZONE = TIME_ZONE
+CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://redis:6379/0")

backend/right_tree/urls.py

@@ -28,6 +28,7 @@ router.register(r'region', views.AuckCHCHRegionInformation, basename='region')
 router.register(r'habitats', views.HabitatViewSet, basename='habitats')
 router.register(r'zones', views.ZoneViewSet, basename='zones')
 router.register(r'habitatimage', views.HabitatImageViewSet, basename='habitatimage')
+router.register(r'questionnaire', views.QuestionnaireViewSet, basename='questionnaire')
 
 router.register(r'download/csv', views.CSVDownloadView, basename='downloadcsv')
 router.register(r'download/pdf', views.PDFDownloadView, basename='downloadpdf')

create_database.sql

@@ -7,5 +7,9 @@ CREATE EXTENSION IF NOT EXISTS postgis;
 GRANT ALL ON geometry_columns TO PUBLIC;
 GRANT ALL ON spatial_ref_sys TO PUBLIC;
 
+CREATE SCHEMA IF NOT EXISTS linz;
+ALTER SCHEMA linz OWNER TO righttree;
+GRANT ALL PRIVILEGES ON SCHEMA linz TO righttree;
+
 ALTER DATABASE righttree OWNER TO righttree;
 GRANT ALL PRIVILEGES ON DATABASE righttree TO righttree;

create_indices.sql

@@ -0,0 +1,4 @@
+CREATE INDEX IF NOT EXISTS full_road_name_lower_idx ON linz.nz_street_address (lower(full_road_name_ascii) text_pattern_ops);
+CREATE INDEX IF NOT EXISTS suburb_locality_lower_idx ON linz.nz_street_address (lower(suburb_locality_ascii) text_pattern_ops);
+CREATE INDEX IF NOT EXISTS town_city_lower_idx ON linz.nz_street_address (lower(town_city_ascii) text_pattern_ops);
+CREATE INDEX IF NOT EXISTS wkb_geometry_idx ON linz.nz_street_address USING GIST(wkb_geometry);

default.env

@@ -1,21 +1,9 @@
-# POSTGRES CONFIG
+LINZ_API_KEY=myapikey
-# ---------------------------------
+DATABASE_NAME=righttree
-POSTGRES_DB=postgres
+DATABASE_USER=righttree
-POSTGRES_USER=postgres
+DATABASE_PASSWORD=righttree
-POSTGRES_PASSWORD=postgres
+DATABASE_HOST=postgres
-
+CELERY_BROKER_URL=redis://redis:6379/0
-
+BASE_URL=localhost:8000
-# RIGHTTREE DATABASE CONFIG (for production)
+DJANGO_SECRET_KEY=changeme
-# ---------------------------------
+DJANGO_DEBUG_MODE=True
-# RIGHTTREE_DB=righttree
-# RIGHTTREE_DB_USER=righttree_admin
-# RIGHTTREE_DB_PASSWORD=[YOUR_PASSWORD]
-
-
-# DJANGO BACKEND CONFIG (uncomment out the config below for production)
-# ---------------------------------
-LINZ_API_KEY=[YOUR_API_KEY]
-# FRONTEND_BASE_URL=righttree.maps.net.nz
-# DJANGO_DEBUG_MODE=False
-# DJANGO_SECRET_KEY=[YOUR_SECRETKEY]
-# BASE_URL=righttree.maps.net.nz

docker-compose.production.yaml

@@ -4,26 +4,31 @@ volumes:
   righttree-postgres-data:
     name: righttree-postgres-data
 
+x-django: &django
+  image: right-tree
+  depends_on:
+    postgres:
+      condition: service_healthy
+  env_file: .env
+  user: "$UID:$GID"
+  restart: always
+
 services:
   backend:
-    restart: always
+    <<: *django
-    build:
-      context: backend
-      dockerfile: Dockerfile
     container_name: backend
-    depends_on:
+    expose:
-      - postgres
+      - "8000"
-    env_file: .env
+    command:
-    ports:
+      - gunicorn
-      - "8000:8000"
+      - --bind=0.0.0.0:8000
-    command: bash -c "gunicorn --bind 0.0.0.0:8000 right_tree.wsgi"
+      - right_tree.wsgi
 
   nginx:
     container_name: nginx
     restart: always
     image: nginx
     depends_on:
-      - postgres
       - backend
     volumes:
       - ./nginx.production.conf:/etc/nginx/nginx.conf
@@ -35,7 +40,7 @@ services:
       - "443:443"
 
   postgres:
-    image: postgis/postgis:13-3.0
+    image: postgis/postgis:13-3.1
     restart: always
     container_name: postgres
     volumes:
@@ -47,4 +52,38 @@ services:
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    healthcheck:
+      test: ["CMD", "pg_isready", "--dbname", "righttree", "--username", "righttree"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
+  redis:
+    image: redis:7.0.8
+    restart: always
+    container_name: redis
+    expose:
+      - "6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  celery:
+    <<: *django
+    container_name: celery
+    command:
+      - celery
+      - -A
+      - right_tree.api
+      - worker
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    deploy:
+      resources:
+        limits:
+          cpus: '1'

docker-compose.yaml

@@ -4,45 +4,23 @@ volumes:
   righttree-postgres-data:
     name: righttree-postgres-data
 
-services:
+x-django: &django
-
-  backend_migrate:
-    restart: on-failure
   image: right-tree
-    container_name: backend_migrate
   depends_on:
-      - postgres
+    postgres:
+      condition: service_healthy
   volumes:
     - ./backend:/app
+  env_file: .env
   user: "$UID:$GID"
-    environment:
-      DATABASE_NAME: righttree
-      DATABASE_USER: righttree
-      DATABASE_PASSWORD: righttree
-      DATABASE_HOST: postgres
-    command:
-      - bash
-      - -c
-      - python manage.py makemigrations --noinput && python manage.py migrate --noinput
-
-  backend:
   restart: unless-stopped
-    image: right-tree
+
+services:
+  backend:
+    <<: *django
     container_name: backend
-    depends_on:
-      - postgres
-      - backend_migrate
-    volumes:
-      - ./backend:/app
-    user: "$UID:$GID"
     expose:
       - "8000"
-    environment:
-      LINZ_API_KEY: myapikeyhere
-      DATABASE_NAME: righttree
-      DATABASE_USER: righttree
-      DATABASE_PASSWORD: righttree
-      DATABASE_HOST: postgres
     command:
       - gunicorn
       - --reload
@@ -75,9 +53,14 @@ services:
       - 5432:5432
     environment:
       POSTGRES_PASSWORD: postgres
+    healthcheck:
+      test: ["CMD", "pg_isready", "--dbname", "righttree", "--username", "righttree"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
   nginx:
-    image: nginx:1.23.3
+    image: nginx
     restart: unless-stopped
     container_name: nginx
     depends_on:
@@ -88,3 +71,31 @@ services:
       - ./backend/right_tree/staticfiles:/etc/nginx/html/staticfiles:ro
     ports:
       - "9000:80"
+
+  redis:
+    image: redis:7.0.8
+    restart: unless-stopped
+    container_name: redis
+    expose:
+      - "6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  celery:
+    <<: *django
+    container_name: celery
+    command:
+      - celery
+      - -A
+      - right_tree.api
+      - worker
+    depends_on:
+      redis:
+        condition: service_healthy
+    deploy:
+      resources:
+        limits:
+          cpus: '1'