diff --git a/Makefile b/Makefile index 89cf447..f8da221 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,16 @@ ingest: docker-compose exec backend python manage.py loaddata \ /app/right_tree/api/data/fixtures/plants.json +ingest_linz: + docker-compose up -d postgres + docker-compose exec -T postgres pg_restore -U righttree -d righttree -n linz -Fc < linz.dump + docker-compose exec -T postgres psql -U righttree -d righttree -f - < create_indices.sql + +migrate: + docker-compose up -d backend postgres + docker-compose exec backend python manage.py makemigrations --noinput + docker-compose exec backend python manage.py migrate --noinput + createsuperuser: docker-compose up -d backend docker-compose exec backend python manage.py createsuperuser diff --git a/backend/.gitignore b/backend/.gitignore index 5bcb7f2..3a5e129 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,19 +1,9 @@ *.pyc -*.sqlite3 __pycache__ -right_tree/api/data/resources/*.cpg -right_tree/api/data/resources/*.dbf -right_tree/api/data/resources/*.pdf -right_tree/api/data/resources/*.prj -right_tree/api/data/resources/*.sbn -right_tree/api/data/resources/*.sbx -right_tree/api/data/resources/*.shp -right_tree/api/data/resources/*.shx -right_tree/api/data/resources/*.txt -right_tree/api/data/resources/*.xml -right_tree/api/data/resources/*.zip +right_tree/api/data/resources +!right_tree/api/data/resources/plant_data.xlsx right_tree/api/data/fixtures/plants.json +right_tree/media right_tree/staticfiles -right_tree/api/data/generated_resources/* diff --git a/backend/Dockerfile b/backend/Dockerfile index feacc5f..d46ec77 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.11-slim-bullseye WORKDIR /app RUN apt update \ - && apt install -y --no-install-recommends gdal-bin \ + && apt install -y --no-install-recommends gdal-bin wkhtmltopdf \ && rm -rf /var/lib/apt/lists/* \ && apt clean @@ -12,3 +12,5 @@ COPY ./requirements.txt /app/requirements.txt RUN pip install -U --no-cache-dir -r requirements.txt COPY . /app + +ENV DJANGO_SETTINGS_MODULE="right_tree.settings" diff --git a/backend/requirements.txt b/backend/requirements.txt index 545b176..e23b063 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -8,3 +8,4 @@ gunicorn==20.1.0 pandas==1.5.3 pdfkit==1.0.0 PyPDF2==1.28.6 +celery[redis]==5.2.7 diff --git a/backend/right_tree/api/admin.py b/backend/right_tree/api/admin.py index f72b920..248b038 100644 --- a/backend/right_tree/api/admin.py +++ b/backend/right_tree/api/admin.py @@ -1,5 +1,10 @@ -from django.contrib import admin -import right_tree.api.models as models +from django.contrib import admin, messages +from django.http import HttpResponseRedirect, FileResponse +from django.utils.text import slugify +from django.utils import timezone + +from right_tree.api import models +from right_tree.api.resource_generation_utils import storage class ZoneAdmin(admin.ModelAdmin): @@ -7,6 +12,71 @@ class ZoneAdmin(admin.ModelAdmin): search_fields = ['name', 'habitat__name', 'variant', 'refined_variant', 'id'] +class QuestionnaireAdmin(admin.ModelAdmin): + list_display = ['address_display', 'location_display', 'soil_variant', 'ecological_district', 'habitat', 'zone'] + actions = ['export'] + + @admin.display(description="Address") + def address_display(self, obj): + return obj.address.full_address + + @admin.display(description="Location") + def location_display(self, obj): + return f"({obj.location.x}, {obj.location.y})" + + @admin.action(description="Export planting guides for selected questionnaires") + def export(self, request, queryset): + export = models.Export.objects.create(creation_date=timezone.now()) + export.questionnaires.set(queryset) + export.export() + + return HttpResponseRedirect('/admin/api/export') + + +class ExportAdmin(admin.ModelAdmin): + list_display = ['creation_date', 'completion_date', 'completion_display'] + actions = ['download'] + + @admin.display(description="Completion") + def completion_display(self, obj): + return f"{obj.completion:.1%}" + + @admin.action(description="Download completed exports") + def download(self, request, queryset): + if queryset.count() > 1: + self.message_user( + request, + 'Cannot download more than one export at a time.', + messages.ERROR, + ) + return + + export = queryset.first() + + if not export.complete: + self.message_user( + request, + 'Cannot download. Export is incomplete.', + messages.ERROR, + ) + return + + filename = f"export_{export.pk}/export.zip" + filepath = storage.path(filename) + + if storage.exists(filename): + return FileResponse( + open(filepath, 'rb'), + filename=f"export_{slugify(export.creation_date)}.zip", + ) + + self.message_user( + request, + 'Cannot download. Export is corrupt.', + messages.ERROR, + ) + + admin.site.register(models.Plant) admin.site.register(models.SoilOrder) admin.site.register(models.SoilLayer) @@ -18,3 +88,5 @@ admin.site.register(models.HabitatImage) admin.site.register(models.Habitat) admin.site.register(models.Zone, ZoneAdmin) admin.site.register(models.ChristchurchRegion) +admin.site.register(models.Questionnaire, QuestionnaireAdmin) +admin.site.register(models.Export, ExportAdmin) diff --git a/backend/right_tree/api/apps.py b/backend/right_tree/api/apps.py index e86d3a3..10ab459 100644 --- a/backend/right_tree/api/apps.py +++ b/backend/right_tree/api/apps.py @@ -3,3 +3,6 @@ from django.apps import AppConfig class ApiConfig(AppConfig): name = 'right_tree.api' + + def ready(self): + import right_tree.api.signals diff --git a/backend/right_tree/api/celery.py b/backend/right_tree/api/celery.py new file mode 100644 index 0000000..8007ec8 --- /dev/null +++ b/backend/right_tree/api/celery.py @@ -0,0 +1,5 @@ +from celery import Celery + +app = Celery('righttree') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() diff --git a/backend/right_tree/api/migrations/0014_address_export_questionnaire.py b/backend/right_tree/api/migrations/0014_address_export_questionnaire.py new file mode 100644 index 0000000..06204b1 --- /dev/null +++ b/backend/right_tree/api/migrations/0014_address_export_questionnaire.py @@ -0,0 +1,49 @@ +# Generated by Django 3.2.17 on 2023-02-21 01:38 + +import django.contrib.gis.db.models.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0013_zone_tooltiptext'), + ] + + operations = [ + migrations.CreateModel( + name='Address', + fields=[ + ('ogc_fid', models.AutoField(primary_key=True, serialize=False)), + ('address_number', models.IntegerField()), + ('suburb_locality_ascii', models.CharField(max_length=255)), + ('town_city_ascii', models.CharField(max_length=255)), + ('full_road_name_ascii', models.CharField(max_length=255)), + ('wkb_geometry', django.contrib.gis.db.models.fields.PointField(srid=4326)), + ('full_address', models.CharField(max_length=500)), + ], + options={ + 'db_table': 'linz"."nz_street_address', + 'managed': False, + }, + ), + migrations.CreateModel( + name='Questionnaire', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('location', django.contrib.gis.db.models.fields.PointField(srid=4326)), + ('soil_variant', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.soilvariant')), + ('zone', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.zone')), + ], + ), + migrations.CreateModel( + name='Export', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creation_date', models.DateTimeField()), + ('completion_date', models.DateTimeField(null=True)), + ('questionnaires', models.ManyToManyField(to='api.Questionnaire')), + ], + ), + ] diff --git a/backend/right_tree/api/models.py b/backend/right_tree/api/models.py index 14f5c8a..8fbc732 100644 --- a/backend/right_tree/api/models.py +++ b/backend/right_tree/api/models.py @@ -1,4 +1,12 @@ +from functools import cached_property +from pathlib import Path + +from django.db.models.functions import Lower from django.contrib.gis.db import models +from django.contrib.gis.db.models.functions import Distance +from django.contrib.postgres.indexes import OpClass +from django.utils.text import slugify + class SoilOrder(models.Model): @@ -162,3 +170,111 @@ class Plant(models.Model): class Meta: ordering = ['stage', 'name', 'commonname', 'synonym'] + + +class Address(models.Model): + ogc_fid = models.AutoField(primary_key=True) + + # fields for queries + address_number = models.IntegerField() # ignore the 'A' in '32A' for now + suburb_locality_ascii = models.CharField(max_length=255) + town_city_ascii = models.CharField(max_length=255) + full_road_name_ascii = models.CharField(max_length=255) + wkb_geometry = models.PointField() + + # use only for displaying to user + full_address = models.CharField(max_length=500) + + class Meta: + db_table = 'linz\".\"nz_street_address' + indexes = [ + models.Index(OpClass(Lower('full_road_name_ascii'), name='text_pattern_ops'), name='full_road_name_lower_idx'), + models.Index(OpClass(Lower('suburb_locality_ascii'), name='text_pattern_ops'), name='suburb_locality_lower_idx'), + models.Index(OpClass(Lower('town_city_ascii'), name='text_pattern_ops'), name='town_city_lower_idx'), + models.Index(fields=['address_number'], name='address_number_idx'), + ] + + # effectively read-only + # https://docs.djangoproject.com/en/3.2/ref/models/options/#managed + managed = False + + +class Questionnaire(models.Model): + location = models.PointField() + soil_variant = models.ForeignKey(SoilVariant, on_delete=models.CASCADE) + zone = models.ForeignKey(Zone, on_delete=models.CASCADE) + + @property + def habitat(self): + return self.zone.habitat + + @property + def ecological_district_layer(self): + return EcologicalDistrictLayer.objects.filter(geom__intersects=self.location).first() + + @property + def ecological_district(self): + return self.ecological_district_layer.ecologic_2 + + @cached_property + def address(self): + return \ + Address.objects.filter( + wkb_geometry__dwithin=(self.location, 1e-3), # order of ~100 metres in degrees for NZ + ).annotate( + distance=Distance('wkb_geometry', self.location), + ).order_by( + 'distance', + ).first() + + @property + def plants(self): + return self.zone.plant_set.all() & self.soil_variant.plant_set.all() + + @cached_property + def slug(self): + return "_".join( + map(slugify, [ + self.address.full_address if self.address else str(self.location), + self.zone.name, + self.zone.variant, + self.zone.refined_variant, + self.soil_variant.name, + str(self.zone.pk), # need this for uniqueness + str(self.soil_variant.pk), # # need this for uniqueness + ]) + ) + + +class Export(models.Model): + creation_date = models.DateTimeField() + completion_date = models.DateTimeField(null=True) + questionnaires = models.ManyToManyField(Questionnaire) + + @property + def complete(self): + return self.completion_date is not None + + @property + def completion(self): + if self.complete: + return 1.0 + + from .resource_generation_utils import storage + + _, files = storage.listdir(f"export_{self.pk}") + + # halved as there are two files (csv, pdf) per questionnaire + return 0.5 * len(files) / self.questionnaires.count() + + def export(self): + from .tasks import generate_pdf + from .resource_generation_utils import storage + + parent = Path(storage.path(f"export_{self.pk}")) + + if not parent.exists(): + parent.mkdir() + + for q in self.questionnaires.all(): + generate_pdf.delay(q.pk, self.pk) diff --git a/backend/right_tree/api/resource_generation_utils.py b/backend/right_tree/api/resource_generation_utils.py index a82d44d..facc39c 100644 --- a/backend/right_tree/api/resource_generation_utils.py +++ b/backend/right_tree/api/resource_generation_utils.py @@ -1,8 +1,7 @@ import csv -from os import write -from pathlib import Path +from io import StringIO +from os.path import splitext -import right_tree.api.data from .filters import * from .wms_utils import get_address_from_coordinates, get_point_from_coordinates @@ -10,6 +9,12 @@ import pdfkit import pandas as pd from PyPDF2 import PdfFileMerger +from django.core.files.storage import get_storage_class + + +Storage = get_storage_class() +storage = Storage() + CSV_FILENAME = 'plants.csv' PLANTING_GUIDE_PDF_FILENAME = 'planting_guide.pdf' @@ -17,17 +22,11 @@ HEADER_FIELDS = ['Names', 'Growth Form / Max Height (m) / Spacing (m) / Forest P 'Tolerances (Water / Drought / Frost / Salinity)', 'Ecosystem Services', 'Carbon Sequestration Rate', 'Planting Stage'] -def get_plant_resource_filepath(filename=CSV_FILENAME, resource_dir='generated_resources'): - """ Retrives the filepath for the plant csv file. - """ - return Path(right_tree.api.data.__file__).resolve().parent / resource_dir / filename - - -def get_location_filters(request): +def get_location_filters(params): """ Retrives the selected location data from the request. """ filter_rows = [['LOCATION FILTERS:', ' ']] - coordinates = request.query_params.get('coordinates') + coordinates = params.get('coordinates') if coordinates is not None: eco_district_layer = ecological_district_coordinate_filter( @@ -45,12 +44,12 @@ def get_location_filters(request): return filter_rows -def get_soil_filters(request): - """ Retrives the selected soil type data from the request. +def get_soil_filters(params): + """ Retrives the selected soil type data from the request params. """ filter_rows = [['SOIL FILTERS:', ' ']] - soil_variant = request.query_params.get('soilVariant') - coordinates = request.query_params.get('coordinates') + soil_variant = params.get('soilVariant') + coordinates = params.get('coordinates') if soil_variant is not None and coordinates is not None: soil_order_obj = soil_order_coordinate_filter(coordinates).first() @@ -63,13 +62,13 @@ def get_soil_filters(request): return filter_rows -def get_site_filters(request): - """ Retrives the selected site data from the request. +def get_site_filters(params): + """ Retrives the selected site data from the request params """ filter_rows = [['SITE FILTERS:', ' ']] - habitat = request.query_params.get('habitat') - zone = request.query_params.get('zone') + habitat = params.get('habitat') + zone = params.get('zone') if zone is not None and habitat is not None: habitat_json = json.loads(habitat) zone_json = json.loads(zone) @@ -83,53 +82,85 @@ def get_site_filters(request): return filter_rows -def get_additional_region_info(request): +def get_additional_region_info(params): """ If the location coordinates fall within the CHCH or Auckland regions then return a description of where to find more information. """ - coordinates = request.query_params.get('coordinates') + coordinates = params.get('coordinates') if coordinates is not None: if is_in_christchurch(coordinates): return [["Your location falls within the ecosystem type covered by the Christchurch Council ecosystem maps - further information can be obtained from Ōtautahi/Christchurch ecosystems map link to: https://ccc.govt.nz/environment/land/ecosystem-map", " "], [' ', ' ']] elif is_in_auckland(coordinates): return [["Your location falls within the ecosystem type covered by the Auckland Council Tiaki Tāmaki Makaurau Conservation map - further information can be obtained from tiaki Tāmaki Makaurau conservation Auckland - link to https://www.tiakitamakimakaurau.nz/conservation-map/", " "], [' ', ' ']] - + return [] -def get_filter_values(request): - """ Retrives all selected values/filters from the request. +def get_filter_values(params): + """ Retrives all selected values/filters from the request parameters """ filter_rows = [] # Add all the location filters - filter_rows += get_location_filters(request) + filter_rows += get_location_filters(params) filter_rows.append([' ', ' ']) # Add the soil filters - filter_rows += get_soil_filters(request) + filter_rows += get_soil_filters(params) filter_rows.append([' ', ' ']) # Add the project site filters - filter_rows += get_site_filters(request) + filter_rows += get_site_filters(params) filter_rows.append([' ', ' ']) - - filter_rows += get_additional_region_info(request) + + filter_rows += get_additional_region_info(params) return filter_rows -def write_csv_filter_info(request, writer): - """ Retrieves and writes filter information to a CSV file given a writer. - """ - for filter_row in get_filter_values(request): - writer.writerow(filter_row) +def generate_csv(data: list[list[str]], output_filename: str) -> str: + with storage.open(output_filename, 'w') as f: + csv.writer(f).writerows(data) + + return storage.path(output_filename) -def write_csv_plant_info(plant_data, writer): - """ Writes plant list information to a CSV file given a writer. +def generate_pdf(data: list[list[str]], output_filename: str): + """ Generates a pdf from a csv given data and a csv generation method. + Requires an html file to be generated as an intermediate step. """ - writer.writerow(HEADER_FIELDS) - for plant in plant_data: - plant_data_row = [ + + name, _ = splitext(output_filename) + csv_filepath = generate_csv(data, f"{name}.csv") + pdf_filepath = storage.path(output_filename) + + with StringIO() as html_buf: + # Convert csv to html + pd.read_csv(csv_filepath).to_html(html_buf) # reading from buffer causes segfault :/ + html_buf.seek(0) + + # Convert html to pdf + pdfkit.from_file(html_buf, pdf_filepath) + + return pdf_filepath + + +def merge_pdfs(pdfs: list[str], output_filename): + """Merge a list of PDF filenames""" + + output_filepath = storage.path(output_filename) + merger = PdfFileMerger() + + for pdf in pdfs: + merger.append(pdf) + + merger.write(output_filepath) + merger.close() + + return output_filepath + + +def serialize_plants_queryset(plants_queryset): + return [HEADER_FIELDS] + [ + [ plant.display_name, plant.display_growth_form, plant.moisture_preferences, @@ -137,69 +168,20 @@ def write_csv_plant_info(plant_data, writer): plant.ecosystem_services, plant.carbon_sequestration, plant.stage - ] - writer.writerow(plant_data_row) + ] for plant in plants_queryset + ] -def create_plant_csv_file(request, plant_data): - """ Constructs a csv file that contains selected filter values and the resulting plant list. - """ - with open(get_plant_resource_filepath(), 'w', encoding='UTF8') as f: - writer = csv.writer(f) - - # Write filter/selected values - write_csv_filter_info(request, writer) - - # Write the plant data - write_csv_plant_info(plant_data, writer) - - -def generate_pdf(data_name, data, csv_generation_function): - """ Generates a pdf from a csv given data and a csv generation method. - Requires an html file to be generated as an intermediate step. - """ - - # Define filepaths - some are required as intermediate files - csv_filepath = get_plant_resource_filepath(f'{data_name}.csv') - html_filepath = get_plant_resource_filepath(f'{data_name}.html') - pdf_filepath = get_plant_resource_filepath(f'{data_name}.pdf') - - # Create an initial csv file with the data - with open(csv_filepath, 'w', encoding='UTF8') as f: - writer = csv.writer(f) - csv_generation_function(data, writer) - - # Convert the csv to and html file then to a pdf - CSV = pd.read_csv(csv_filepath) - CSV.to_html(html_filepath) - pdfkit.from_file(str(html_filepath), str(pdf_filepath)) - - -def create_planting_guide_pdf(request, plant_data): +def create_planting_guide_pdf(filter_data, plant_data, output_filename): """ Creates a planting guide pdf document with a pre-generated planting guide with filter and plant list tabular informtation appended. """ - - # Define the names of the resource files as used in multiple places - filter_file_prefix = "filters" - plant_list_file_prefix = "plant_list" - - # Generate the filters and plant list pdf files - generate_pdf(filter_file_prefix, request, write_csv_filter_info) # TODO: space values appear as NaN... this should be fixed - generate_pdf(plant_list_file_prefix, plant_data, write_csv_plant_info) - - # Create a list of pdfs that need to be merged - planting_guide_pdf_filepath = get_plant_resource_filepath(PLANTING_GUIDE_PDF_FILENAME, 'resources') - filter_pdf_filepath = get_plant_resource_filepath(f'{filter_file_prefix}.pdf') - plant_list_pdf_filepath = get_plant_resource_filepath(f'{plant_list_file_prefix}.pdf') - pdfs = [planting_guide_pdf_filepath, filter_pdf_filepath, plant_list_pdf_filepath] - - # Merge pdfs - merger = PdfFileMerger() - for pdf in pdfs: - merger.append(str(pdf)) - - # Create the final output pdf with all merged documents - output_pdf_filepath = get_plant_resource_filepath(PLANTING_GUIDE_PDF_FILENAME) - merger.write(str(output_pdf_filepath)) - merger.close() + # TODO: space values appear as NaN... this should be fixed + return merge_pdfs( + [ + storage.path(PLANTING_GUIDE_PDF_FILENAME), + generate_pdf(filter_data, f"{output_filename}.filters"), + generate_pdf(plant_data, f"{output_filename}.plants"), + ], + output_filename, + ) diff --git a/backend/right_tree/api/serializers.py b/backend/right_tree/api/serializers.py index 52ddd36..5520093 100644 --- a/backend/right_tree/api/serializers.py +++ b/backend/right_tree/api/serializers.py @@ -1,4 +1,4 @@ -from rest_framework import serializers +from rest_framework import serializers, exceptions from right_tree.api.models import * @@ -95,3 +95,18 @@ class PlantSerializer(serializers.HyperlinkedModelSerializer): class AddressSerializer(serializers.Serializer): full_address = serializers.CharField(max_length=500) + + +class QuestionnaireSerializer(serializers.ModelSerializer): + + soil_variant = serializers.CharField(max_length=10) + + class Meta: + model = Questionnaire + fields = '__all__' + + def validate_soil_variant(self, value): + try: + return SoilVariant.objects.get(name__startswith=value) + except SoilVariant.DoesNotExist as e: + raise exceptions.ValidationError(e) diff --git a/backend/right_tree/api/signals.py b/backend/right_tree/api/signals.py new file mode 100644 index 0000000..59feee3 --- /dev/null +++ b/backend/right_tree/api/signals.py @@ -0,0 +1,12 @@ +from shutil import rmtree + +from django.db.models.signals import post_delete +from django.dispatch import receiver + +from .models import Export +from .resource_generation_utils import storage + + +@receiver(post_delete, sender=Export) +def delete_export(sender, instance, *args, **kwargs): + rmtree(storage.path(f"export_{instance.pk}")) diff --git a/backend/right_tree/api/tasks.py b/backend/right_tree/api/tasks.py new file mode 100644 index 0000000..dd334ae --- /dev/null +++ b/backend/right_tree/api/tasks.py @@ -0,0 +1,53 @@ +import json +import logging +import pathlib +import zipfile + +from celery import shared_task + +from django.utils import timezone + +from .models import Questionnaire, Export +from .resource_generation_utils import create_planting_guide_pdf, get_filter_values, serialize_plants_queryset, storage + + +@shared_task +def generate_pdf(questionnaire_id, export_id): + q = Questionnaire.objects.get(pk=questionnaire_id) + e = Export.objects.get(pk=export_id) + z = q.zone + filename = f"export_{e.pk}/{q.slug}.pdf" + + try: + create_planting_guide_pdf( + get_filter_values({ # awful hack to reuse some logic we already have + 'coordinates': {'lat': q.location.y, 'lng': q.location.x}, + 'soilVariant': q.soil_variant.name[0], + 'habitat': json.dumps({'name': z.habitat.name}), + 'zone': json.dumps({'name': z.name, 'variant': z.variant, 'refined_variant': z.refined_variant}), + }), + serialize_plants_queryset(q.plants), + filename, + ) + except Exception as e: + logging.warning(e) + else: + if not storage.exists(filename): + raise FileNotFoundError(f"There was an error creating file: {filename}") + finally: + if e.completion >= 1: + generate_zip.delay(export_id) + + +@shared_task +def generate_zip(export_id): + export = Export.objects.get(pk=export_id) + zfilepath = storage.path(f"export_{export_id}/export.zip") + + with zipfile.ZipFile(zfilepath, 'w', zipfile.ZIP_DEFLATED) as zf: + for q in export.questionnaires.all(): + fpath = pathlib.Path(storage.path(f"export_{export_id}/{q.slug}.pdf")) + zf.write(fpath, fpath.name) + + export.completion_date = timezone.now() + export.save() diff --git a/backend/right_tree/api/views.py b/backend/right_tree/api/views.py index c457fed..1d772d4 100644 --- a/backend/right_tree/api/views.py +++ b/backend/right_tree/api/views.py @@ -1,15 +1,15 @@ -from django.http import HttpResponseBadRequest, HttpResponse, FileResponse +from django.http import HttpResponseBadRequest, FileResponse from django.shortcuts import get_object_or_404 -from rest_framework import viewsets +from django.utils import timezone +from django.utils.text import slugify +from rest_framework import viewsets, permissions from rest_framework.response import Response -from wsgiref.util import FileWrapper - -from right_tree.api.models import Habitat, HabitatImage, Plant, EcologicalDistrictLayer, SoilOrder, Zone -from right_tree.api.serializers import HabitatImageSerializer, HabitatSerializer, PlantSerializer, SoilOrderSerializer, EcologicalDistrictLayerSerializer, AddressSerializer, ZoneSerializer +from .models import Habitat, HabitatImage, Plant, EcologicalDistrictLayer, SoilOrder, Zone, Questionnaire +from .serializers import HabitatImageSerializer, HabitatSerializer, PlantSerializer, SoilOrderSerializer, EcologicalDistrictLayerSerializer, AddressSerializer, ZoneSerializer, QuestionnaireSerializer from .filters import * from .wms_utils import get_address_from_coordinates, search_address -from .resource_generation_utils import create_plant_csv_file, get_plant_resource_filepath, create_planting_guide_pdf, PLANTING_GUIDE_PDF_FILENAME +from .resource_generation_utils import generate_csv, get_filter_values, serialize_plants_queryset, create_planting_guide_pdf, PLANTING_GUIDE_PDF_FILENAME, CSV_FILENAME, storage class PlantViewSet(viewsets.ModelViewSet): @@ -123,12 +123,16 @@ class CSVDownloadView(viewsets.ViewSet): def list(self, request, *args, **kwargs): filtered_plants = get_filtered_plants(request) - create_plant_csv_file(request, filtered_plants) + plant_data = serialize_plants_queryset(filtered_plants) + filename = f"plants_{slugify(timezone.now())}.csv" - csv_file = open(get_plant_resource_filepath(), 'rb') - response = HttpResponse(csv_file, content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="plants.csv"' - return response + generate_csv(plant_data, filename) + + return FileResponse( + storage.open(filename, 'rb'), + filename='plants.csv', + content_type='text/csv', + ) class PDFDownloadView(viewsets.ViewSet): @@ -136,10 +140,22 @@ class PDFDownloadView(viewsets.ViewSet): """ def list(self, request, *args, **kwargs): + filter_data = get_filter_values(request.query_params) filtered_plants = get_filtered_plants(request) + plant_data = serialize_plants_queryset(filtered_plants) + filename = f"planting_guide_{slugify(timezone.now())}.pdf" - create_planting_guide_pdf(request, filtered_plants) - pdf_file = open(get_plant_resource_filepath(PLANTING_GUIDE_PDF_FILENAME), 'rb') + create_planting_guide_pdf(filter_data, plant_data, filename) - response = HttpResponse(FileWrapper(pdf_file), content_type='application/pdf') - return response + return FileResponse( + storage.open(filename, 'rb'), + filename=PLANTING_GUIDE_PDF_FILENAME, + content_type='application/pdf', + ) + + +class QuestionnaireViewSet(viewsets.ModelViewSet): + serializer_class = QuestionnaireSerializer + queryset = Questionnaire.objects.all() + http_method_names = ("post",) + permission_classes = [permissions.AllowAny] diff --git a/backend/right_tree/api/wms_utils.py b/backend/right_tree/api/wms_utils.py index 9aac9e6..bcadac0 100644 --- a/backend/right_tree/api/wms_utils.py +++ b/backend/right_tree/api/wms_utils.py @@ -5,8 +5,11 @@ import requests from urllib.parse import urlencode from unicodedata import normalize +from django.db.models import Q from django.contrib.gis.geos import Point, GEOSGeometry -from django.conf import settings + +from .models import Address + LINZ_API_KEY = os.getenv("LINZ_API_KEY") LINZ_WFS_ENDPOINT = f"https://data.linz.govt.nz/services;key={LINZ_API_KEY}/wfs" @@ -24,10 +27,14 @@ class WFSError(Exception): def get_point_from_coordinates(coordinates): """Given a coordinates json string, returns the coordinates as a Point object""" - coordinates_json = json.loads(coordinates) - pnt = Point(coordinates_json["lng"], - coordinates_json["lat"], srid=4326) - return pnt + if isinstance(coordinates, Point): + return coordinates + elif isinstance(coordinates, str): + coordinates_json = json.loads(coordinates) + elif isinstance(coordinates, dict): + coordinates_json = coordinates + + return Point(coordinates_json["lng"], coordinates_json["lat"], srid=4326) def wfs_getfeature(endpoint, **kwargs): @@ -94,7 +101,6 @@ def get_address_from_coordinates(coordinates): def search_address(address): - # normalize accent characters etc. address = normalize( "NFKD", @@ -104,31 +110,22 @@ def search_address(address): nums = search_num.findall(address) strings = search_str.findall(address) - num_terms = [f"address_number = {n}" for n in nums] - string_terms = [] + num_filter = Q() + str_filter = Q() + + for n in nums: + num_filter |= Q(address_number=n) for s in strings: - string_terms += [ - f"full_road_name_ascii ILIKE '{s}%'", - f"town_city_ascii ILIKE '{s}%'", - f"suburb_locality_ascii ILIKE '{s}%'", - ] - - num_expr = " OR ".join(f"({term})" for term in num_terms) - string_expr = " OR ".join(f"({term})" for term in string_terms) - - cql_filter = f"({num_expr}) AND ({string_expr})" - - resp = linz_wfs_getfeature( - typeNames=PROPERTY_INFO_LAYER, - cql_filter=cql_filter, - count=10, - ) - features = resp.get("features", []) + str_filter |= ( + Q(full_road_name_ascii__istartswith=s) | + Q(town_city_ascii__istartswith=s) | + Q(suburb_locality_ascii__istartswith=s) + ) return [ { - 'coordinates': feature['geometry']['coordinates'], - 'address': feature['properties']['full_address'], - } for feature in features + 'coordinates': (addr.wkb_geometry.x, addr.wkb_geometry.y), + 'address': addr.full_address, + } for addr in Address.objects.filter(num_filter & str_filter).distinct()[:10] ] diff --git a/backend/right_tree/settings.py b/backend/right_tree/settings.py index 75e1c6d..8f43ed0 100644 --- a/backend/right_tree/settings.py +++ b/backend/right_tree/settings.py @@ -81,15 +81,21 @@ WSGI_APPLICATION = 'right_tree.wsgi.application' # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases - +db = { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': os.getenv("DATABASE_NAME", "righttree"), + 'USER': os.getenv("DATABASE_USER", "righttree"), + 'PASSWORD': os.getenv("DATABASE_PASSWORD", "righttree"), + 'HOST': os.getenv("DATABASE_HOST", "postgres"), + 'PORT': int(os.getenv("DATABASE_PORT", 5432)), +} DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': os.getenv("DATABASE_NAME", "righttree"), - 'USER': os.getenv("DATABASE_USER", "righttree"), - 'PASSWORD': os.getenv("DATABASE_PASSWORD", "righttree"), - 'HOST': os.getenv("DATABASE_HOST", "postgres"), - 'PORT': int(os.getenv("DATABASE_PORT", 5432)), + 'default': db, + 'linz': { + **db, + 'OPTIONS': { + 'options': '-c search_path=linz' + }, } } @@ -118,7 +124,7 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +TIME_ZONE = 'Pacific/Auckland' USE_I18N = True @@ -131,8 +137,9 @@ USE_TZ = True # https://docs.djangoproject.com/en/3.2/howto/static-files/ STATIC_URL = '/staticfiles/' -PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) -STATIC_ROOT = os.path.join(PROJECT_DIR, 'staticfiles') +PROJECT_DIR = Path(os.path.abspath(__file__)).parent +STATIC_ROOT = PROJECT_DIR / 'staticfiles' +MEDIA_ROOT = PROJECT_DIR / 'media' # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field @@ -142,3 +149,7 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' CORS_ALLOW_HEADERS = [ 'access-control-allow-origin' ] + +# Celery configuration +CELERY_TIMEZONE = TIME_ZONE +CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://redis:6379/0") diff --git a/backend/right_tree/urls.py b/backend/right_tree/urls.py index d29fe59..daa0743 100644 --- a/backend/right_tree/urls.py +++ b/backend/right_tree/urls.py @@ -28,6 +28,7 @@ router.register(r'region', views.AuckCHCHRegionInformation, basename='region') router.register(r'habitats', views.HabitatViewSet, basename='habitats') router.register(r'zones', views.ZoneViewSet, basename='zones') router.register(r'habitatimage', views.HabitatImageViewSet, basename='habitatimage') +router.register(r'questionnaire', views.QuestionnaireViewSet, basename='questionnaire') router.register(r'download/csv', views.CSVDownloadView, basename='downloadcsv') router.register(r'download/pdf', views.PDFDownloadView, basename='downloadpdf') diff --git a/create_database.sql b/create_database.sql index b943594..ca78e18 100644 --- a/create_database.sql +++ b/create_database.sql @@ -7,5 +7,9 @@ CREATE EXTENSION IF NOT EXISTS postgis; GRANT ALL ON geometry_columns TO PUBLIC; GRANT ALL ON spatial_ref_sys TO PUBLIC; +CREATE SCHEMA IF NOT EXISTS linz; +ALTER SCHEMA linz OWNER TO righttree; +GRANT ALL PRIVILEGES ON SCHEMA linz TO righttree; + ALTER DATABASE righttree OWNER TO righttree; GRANT ALL PRIVILEGES ON DATABASE righttree TO righttree; diff --git a/create_indices.sql b/create_indices.sql new file mode 100644 index 0000000..aeafaa3 --- /dev/null +++ b/create_indices.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS full_road_name_lower_idx ON linz.nz_street_address (lower(full_road_name_ascii) text_pattern_ops); +CREATE INDEX IF NOT EXISTS suburb_locality_lower_idx ON linz.nz_street_address (lower(suburb_locality_ascii) text_pattern_ops); +CREATE INDEX IF NOT EXISTS town_city_lower_idx ON linz.nz_street_address (lower(town_city_ascii) text_pattern_ops); +CREATE INDEX IF NOT EXISTS wkb_geometry_idx ON linz.nz_street_address USING GIST(wkb_geometry); diff --git a/default.env b/default.env index aea3f76..e3b218c 100644 --- a/default.env +++ b/default.env @@ -1,21 +1,9 @@ -# POSTGRES CONFIG -# --------------------------------- -POSTGRES_DB=postgres -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres - - -# RIGHTTREE DATABASE CONFIG (for production) -# --------------------------------- -# RIGHTTREE_DB=righttree -# RIGHTTREE_DB_USER=righttree_admin -# RIGHTTREE_DB_PASSWORD=[YOUR_PASSWORD] - - -# DJANGO BACKEND CONFIG (uncomment out the config below for production) -# --------------------------------- -LINZ_API_KEY=[YOUR_API_KEY] -# FRONTEND_BASE_URL=righttree.maps.net.nz -# DJANGO_DEBUG_MODE=False -# DJANGO_SECRET_KEY=[YOUR_SECRETKEY] -# BASE_URL=righttree.maps.net.nz +LINZ_API_KEY=myapikey +DATABASE_NAME=righttree +DATABASE_USER=righttree +DATABASE_PASSWORD=righttree +DATABASE_HOST=postgres +CELERY_BROKER_URL=redis://redis:6379/0 +BASE_URL=localhost:8000 +DJANGO_SECRET_KEY=changeme +DJANGO_DEBUG_MODE=True diff --git a/docker-compose.production.yaml b/docker-compose.production.yaml index c76e32d..c258def 100644 --- a/docker-compose.production.yaml +++ b/docker-compose.production.yaml @@ -4,26 +4,31 @@ volumes: righttree-postgres-data: name: righttree-postgres-data +x-django: &django + image: right-tree + depends_on: + postgres: + condition: service_healthy + env_file: .env + user: "$UID:$GID" + restart: always + services: backend: - restart: always - build: - context: backend - dockerfile: Dockerfile + <<: *django container_name: backend - depends_on: - - postgres - env_file: .env - ports: - - "8000:8000" - command: bash -c "gunicorn --bind 0.0.0.0:8000 right_tree.wsgi" + expose: + - "8000" + command: + - gunicorn + - --bind=0.0.0.0:8000 + - right_tree.wsgi nginx: container_name: nginx restart: always image: nginx depends_on: - - postgres - backend volumes: - ./nginx.production.conf:/etc/nginx/nginx.conf @@ -35,7 +40,7 @@ services: - "443:443" postgres: - image: postgis/postgis:13-3.0 + image: postgis/postgis:13-3.1 restart: always container_name: postgres volumes: @@ -47,4 +52,38 @@ services: - POSTGRES_DB=${POSTGRES_DB} - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD", "pg_isready", "--dbname", "righttree", "--username", "righttree"] + interval: 10s + timeout: 5s + retries: 5 + redis: + image: redis:7.0.8 + restart: always + container_name: redis + expose: + - "6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + celery: + <<: *django + container_name: celery + command: + - celery + - -A + - right_tree.api + - worker + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + deploy: + resources: + limits: + cpus: '1' diff --git a/docker-compose.yaml b/docker-compose.yaml index 7aad762..fb0ce54 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,45 +4,23 @@ volumes: righttree-postgres-data: name: righttree-postgres-data +x-django: &django + image: right-tree + depends_on: + postgres: + condition: service_healthy + volumes: + - ./backend:/app + env_file: .env + user: "$UID:$GID" + restart: unless-stopped + services: - - backend_migrate: - restart: on-failure - image: right-tree - container_name: backend_migrate - depends_on: - - postgres - volumes: - - ./backend:/app - user: "$UID:$GID" - environment: - DATABASE_NAME: righttree - DATABASE_USER: righttree - DATABASE_PASSWORD: righttree - DATABASE_HOST: postgres - command: - - bash - - -c - - python manage.py makemigrations --noinput && python manage.py migrate --noinput - backend: - restart: unless-stopped - image: right-tree + <<: *django container_name: backend - depends_on: - - postgres - - backend_migrate - volumes: - - ./backend:/app - user: "$UID:$GID" expose: - "8000" - environment: - LINZ_API_KEY: myapikeyhere - DATABASE_NAME: righttree - DATABASE_USER: righttree - DATABASE_PASSWORD: righttree - DATABASE_HOST: postgres command: - gunicorn - --reload @@ -75,9 +53,14 @@ services: - 5432:5432 environment: POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD", "pg_isready", "--dbname", "righttree", "--username", "righttree"] + interval: 10s + timeout: 5s + retries: 5 nginx: - image: nginx:1.23.3 + image: nginx restart: unless-stopped container_name: nginx depends_on: @@ -88,3 +71,31 @@ services: - ./backend/right_tree/staticfiles:/etc/nginx/html/staticfiles:ro ports: - "9000:80" + + redis: + image: redis:7.0.8 + restart: unless-stopped + container_name: redis + expose: + - "6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + celery: + <<: *django + container_name: celery + command: + - celery + - -A + - right_tree.api + - worker + depends_on: + redis: + condition: service_healthy + deploy: + resources: + limits: + cpus: '1' diff --git a/linz.dump b/linz.dump new file mode 100644 index 0000000..fd834af Binary files /dev/null and b/linz.dump differ