From 6dfa65ee77c7330f747b9accce850942237ac6e8 Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Fri, 15 Oct 2021 14:31:52 +1300 Subject: [PATCH 1/8] Add django gis extension --- backend/Dockerfile | 10 ++++++++++ backend/right_tree/settings.py | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 850475b..fb85742 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -6,6 +6,16 @@ ENV DJANGO_SUPERUSER_PASSWORD=admin WORKDIR /app +RUN apt update && \ + apt install -y --no-install-recommends \ + gdal-bin \ + libxml2 libxml2-dev gettext \ + libxslt1-dev libjpeg-dev libpng-dev libpq-dev libgdal-dev \ + software-properties-common g++ \ + zlib1g-dev libgeos-dev libproj-dev \ + sqlite3 spatialite-bin libsqlite3-mod-spatialite && \ + apt clean + COPY ./requirements.txt /app/requirements.txt RUN pip install -U --no-cache-dir -r requirements.txt diff --git a/backend/right_tree/settings.py b/backend/right_tree/settings.py index 0f127e6..e5cecaf 100644 --- a/backend/right_tree/settings.py +++ b/backend/right_tree/settings.py @@ -37,6 +37,7 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django.contrib.gis', 'rest_framework', 'corsheaders', @@ -81,7 +82,7 @@ WSGI_APPLICATION = 'right_tree.wsgi.application' DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql', + 'ENGINE': 'django.contrib.gis.db.backends.postgis', 'NAME': 'right_tree', 'USER': 'postgres', 'PASSWORD': 'postgres', -- 2.45.2 From e068ed047b092ac96421eda55357467548e905db Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Fri, 15 Oct 2021 14:33:13 +1300 Subject: [PATCH 2/8] Add relational models for plants and related fields (excludes habitat and zone) --- .../right_tree/api/migrations/0001_initial.py | 72 ++++++++++++++++- backend/right_tree/api/models.py | 78 ++++++++++++++++++- 2 files changed, 146 insertions(+), 4 deletions(-) diff --git a/backend/right_tree/api/migrations/0001_initial.py b/backend/right_tree/api/migrations/0001_initial.py index 14aa118..17d6a07 100644 --- a/backend/right_tree/api/migrations/0001_initial.py +++ b/backend/right_tree/api/migrations/0001_initial.py @@ -1,6 +1,8 @@ -# Generated by Django 3.2.8 on 2021-10-06 18:32 +# Generated by Django 3.2.8 on 2021-10-15 01:23 +import django.contrib.gis.db.models.fields from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): @@ -11,11 +13,77 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='EcologicalRegion', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=50, unique=True)), + ], + ), + migrations.CreateModel( + name='SoilOrder', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('code', models.CharField(max_length=1, unique=True)), + ('name', models.CharField(max_length=50, unique=True)), + ], + ), + migrations.CreateModel( + name='SoilVariant', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=10, unique=True)), + ], + ), + migrations.CreateModel( + name='ToleranceLevel', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('level', models.CharField(max_length=1)), + ], + ), + migrations.CreateModel( + name='SoilLayer', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('nzsc_class', models.CharField(max_length=4)), + ('nzsc_group', models.CharField(max_length=2)), + ('shape_leng', models.FloatField()), + ('geom', django.contrib.gis.db.models.fields.PolygonField(srid=2193)), + ('nzsc_order', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.soilorder')), + ], + ), migrations.CreateModel( name='Plant', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.TextField()), + ('name', models.CharField(max_length=50, unique=True)), + ('commonname', models.CharField(blank=True, max_length=50, null=True)), + ('maxheight', models.FloatField()), + ('spacing', models.FloatField()), + ('synonym', models.CharField(blank=True, max_length=200, null=True)), + ('purpose', models.TextField(blank=True, null=True)), + ('stage', models.PositiveIntegerField()), + ('growth_form', models.CharField(blank=True, max_length=50, null=True)), + ('drought_tolerance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='drought_tolerance', to='api.tolerancelevel')), + ('ecological_regions', models.ManyToManyField(to='api.EcologicalRegion')), + ('frost_tolerance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='frost_tolerance', to='api.tolerancelevel')), + ('salinity_tolerance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='salinity_tolerance', to='api.tolerancelevel')), + ('soil_order', models.ManyToManyField(to='api.SoilOrder')), + ('soil_variants', models.ManyToManyField(to='api.SoilVariant')), + ('water_tolerance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='water_tolerance', to='api.tolerancelevel')), + ], + ), + migrations.CreateModel( + name='EcologicalDistrictLayer', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ecological', models.CharField(max_length=5)), + ('ecologic_1', models.CharField(max_length=50)), + ('shape_leng', models.FloatField()), + ('shape_area', models.FloatField()), + ('geom', django.contrib.gis.db.models.fields.PolygonField(srid=2193)), + ('ecologic_2', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.ecologicalregion')), ], ), ] diff --git a/backend/right_tree/api/models.py b/backend/right_tree/api/models.py index f1ade3a..8d0292d 100644 --- a/backend/right_tree/api/models.py +++ b/backend/right_tree/api/models.py @@ -1,4 +1,78 @@ -from django.db import models +from django.contrib.gis.db import models + + +class SoilOrder(models.Model): + code = models.CharField(unique=True, max_length=1) + name = models.CharField(unique=True, max_length=50) + + def __str__(self): + return f"{self.name} ({self.code})" + + +class SoilVariant(models.Model): + name = models.CharField(unique=True, max_length=10) + + def __str__(self): + return self.name + + +class SoilLayer(models.Model): + nzsc_class = models.CharField(max_length=4) + nzsc_group = models.CharField(max_length=2) + nzsc_order = models.ForeignKey(SoilOrder, on_delete=models.CASCADE) + shape_leng = models.FloatField() + geom = models.PolygonField(srid=2193) + + def __str__(self): + return self.nzsc_class + + +class EcologicalRegion(models.Model): + name = models.CharField(unique=True, max_length=50) + + def __str__(self): + return self.name + + +class EcologicalDistrictLayer(models.Model): + ecological = models.CharField(max_length=5) + ecologic_1 = models.CharField(max_length=50) + ecologic_2 = models.ForeignKey(EcologicalRegion, on_delete=models.CASCADE) + shape_leng = models.FloatField() + shape_area = models.FloatField() + geom = models.PolygonField(srid=2193) + + def __str__(self): + return f"{self.ecologic_1} ({self.ecologic_2})" + + +class ToleranceLevel(models.Model): + level = models.CharField(max_length=1) + + def __str__(self): + return self.level + class Plant(models.Model): - name = models.TextField() + name = models.CharField(unique=True, max_length=50) + commonname = models.CharField(null=True, blank=True, max_length=50) + maxheight = models.FloatField() + spacing = models.FloatField() + synonym = models.CharField(null=True, blank=True, max_length=200) + water_tolerance = models.ForeignKey( + ToleranceLevel, related_name='water_tolerance', on_delete=models.CASCADE) + drought_tolerance = models.ForeignKey( + ToleranceLevel, related_name='drought_tolerance', on_delete=models.CASCADE) + frost_tolerance = models.ForeignKey( + ToleranceLevel, related_name='frost_tolerance', on_delete=models.CASCADE) + salinity_tolerance = models.ForeignKey( + ToleranceLevel, related_name='salinity_tolerance', on_delete=models.CASCADE) + purpose = models.TextField(null=True, blank=True) + stage = models.PositiveIntegerField() + growth_form = models.CharField(null=True, blank=True, max_length=50) + ecological_regions = models.ManyToManyField(EcologicalRegion) + soil_order = models.ManyToManyField(SoilOrder) + soil_variants = models.ManyToManyField(SoilVariant) + + def __str__(self): + return self.name -- 2.45.2 From d47416273d7da0bdb7a09f26ebd49b01e1f57a25 Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Fri, 15 Oct 2021 14:34:48 +1300 Subject: [PATCH 3/8] Add django fixtures for base data --- backend/right_tree/api/data/__init__.py | 0 .../api/data/fixtures/eco_regions.json | 555 ++++++++++++++++++ .../data/fixtures/soil_order_mappings.json | 170 ++++++ .../api/data/fixtures/soil_variants.json | 23 + .../api/data/fixtures/tolerance_levels.json | 23 + 5 files changed, 771 insertions(+) create mode 100644 backend/right_tree/api/data/__init__.py create mode 100644 backend/right_tree/api/data/fixtures/eco_regions.json create mode 100644 backend/right_tree/api/data/fixtures/soil_order_mappings.json create mode 100644 backend/right_tree/api/data/fixtures/soil_variants.json create mode 100644 backend/right_tree/api/data/fixtures/tolerance_levels.json diff --git a/backend/right_tree/api/data/__init__.py b/backend/right_tree/api/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/right_tree/api/data/fixtures/eco_regions.json b/backend/right_tree/api/data/fixtures/eco_regions.json new file mode 100644 index 0000000..7663a85 --- /dev/null +++ b/backend/right_tree/api/data/fixtures/eco_regions.json @@ -0,0 +1,555 @@ +[ + { + "model": "api.ecologicalregion", + "pk": 1, + "fields": { + "name": "Aorrangi" + } + }, + { + "model": "api.ecologicalregion", + "pk": 2, + "fields": { + "name": "Aspiring" + } + }, + { + "model": "api.ecologicalregion", + "pk": 3, + "fields": { + "name": "Auckland" + } + }, + { + "model": "api.ecologicalregion", + "pk": 4, + "fields": { + "name": "Banks" + } + }, + { + "model": "api.ecologicalregion", + "pk": 5, + "fields": { + "name": "Canterbury Foothills" + } + }, + { + "model": "api.ecologicalregion", + "pk": 6, + "fields": { + "name": "Canterbury Plains" + } + }, + { + "model": "api.ecologicalregion", + "pk": 7, + "fields": { + "name": "Catlins" + } + }, + { + "model": "api.ecologicalregion", + "pk": 8, + "fields": { + "name": "Central Otago" + } + }, + { + "model": "api.ecologicalregion", + "pk": 9, + "fields": { + "name": "Central Volcanic Plateau" + } + }, + { + "model": "api.ecologicalregion", + "pk": 10, + "fields": { + "name": "Clarence" + } + }, + { + "model": "api.ecologicalregion", + "pk": 11, + "fields": { + "name": "Coromandel" + } + }, + { + "model": "api.ecologicalregion", + "pk": 12, + "fields": { + "name": "D'Archiac" + } + }, + { + "model": "api.ecologicalregion", + "pk": 13, + "fields": { + "name": "East Cape" + } + }, + { + "model": "api.ecologicalregion", + "pk": 14, + "fields": { + "name": "Eastern Hawkes Bay" + } + }, + { + "model": "api.ecologicalregion", + "pk": 15, + "fields": { + "name": "Eastern Northland" + } + }, + { + "model": "api.ecologicalregion", + "pk": 16, + "fields": { + "name": "Eastern Volcanic Plateau" + } + }, + { + "model": "api.ecologicalregion", + "pk": 17, + "fields": { + "name": "Eastern Wairarapa" + } + }, + { + "model": "api.ecologicalregion", + "pk": 18, + "fields": { + "name": "Egmont" + } + }, + { + "model": "api.ecologicalregion", + "pk": 19, + "fields": { + "name": "Fiord" + } + }, + { + "model": "api.ecologicalregion", + "pk": 20, + "fields": { + "name": "Gore" + } + }, + { + "model": "api.ecologicalregion", + "pk": 21, + "fields": { + "name": "Hawdon" + } + }, + { + "model": "api.ecologicalregion", + "pk": 22, + "fields": { + "name": "Hawkes Bay" + } + }, + { + "model": "api.ecologicalregion", + "pk": 23, + "fields": { + "name": "Heron" + } + }, + { + "model": "api.ecologicalregion", + "pk": 24, + "fields": { + "name": "Inland Marlborough" + } + }, + { + "model": "api.ecologicalregion", + "pk": 25, + "fields": { + "name": "Kaikoura" + } + }, + { + "model": "api.ecologicalregion", + "pk": 26, + "fields": { + "name": "Kaimanawa" + } + }, + { + "model": "api.ecologicalregion", + "pk": 27, + "fields": { + "name": "Kaipara" + } + }, + { + "model": "api.ecologicalregion", + "pk": 28, + "fields": { + "name": "Kakanui" + } + }, + { + "model": "api.ecologicalregion", + "pk": 29, + "fields": { + "name": "King Country" + } + }, + { + "model": "api.ecologicalregion", + "pk": 30, + "fields": { + "name": "Lakes" + } + }, + { + "model": "api.ecologicalregion", + "pk": 31, + "fields": { + "name": "Lammerlaw" + } + }, + { + "model": "api.ecologicalregion", + "pk": 32, + "fields": { + "name": "Lowry" + } + }, + { + "model": "api.ecologicalregion", + "pk": 33, + "fields": { + "name": "MacKenzie" + } + }, + { + "model": "api.ecologicalregion", + "pk": 34, + "fields": { + "name": "Makarewa" + } + }, + { + "model": "api.ecologicalregion", + "pk": 35, + "fields": { + "name": "Manawatu" + } + }, + { + "model": "api.ecologicalregion", + "pk": 36, + "fields": { + "name": "Manawatu Gorge" + } + }, + { + "model": "api.ecologicalregion", + "pk": 37, + "fields": { + "name": "Mavora" + } + }, + { + "model": "api.ecologicalregion", + "pk": 38, + "fields": { + "name": "Moawhango" + } + }, + { + "model": "api.ecologicalregion", + "pk": 39, + "fields": { + "name": "Molesworth" + } + }, + { + "model": "api.ecologicalregion", + "pk": 40, + "fields": { + "name": "Nelson" + } + }, + { + "model": "api.ecologicalregion", + "pk": 41, + "fields": { + "name": "North Westland" + } + }, + { + "model": "api.ecologicalregion", + "pk": 42, + "fields": { + "name": "North-west Nelson" + } + }, + { + "model": "api.ecologicalregion", + "pk": 43, + "fields": { + "name": "Northern Northland" + } + }, + { + "model": "api.ecologicalregion", + "pk": 44, + "fields": { + "name": "Northern Volcanic Plateau" + } + }, + { + "model": "api.ecologicalregion", + "pk": 45, + "fields": { + "name": "Olivine" + } + }, + { + "model": "api.ecologicalregion", + "pk": 46, + "fields": { + "name": "Otago Coast" + } + }, + { + "model": "api.ecologicalregion", + "pk": 47, + "fields": { + "name": "Pahiatua" + } + }, + { + "model": "api.ecologicalregion", + "pk": 48, + "fields": { + "name": "Pareora" + } + }, + { + "model": "api.ecologicalregion", + "pk": 49, + "fields": { + "name": "Poor Knights" + } + }, + { + "model": "api.ecologicalregion", + "pk": 50, + "fields": { + "name": "Puketeraki" + } + }, + { + "model": "api.ecologicalregion", + "pk": 51, + "fields": { + "name": "Rakiura" + } + }, + { + "model": "api.ecologicalregion", + "pk": 52, + "fields": { + "name": "Rangitikei" + } + }, + { + "model": "api.ecologicalregion", + "pk": 53, + "fields": { + "name": "Raukumara" + } + }, + { + "model": "api.ecologicalregion", + "pk": 54, + "fields": { + "name": "Richmond" + } + }, + { + "model": "api.ecologicalregion", + "pk": 55, + "fields": { + "name": "Rodney" + } + }, + { + "model": "api.ecologicalregion", + "pk": 56, + "fields": { + "name": "Ruahine" + } + }, + { + "model": "api.ecologicalregion", + "pk": 57, + "fields": { + "name": "Sounds-Wellington" + } + }, + { + "model": "api.ecologicalregion", + "pk": 58, + "fields": { + "name": "Southland Foothills" + } + }, + { + "model": "api.ecologicalregion", + "pk": 59, + "fields": { + "name": "Spenser" + } + }, + { + "model": "api.ecologicalregion", + "pk": 60, + "fields": { + "name": "Tainui" + } + }, + { + "model": "api.ecologicalregion", + "pk": 61, + "fields": { + "name": "Taranaki" + } + }, + { + "model": "api.ecologicalregion", + "pk": 62, + "fields": { + "name": "Tararua" + } + }, + { + "model": "api.ecologicalregion", + "pk": 63, + "fields": { + "name": "Tasman" + } + }, + { + "model": "api.ecologicalregion", + "pk": 64, + "fields": { + "name": "Te Paki" + } + }, + { + "model": "api.ecologicalregion", + "pk": 65, + "fields": { + "name": "Te Wae Wae" + } + }, + { + "model": "api.ecologicalregion", + "pk": 66, + "fields": { + "name": "Three Kings" + } + }, + { + "model": "api.ecologicalregion", + "pk": 67, + "fields": { + "name": "Tongariro" + } + }, + { + "model": "api.ecologicalregion", + "pk": 68, + "fields": { + "name": "Urewera" + } + }, + { + "model": "api.ecologicalregion", + "pk": 69, + "fields": { + "name": "Waikaia" + } + }, + { + "model": "api.ecologicalregion", + "pk": 70, + "fields": { + "name": "Waikato" + } + }, + { + "model": "api.ecologicalregion", + "pk": 71, + "fields": { + "name": "Wainono" + } + }, + { + "model": "api.ecologicalregion", + "pk": 72, + "fields": { + "name": "Wairarapa Plains" + } + }, + { + "model": "api.ecologicalregion", + "pk": 73, + "fields": { + "name": "Wairau" + } + }, + { + "model": "api.ecologicalregion", + "pk": 74, + "fields": { + "name": "Wairoa" + } + }, + { + "model": "api.ecologicalregion", + "pk": 75, + "fields": { + "name": "Waitaki" + } + }, + { + "model": "api.ecologicalregion", + "pk": 76, + "fields": { + "name": "Western Northland" + } + }, + { + "model": "api.ecologicalregion", + "pk": 77, + "fields": { + "name": "Western Volcanic Plateau" + } + }, + { + "model": "api.ecologicalregion", + "pk": 78, + "fields": { + "name": "Whataroa" + } + }, + { + "model": "api.ecologicalregion", + "pk": 79, + "fields": { + "name": "Whatkatane" + } + } +] \ No newline at end of file diff --git a/backend/right_tree/api/data/fixtures/soil_order_mappings.json b/backend/right_tree/api/data/fixtures/soil_order_mappings.json new file mode 100644 index 0000000..39847c5 --- /dev/null +++ b/backend/right_tree/api/data/fixtures/soil_order_mappings.json @@ -0,0 +1,170 @@ +[ + { + "model": "api.soilorder", + "pk": 1, + "fields": { + "code": "A", + "name": "Anthropic" + } + }, + { + "model": "api.soilorder", + "pk": 2, + "fields": { + "code": "B", + "name": "Brown" + } + }, + { + "model": "api.soilorder", + "pk": 3, + "fields": { + "code": "G", + "name": "Gley" + } + }, + { + "model": "api.soilorder", + "pk": 4, + "fields": { + "code": "L", + "name": "Allophanic" + } + }, + { + "model": "api.soilorder", + "pk": 5, + "fields": { + "code": "N", + "name": "Granular" + } + }, + { + "model": "api.soilorder", + "pk": 6, + "fields": { + "code": "E", + "name": "Melanic" + } + }, + { + "model": "api.soilorder", + "pk": 7, + "fields": { + "code": "O", + "name": "Organic" + } + }, + { + "model": "api.soilorder", + "pk": 8, + "fields": { + "code": "X", + "name": "Oxidic" + } + }, + { + "model": "api.soilorder", + "pk": 9, + "fields": { + "code": "P", + "name": "Pallic" + } + }, + { + "model": "api.soilorder", + "pk": 10, + "fields": { + "code": "Z", + "name": "Podzol" + } + }, + { + "model": "api.soilorder", + "pk": 11, + "fields": { + "code": "M", + "name": "Pumice" + } + }, + { + "model": "api.soilorder", + "pk": 12, + "fields": { + "code": "W", + "name": "Raw" + } + }, + { + "model": "api.soilorder", + "pk": 13, + "fields": { + "code": "R", + "name": "Recent" + } + }, + { + "model": "api.soilorder", + "pk": 14, + "fields": { + "code": "S", + "name": "Semi" + } + }, + { + "model": "api.soilorder", + "pk": 15, + "fields": { + "code": "U", + "name": "Ultic" + } + }, + { + "model": "api.soilorder", + "pk": 16, + "fields": { + "code": "i", + "name": "Ice" + } + }, + { + "model": "api.soilorder", + "pk": 17, + "fields": { + "code": "t", + "name": "Town" + } + }, + { + "model": "api.soilorder", + "pk": 18, + "fields": { + "code": "r", + "name": "River" + } + }, + { + "model": "api.soilorder", + "pk": 19, + "fields": { + "code": "e", + "name": "Estu" + } + }, + { + "model": "api.soilorder", + "pk": 20, + "fields": { + "code": "l", + "name": "Lake" + } + }, + { + "model": "api.soilorder", + "pk": 21, + "fields": { + "code": "q", + "name": "Quar" + } + } +] \ No newline at end of file diff --git a/backend/right_tree/api/data/fixtures/soil_variants.json b/backend/right_tree/api/data/fixtures/soil_variants.json new file mode 100644 index 0000000..0d882b3 --- /dev/null +++ b/backend/right_tree/api/data/fixtures/soil_variants.json @@ -0,0 +1,23 @@ +[ + { + "model": "api.soilvariant", + "pk": 1, + "fields": { + "name": "Wet" + } + }, + { + "model": "api.soilvariant", + "pk": 2, + "fields": { + "name": "Mesic" + } + }, + { + "model": "api.soilvariant", + "pk": 3, + "fields": { + "name": "Dry" + } + } +] \ No newline at end of file diff --git a/backend/right_tree/api/data/fixtures/tolerance_levels.json b/backend/right_tree/api/data/fixtures/tolerance_levels.json new file mode 100644 index 0000000..165a754 --- /dev/null +++ b/backend/right_tree/api/data/fixtures/tolerance_levels.json @@ -0,0 +1,23 @@ +[ + { + "model": "api.tolerancelevel", + "pk": 1, + "fields": { + "level": "M" + } + }, + { + "model": "api.tolerancelevel", + "pk": 2, + "fields": { + "level": "H" + } + }, + { + "model": "api.tolerancelevel", + "pk": 3, + "fields": { + "level": "L" + } + } +] \ No newline at end of file -- 2.45.2 From b02d7d50de67dac94d535105e732a11528cbeed4 Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Fri, 15 Oct 2021 14:38:14 +1300 Subject: [PATCH 4/8] Register models with django admin --- backend/right_tree/api/admin.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/right_tree/api/admin.py b/backend/right_tree/api/admin.py index 8c38f3f..36da8b8 100644 --- a/backend/right_tree/api/admin.py +++ b/backend/right_tree/api/admin.py @@ -1,3 +1,10 @@ from django.contrib import admin +from right_tree.api.models import Plant, SoilOrder, SoilLayer, SoilVariant, EcologicalRegion, EcologicalDistrictLayer, ToleranceLevel -# Register your models here. +admin.site.register(Plant) +admin.site.register(SoilOrder) +admin.site.register(SoilLayer) +admin.site.register(SoilVariant) +admin.site.register(EcologicalRegion) +admin.site.register(EcologicalDistrictLayer) +admin.site.register(ToleranceLevel) -- 2.45.2 From aa57c6e101ca6c805ef115f5901d5b98646f3efa Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Fri, 15 Oct 2021 14:39:03 +1300 Subject: [PATCH 5/8] Add django command to populate the database with shapefile data --- backend/right_tree/api/management/__init__.py | 0 .../api/management/commands/loadshapefiles.py | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 backend/right_tree/api/management/__init__.py create mode 100644 backend/right_tree/api/management/commands/loadshapefiles.py diff --git a/backend/right_tree/api/management/__init__.py b/backend/right_tree/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/right_tree/api/management/commands/loadshapefiles.py b/backend/right_tree/api/management/commands/loadshapefiles.py new file mode 100644 index 0000000..4fa9f96 --- /dev/null +++ b/backend/right_tree/api/management/commands/loadshapefiles.py @@ -0,0 +1,44 @@ +from django.core.management.base import BaseCommand +from django.contrib.gis.utils import LayerMapping + +from pathlib import Path + +import right_tree.api.data +from right_tree.api.models import SoilLayer, EcologicalDistrictLayer + +# Auto-generated `LayerMapping` dictionary for SoilLayers model +soillayer_mapping = { + 'nzsc_class': 'nzsc_class', + 'nzsc_group': 'nzsc_group', + 'nzsc_order': {'code': 'nzsc_order'}, + 'shape_leng': 'SHAPE_Leng', + 'geom': 'POLYGON', +} + +# Auto-generated `LayerMapping` dictionary for ecologicaldistrictlayer model +ecologicaldistrictlayer_mapping = { + 'ecological': 'ECOLOGICAL', + 'ecologic_1': 'ECOLOGIC_1', + 'ecologic_2': {'name': 'ECOLOGIC_2'}, + 'shape_leng': 'SHAPE_Leng', + 'shape_area': 'SHAPE_Area', + 'geom': 'POLYGON', +} + +# Shapefiles +soillayer_shp = Path(right_tree.api.data.__file__).resolve().parent / 'resources' / 'fundamental_soil_layers' / 'fundamental-soil-layers-new-zealand-soil-classification.shp' +ecologicaldistrictlayer_shp = Path(right_tree.api.data.__file__).resolve().parent / 'resources' / 'ecological_districts' / 'DOC_EcologicalDistricts_2021_08_02.shp' + +class Command(BaseCommand): + help = 'Ingests the shapefile data for ecological regions and soil layers.' + + def handle(self, *args, **options): + self.stdout.write('Loading soil layers...') + soil_lm = LayerMapping(SoilLayer, soillayer_shp, soillayer_mapping, transform=False) + soil_lm.save(strict=True) + self.stdout.write(self.style.SUCCESS('Soil layers loaded succesfully.')) + + self.stdout.write('Loading ecological district layers...') + ecologicaldistrictlayer_lm = LayerMapping(EcologicalDistrictLayer, ecologicaldistrictlayer_shp, ecologicaldistrictlayer_mapping, transform=False) + ecologicaldistrictlayer_lm.save(strict=True) + self.stdout.write(self.style.SUCCESS('Ecological district layers loaded succesfully.')) -- 2.45.2 From 299e609c25e21f2956f6f2d17a43f63af05549fb Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Fri, 15 Oct 2021 14:40:14 +1300 Subject: [PATCH 6/8] Add django command to populate the database with spreadsheet plant data --- backend/.gitignore | 5 +- backend/requirements.txt | 1 + .../commands/_spreadsheet_helpers.py | 41 +++++ .../commands/createplantfixtures.py | 165 ++++++++++++++++++ .../api/management/commands/resetplants.py | 11 ++ 5 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 backend/right_tree/api/management/commands/_spreadsheet_helpers.py create mode 100644 backend/right_tree/api/management/commands/createplantfixtures.py create mode 100644 backend/right_tree/api/management/commands/resetplants.py diff --git a/backend/.gitignore b/backend/.gitignore index e30397b..28d3762 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,3 +1,6 @@ *.pyc *.sqlite3 -__pycache__ \ No newline at end of file +__pycache__ + +resources +right_tree/api/data/fixtures/plants.json \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt index f3dc6cb..35f9bd1 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,3 +2,4 @@ Django==3.2.8 psycopg2-binary>=2.8 djangorestframework==3.12.4 django-cors-headers==3.10.0 +openpyxl==3.0.9 \ No newline at end of file diff --git a/backend/right_tree/api/management/commands/_spreadsheet_helpers.py b/backend/right_tree/api/management/commands/_spreadsheet_helpers.py new file mode 100644 index 0000000..974f81f --- /dev/null +++ b/backend/right_tree/api/management/commands/_spreadsheet_helpers.py @@ -0,0 +1,41 @@ +from openpyxl import load_workbook + +def get_pk_mapping(object, mapping_key="name"): + pk_mapping = {} + for instance in object.objects.all(): + pk_mapping[getattr(instance, mapping_key)] = instance.pk + + return pk_mapping + + +def get_col_mappings(sheet, start_col, row_index): + col_mappings = {} + for row in sheet.iter_rows(min_col=start_col, min_row=row_index, max_row=row_index, values_only=True): + for i, col_name in enumerate(row): + col_mappings[col_name] = i + + return col_mappings + + +def get_pk_list_from_str(values_str, pk_mapping, fixes={}): + pk_list = [] + for value in values_str.split(','): + processed_value = value.lstrip().rstrip().replace( + '_', ' ').replace('-', ' ').replace('’', '\'') + + # Applies any mapping adjustments between spreadsheet data and the database values + if fixes and processed_value in fixes: + processed_value = fixes[processed_value] + + # Adds the pk value for the value in the databse + if processed_value in pk_mapping: + pk_list.append(pk_mapping[processed_value]) + + return pk_list + + +def get_spreadsheet(data_path, spreadsheet_filename): + spreadsheet_path = data_path / 'resources' / spreadsheet_filename + workbook = load_workbook(filename=spreadsheet_path) + return workbook.active + diff --git a/backend/right_tree/api/management/commands/createplantfixtures.py b/backend/right_tree/api/management/commands/createplantfixtures.py new file mode 100644 index 0000000..33dc4bb --- /dev/null +++ b/backend/right_tree/api/management/commands/createplantfixtures.py @@ -0,0 +1,165 @@ +from django.core.management.base import BaseCommand + +import json +from pathlib import Path + +import right_tree.api.data +from ._spreadsheet_helpers import * +from right_tree.api.models import EcologicalRegion, SoilOrder, SoilVariant, ToleranceLevel + + +ECO_REGION_ADJUSTMENTS = { + "Whakatane": "Whatkatane", + "North West Nelson": "North-west Nelson", + "Aorangi": "Aorrangi", + "Mackenzie": "MacKenzie", + "Southland Hills": "Southland Foothills", + "Sounds Wellington": "Sounds-Wellington" +} + +PLANT_COLS = { + 'name': {"expected_type": str, "max_length": 50}, + 'maxheight': {"expected_type": float}, + 'spacing': {"expected_type": float}, + 'commonname': {"expected_type": str, "null_allowed": True, "max_length": 50}, + 'synonym': {"expected_type": str, "null_allowed": True, "max_length": 200}, + 'region': {"expected_type": list, "model_name": "ecological_regions"}, + 'soilorder': {"expected_type": list, "model_name": "soil_order"}, + 'wet': {"expected_type": list, "model_name": "soil_variants"}, + 'mesic': {"expected_type": list, "model_name": "soil_variants"}, + 'dry': {"expected_type": list, "model_name": "soil_variants"}, + 'water': {"expected_type": int, "model_name": "water_tolerance"}, + 'drought': {"expected_type": int, "model_name": "drought_tolerance"}, + 'frost': {"expected_type": int, "model_name": "frost_tolerance"}, + 'salinity': {"expected_type": int, "model_name": "salinity_tolerance"}, + 'purpose': {"expected_type": str, "null_allowed": True}, + 'stage': {"expected_type": int}, + 'growthform': {"expected_type": str, "model_name": "growth_form", "null_allowed": True, "max_length": 50} +} + +SPREADSHEET_FILENAME = 'plant_data.xlsx' +DATA_START_COL = 3 +DATA_START_ROW = 7 +INFO_HEADER_ROW = 6 + +DATA_DIR_PATH = Path(right_tree.api.data.__file__).resolve().parent + +ECO_REGION_PK_MAPPING = get_pk_mapping(EcologicalRegion) +SOIL_ORDER_PK_MAPPING = get_pk_mapping(SoilOrder) +SOIL_VARIANT_PK_MAPPING = get_pk_mapping(SoilVariant) +TOLERANCE_PK_MAPPING = get_pk_mapping(ToleranceLevel, "level") + +SPREADSHEET = get_spreadsheet(DATA_DIR_PATH, SPREADSHEET_FILENAME) +INFO_COL_INDEXES = get_col_mappings( + SPREADSHEET, DATA_START_COL, INFO_HEADER_ROW) + +PLANT_JSON_TEMPLATE = { + "model": "api.plant", + "pk": None, + "fields": {} +} + + +def check_field_type(field, field_value): + expected_field_type = PLANT_COLS[field]['expected_type'] + model_field_name = PLANT_COLS[field].get('model_name', field) + null_allowed = PLANT_COLS[field].get('null_allowed', False) + max_length = PLANT_COLS[field].get('max_length', False) + + is_valid_type = isinstance(field_value, expected_field_type) + is_int_when_float = expected_field_type == float and isinstance( + field_value, int) + is_valid_null = field_value is None and null_allowed + is_over_max_length = max_length and isinstance( + field_value, str) and len(field_value) > max_length + + if not(is_valid_type or is_int_when_float or is_valid_null): + raise TypeError( + f"Invalid json type for field {model_field_name} with value {field_value}. Expected '{expected_field_type}' but got '{type(field_value)}'.") + elif is_over_max_length: + raise TypeError( + f"Invalid string length for {model_field_name} with value {field_value}. Expected length to be under {max_length} but was {len(field_value)}.") + + +def get_plant_json_from_row(row_data): + plant_json_fields = {} + for field, field_index in INFO_COL_INDEXES.items(): + if field not in PLANT_COLS: + continue + + model_field_name = PLANT_COLS[field].get('model_name', field) + try: + if field == "region": + regions_list = get_pk_list_from_str( + row_data[field_index], ECO_REGION_PK_MAPPING, ECO_REGION_ADJUSTMENTS) + plant_json_fields[model_field_name] = regions_list + + elif field == "soilorder": + soil_orders_list = get_pk_list_from_str( + row_data[field_index], SOIL_ORDER_PK_MAPPING) + plant_json_fields[model_field_name] = soil_orders_list + + elif field in {'wet', 'mesic', 'dry'}: + soil_variant_pk = SOIL_VARIANT_PK_MAPPING[field.capitalize()] + plant_json_fields[model_field_name] = plant_json_fields.get( + model_field_name, []) + [soil_variant_pk] + + elif field in {'water', 'drought', 'frost', 'salinity'}: + plant_json_fields[model_field_name] = TOLERANCE_PK_MAPPING[row_data[field_index]] + + elif field in PLANT_COLS: + plant_json_fields[model_field_name] = row_data[field_index] + + check_field_type(field, plant_json_fields[model_field_name]) + + except Exception as e: + name_index = INFO_COL_INDEXES['name'] + print( + f"Error occured while adding the row for {row_data[name_index]}.") + print(F"{type(e)}: {e}") + print("SKIPPING ROW...") + print("----------------------------------------------") + + return {} + + plant_json = PLANT_JSON_TEMPLATE.copy() + plant_json["fields"] = plant_json_fields + + return plant_json + + +def get_plant_json_fixture(sheet): + plant_json_fixture = [] + skipped_count = 0 + created_count = 0 + + for row in sheet.iter_rows(min_col=DATA_START_COL, min_row=DATA_START_ROW, values_only=True): + plant_json = get_plant_json_from_row(row) + + # If there is invalid data in a row, it will be skipped + if plant_json != {}: + plant_json_fixture.append(plant_json) + created_count += 1 + else: + skipped_count += 1 + + print("Created plants fixture.") + print(f"Rows Created: {created_count}") + print(f"Rows Skipped: {skipped_count}") + return plant_json_fixture + + +def save_plant_fixture(fixture): + fixture_filepath = DATA_DIR_PATH / 'fixtures' / 'plants.json' + fixture_filepath.write_text(json.dumps(fixture)) + + +class Command(BaseCommand): + help = 'Ingests the plant spreadsheet data into the database' + + def handle(self, *args, **options): + self.stdout.write('Creating plant fixtures...') + plant_fixture = get_plant_json_fixture(SPREADSHEET) + save_plant_fixture(plant_fixture) + self.stdout.write(self.style.SUCCESS( + 'Plant fixtures created and saved successfully.')) diff --git a/backend/right_tree/api/management/commands/resetplants.py b/backend/right_tree/api/management/commands/resetplants.py new file mode 100644 index 0000000..4ae2214 --- /dev/null +++ b/backend/right_tree/api/management/commands/resetplants.py @@ -0,0 +1,11 @@ +from django.core.management.base import BaseCommand +from right_tree.api.models import Plant + + +class Command(BaseCommand): + help = 'Removes all plant objects from the database' + + def handle(self, *args, **options): + self.stdout.write(self.style.WARNING( + 'Removing all plant objects from the database.')) + Plant.objects.all().delete() -- 2.45.2 From 1cb7403f6f04ce5d4c96db1234da2f847e4a17ae Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Fri, 15 Oct 2021 15:20:14 +1300 Subject: [PATCH 7/8] Update dev commands with docs --- README.md | 91 ++++++++++++++++--- ..._init_righttree.sql => create_database.sql | 0 database/init_database.sh | 3 - dev | 89 ++++++++++++++++++ docker-compose.yaml | 7 +- 5 files changed, 167 insertions(+), 23 deletions(-) rename database/init/001_init_righttree.sql => create_database.sql (100%) delete mode 100755 database/init_database.sh create mode 100755 dev diff --git a/README.md b/README.md index 1fda6fa..a8d3ceb 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ # RightTree Right Plant Right Place Right Time implementation using React and Django. - -## Running application for development -### Initial Setup +## Initial Setup Before running the applications please ensure the following prerequisites have been met. #### Software @@ -15,20 +13,56 @@ $ sudo apt install git docker-compose To install `docker`, follow the [official installation documentation](https://docs.docker.com/get-docker/). [Instructions are also available for `docker-compose`](https://docs.docker.com/compose/install/). -#### Initialise database +You may also need to give the `dev` script executable permissions using the following command: + +``` +chmod +x ./dev +``` + +### Add shapefiles for database population + +Please unzip and add the following shapefiles to the `./backend/right_tree/api/data/resources` directory. It should include all the files required by the shapefile and use naming conventions as follows: + +**Ecological Districts Shapefile:** +``` +backend/right_tree/api/data/resources/ecological_districts/ +- DOC_EcologicalDistricts_2021_08_02.cpg +- DOC_EcologicalDistricts_2021_08_02.dbf +- DOC_EcologicalDistricts_2021_08_02.prj +- DOC_EcologicalDistricts_2021_08_02.sbn +- DOC_EcologicalDistricts_2021_08_02.sbx +- DOC_EcologicalDistricts_2021_08_02.shp +- DOC_EcologicalDistricts_2021_08_02.shp.xml +- DOC_EcologicalDistricts_2021_08_02.shx +``` + +**Ecological Districts Shapefile:** +``` +backend/right_tree/api/data/resources/fundamental_soil_layers/ +- fundamental-soil-layers-new-zealand-soil-classification.cpg +- fundamental-soil-layers-new-zealand-soil-classification.dbf +- fundamental-soil-layers-new-zealand-soil-classification.prj +- fundamental-soil-layers-new-zealand-soil-classification.shp +- fundamental-soil-layers-new-zealand-soil-classification.shx +- fundamental-soil-layers-new-zealand-soil-classification.xml +``` +### Add spreadsheet data for database population + +The plant spreadsheet should be renamed as `plant_data.xlsx` and placed in the `./backend/right_tree/api/data/resources` directory. +## Running application for development +### Initial build + +Builds the Django backend docker image. This may need to be re-run if any new dependencies are added. +``` +./dev build +``` + +### Initialise database Creates `right_tree` database and installs `postgis` extensions. ``` -chmod +x ./database/init_database.sh -./database/init_database.sh -``` - -#### Initial build - -Builds the Django backend docker image. This may need to be re-run if any new dependencies are added. -``` -docker-compose build +./dev init_database ``` ### Run web application @@ -36,7 +70,7 @@ docker-compose build Starts up the applications including the frontend, backend and database. ``` -docker-compose up +./dev start ``` Once running the components can be accessed as follows: @@ -45,4 +79,31 @@ Once running the components can be accessed as follows: | --- | --- | | React Frontend | http://localhost:3000 | | Django Backend | http://localhost:8000 | -| Database | postgis://localhost:5432 | \ No newline at end of file +| Database | postgis://localhost:5432 | + +## Available commands + +Other commands can be run using the following. +``` +./dev +``` + +A summary of available commands are outlined below. Note that if the command requires the application to be running (`Requires Run`) please execute `./dev start` in another terminal before running that command. + +| Command | Description | Requires Run | +| --- | --- | --- | +| `create_database` | Removes the existing database and data. Then it creates the `right_tree` database within a fresh postgis database instance. | No +| `makemigrations` | Performs the django `makemigrations` command in the backend container. | Yes +| `migrate` | Performs the django `migrate` command in the backend container. | Yes +| `createsuperuser` | Performs the django `createsuperuser` command in the backend container. | Yes +| `load_fixtures` | Performs the django `loaddata` command in the backend container. This loads all the fixtures found in the `/backend/right_tree/api/data/fixtures` directory. | Yes +| `load_shapefiles` | Performs the custom `loadshapefiles` command in the backend container. This loads the ecological districts and soil layers shape files in `c`. | Yes +| `create_plant_fixtures` | Performs the custom `createplantfixtures` command in the backend container. This loads the plant spreadsheet data from `/backend/right_tree/api/data/resources/plant_data.xlsx`. Requires the fixtures to be applied and shapefiles loaded. | Yes +| `reset_plants` | Performs the custom `resetplants` command in the backend container. This removes all plant entries from the database. | Yes +| `load_plant_fixtures` | Loads the `/backend/right_tree/api/data/fixtures/plants.json` fixture. Requires the `plants.json` file to be created (`./dev create_plant_fixtures`) and the plant table to be empty (`./dev reset_plants`). | Yes +| `load_plants` | Creates plants fixtures and loads them into a fresh plant table in the database. Requires the fixtures to be applied and shapefiles loaded. | Yes +| `populate_database` | Populates the `right_tree` database with base data (fixtures), provided shapefiles and plant spreadsheet data. Requires the database to be created. | No +| `init_database` | Creates and populates the database | No +| `reset_database` | Removes, recreates and populates the database | No +| `build` | Builds required images | No +| `start` | Runs all services including the frontend, backend and postgres database | No \ No newline at end of file diff --git a/database/init/001_init_righttree.sql b/create_database.sql similarity index 100% rename from database/init/001_init_righttree.sql rename to create_database.sql diff --git a/database/init_database.sh b/database/init_database.sh deleted file mode 100755 index 04ad0fe..0000000 --- a/database/init_database.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker-compose down --remove-orphans --volumes -docker-compose up postgres | sed '/PostgreSQL init process complete; ready for start up./q' -docker-compose down diff --git a/dev b/dev new file mode 100755 index 0000000..156b514 --- /dev/null +++ b/dev @@ -0,0 +1,89 @@ +#!/bin/bash + +cmd_create_database() { + echo "Creating right_tree database..." + docker-compose down --remove-orphans --volumes + docker-compose -f docker-compose.yaml up postgres | sed '/PostgreSQL init process complete; ready for start up./q' + docker-compose down +} + +cmd_makemigrations() { + echo "Creating database migrations..." + docker-compose exec django-backend python manage.py makemigrations --no-input +} + +cmd_migrate() { + echo "Running database migrations..." + docker-compose exec django-backend python manage.py migrate +} + +cmd_createsuperuser() { + echo "Loading shapefiles into the database..." + docker-compose exec django-backend python manage.py createsuperuser --noinput +} + +cmd_load_fixtures() { + echo "Loading fixtures..." + docker-compose exec django-backend bash -c "python manage.py loaddata right_tree/api/data/fixtures/*.json" +} + +cmd_load_shapefiles() { + echo "Loading shapefiles into the database..." + docker-compose exec django-backend python manage.py loadshapefiles +} + +cmd_create_plant_fixtures() { + echo "Creates fixtures for plants using spreadsheet." + docker-compose exec django-backend python manage.py createplantfixtures +} + +cmd_reset_plants() { + echo "Resetting plants..." + docker-compose exec django-backend python manage.py resetplants +} + +cmd_load_plant_fixtures() { + echo "Loading plants..." + docker-compose exec django-backend python manage.py loaddata right_tree/api/data/fixtures/plants.json +} + +cmd_load_plants() { + cmd_create_plant_fixtures + cmd_reset_plants + cmd_load_plant_fixtures +} + +cmd_populate_database() { + echo "Populating the database..." + docker-compose up -d django-backend postgres + + cmd_makemigrations + cmd_migrate + cmd_createsuperuser + cmd_load_fixtures + cmd_load_shapefiles + cmd_load_plants + + docker-compose down +} + +cmd_init_database() { + cmd_create_database + cmd_populate_database +} + +cmd_reset_database() { + cmd_init_database +} + +cmd_build() { + docker-compose build +} + +cmd_start() { + docker-compose up +} + +# Run the command +cmd="$1" +"cmd_$cmd" "$@" diff --git a/docker-compose.yaml b/docker-compose.yaml index 440cc6a..aa48ae5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,10 +17,7 @@ services: - ./backend:/app ports: - "8000:8000" - command: bash -c "./manage.py makemigrations; - ./manage.py migrate; - ./manage.py createsuperuser --noinput; - ./manage.py runserver 0.0.0.0:8000" + command: bash -c "./manage.py runserver 0.0.0.0:8000" react-frontend: image: node:16-alpine3.11 @@ -39,7 +36,7 @@ services: container_name: postgres volumes: - local-postgres-data:/var/lib/postgresql/data - - ./database/init:/docker-entrypoint-initdb.d + - ./create_database.sql:/docker-entrypoint-initdb.d/create_database.sql ports: - "5432:5432" environment: -- 2.45.2 From 530089c19c672bdef44cd3da54951353a3ca4529 Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Tue, 19 Oct 2021 10:39:07 +1300 Subject: [PATCH 8/8] Add documentation to the spreadsheet processing methods --- .../commands/_spreadsheet_helpers.py | 12 +++++++++++- .../management/commands/createplantfixtures.py | 18 +++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/backend/right_tree/api/management/commands/_spreadsheet_helpers.py b/backend/right_tree/api/management/commands/_spreadsheet_helpers.py index 974f81f..719ceb5 100644 --- a/backend/right_tree/api/management/commands/_spreadsheet_helpers.py +++ b/backend/right_tree/api/management/commands/_spreadsheet_helpers.py @@ -1,6 +1,9 @@ from openpyxl import load_workbook + def get_pk_mapping(object, mapping_key="name"): + """ Returns a dictionary mapping a django model primary key to another given field. + """ pk_mapping = {} for instance in object.objects.all(): pk_mapping[getattr(instance, mapping_key)] = instance.pk @@ -9,6 +12,8 @@ def get_pk_mapping(object, mapping_key="name"): def get_col_mappings(sheet, start_col, row_index): + """ Returns a dictionary that maps a spreadsheet cell value to a corresponding column index. + """ col_mappings = {} for row in sheet.iter_rows(min_col=start_col, min_row=row_index, max_row=row_index, values_only=True): for i, col_name in enumerate(row): @@ -18,6 +23,9 @@ def get_col_mappings(sheet, start_col, row_index): def get_pk_list_from_str(values_str, pk_mapping, fixes={}): + """ Given a list of comma separated values from the spreadsheet. Returns a list of primary keys that + correspond to the relevant values with any given mapping fixes applied. + """ pk_list = [] for value in values_str.split(','): processed_value = value.lstrip().rstrip().replace( @@ -35,7 +43,9 @@ def get_pk_list_from_str(values_str, pk_mapping, fixes={}): def get_spreadsheet(data_path, spreadsheet_filename): + """ Returns a spreadsheet from a resources directory given the data path and + spreadsheet filename. + """ spreadsheet_path = data_path / 'resources' / spreadsheet_filename workbook = load_workbook(filename=spreadsheet_path) return workbook.active - diff --git a/backend/right_tree/api/management/commands/createplantfixtures.py b/backend/right_tree/api/management/commands/createplantfixtures.py index 33dc4bb..cba37a4 100644 --- a/backend/right_tree/api/management/commands/createplantfixtures.py +++ b/backend/right_tree/api/management/commands/createplantfixtures.py @@ -7,7 +7,7 @@ import right_tree.api.data from ._spreadsheet_helpers import * from right_tree.api.models import EcologicalRegion, SoilOrder, SoilVariant, ToleranceLevel - +# Mapping adjustments between the shapefile ecological regions and those in the spreadsheet ECO_REGION_ADJUSTMENTS = { "Whakatane": "Whatkatane", "North West Nelson": "North-west Nelson", @@ -17,6 +17,7 @@ ECO_REGION_ADJUSTMENTS = { "Sounds Wellington": "Sounds-Wellington" } +# Relevant columns and information used to retrieve information from the spreadsheet PLANT_COLS = { 'name': {"expected_type": str, "max_length": 50}, 'maxheight': {"expected_type": float}, @@ -37,22 +38,27 @@ PLANT_COLS = { 'growthform': {"expected_type": str, "model_name": "growth_form", "null_allowed": True, "max_length": 50} } +# Spreadsheet constants SPREADSHEET_FILENAME = 'plant_data.xlsx' DATA_START_COL = 3 DATA_START_ROW = 7 INFO_HEADER_ROW = 6 +# Data directory path DATA_DIR_PATH = Path(right_tree.api.data.__file__).resolve().parent +# Mappings between values in the spreadsheet and primary key values in the database ECO_REGION_PK_MAPPING = get_pk_mapping(EcologicalRegion) SOIL_ORDER_PK_MAPPING = get_pk_mapping(SoilOrder) SOIL_VARIANT_PK_MAPPING = get_pk_mapping(SoilVariant) TOLERANCE_PK_MAPPING = get_pk_mapping(ToleranceLevel, "level") +# Spreadsheet and corresponding value to column index mappings SPREADSHEET = get_spreadsheet(DATA_DIR_PATH, SPREADSHEET_FILENAME) INFO_COL_INDEXES = get_col_mappings( SPREADSHEET, DATA_START_COL, INFO_HEADER_ROW) +# Template for the plant json to add as an entry for the fixtures PLANT_JSON_TEMPLATE = { "model": "api.plant", "pk": None, @@ -61,6 +67,8 @@ PLANT_JSON_TEMPLATE = { def check_field_type(field, field_value): + """ Checks the validity of a feild value collected from the spreadsheet + """ expected_field_type = PLANT_COLS[field]['expected_type'] model_field_name = PLANT_COLS[field].get('model_name', field) null_allowed = PLANT_COLS[field].get('null_allowed', False) @@ -82,6 +90,8 @@ def check_field_type(field, field_value): def get_plant_json_from_row(row_data): + """ Returns a json object representing a plant row from the spreadsheet. + """ plant_json_fields = {} for field, field_index in INFO_COL_INDEXES.items(): if field not in PLANT_COLS: @@ -129,6 +139,8 @@ def get_plant_json_from_row(row_data): def get_plant_json_fixture(sheet): + """ Returns a django fixture json that represents the plant information extracted from the spreadsheet. + """ plant_json_fixture = [] skipped_count = 0 created_count = 0 @@ -143,13 +155,17 @@ def get_plant_json_fixture(sheet): else: skipped_count += 1 + # Print summary of data extraction from the spreadsheet print("Created plants fixture.") print(f"Rows Created: {created_count}") print(f"Rows Skipped: {skipped_count}") + return plant_json_fixture def save_plant_fixture(fixture): + """ Saves the plant fixture to the django api fixtures directory. + """ fixture_filepath = DATA_DIR_PATH / 'fixtures' / 'plants.json' fixture_filepath.write_text(json.dumps(fixture)) -- 2.45.2