diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..89cf447 --- /dev/null +++ b/Makefile @@ -0,0 +1,60 @@ +#!/usr/bin/env make + +SHELL = /bin/bash +UID := $(shell id -u) +GID := $(shell id -g) + +export UID +export GID + +frontend/node_modules: + docker run --rm -v ${PWD}/frontend:/app -w /app -u ${UID}:${GID} node:16-bullseye npm i + +backend/right_tree/staticfiles: + docker run --rm -v ${PWD}/backend:/app -w /app -u ${UID}:${GID} right-tree python manage.py collectstatic --noinput + +ingest: + docker-compose up -d backend postgres + docker-compose exec backend python manage.py loaddata \ + /app/right_tree/api/data/fixtures/001_eco_regions.json \ + /app/right_tree/api/data/fixtures/002_tolerance_levels.json \ + /app/right_tree/api/data/fixtures/003_soil_variants.json \ + /app/right_tree/api/data/fixtures/004_soil_order_mappings.json \ + /app/right_tree/api/data/fixtures/005_habitats.json \ + /app/right_tree/api/data/fixtures/006_zones.json \ + /app/right_tree/api/data/fixtures/007_habitat_images.json + docker-compose exec backend python manage.py loadshapefiles + docker-compose exec backend python manage.py createplantfixtures + docker-compose exec backend python manage.py loaddata \ + /app/right_tree/api/data/fixtures/plants.json + +createsuperuser: + docker-compose up -d backend + docker-compose exec backend python manage.py createsuperuser + +shell: + docker-compose up -d backend + docker-compose exec backend python manage.py shell + +psql: + docker-compose up -d postgres + docker-compose exec postgres psql -U righttree -d righttree + +build: + docker build --no-cache -t right-tree backend + +start: frontend/node_modules backend/right_tree/staticfiles + docker-compose up -d + docker-compose logs -f + +logs: + docker-compose logs -f + +stop: + docker-compose down + +clean: stop + git clean -dxf + +reset: clean + docker-compose down --volumes --remove-orphans diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..04a9a55 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,3 @@ +__pycache__/ +staticfiles/ +*.pyc diff --git a/backend/.gitignore b/backend/.gitignore index 6cc3cd8..5bcb7f2 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -2,7 +2,18 @@ *.sqlite3 __pycache__ -resources +right_tree/api/data/resources/*.cpg +right_tree/api/data/resources/*.dbf +right_tree/api/data/resources/*.pdf +right_tree/api/data/resources/*.prj +right_tree/api/data/resources/*.sbn +right_tree/api/data/resources/*.sbx +right_tree/api/data/resources/*.shp +right_tree/api/data/resources/*.shx +right_tree/api/data/resources/*.txt +right_tree/api/data/resources/*.xml +right_tree/api/data/resources/*.zip + right_tree/api/data/fixtures/plants.json right_tree/staticfiles -right_tree/api/data/generated_resources/* \ No newline at end of file +right_tree/api/data/generated_resources/* diff --git a/backend/Dockerfile b/backend/Dockerfile index 904ab50..feacc5f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,17 +1,11 @@ -FROM python:3.8-slim-bullseye +FROM python:3.11-slim-bullseye WORKDIR /app -RUN apt update && \ - apt install -y --no-install-recommends \ - gdal-bin \ - libxml2 libxml2-dev gettext \ - libxslt1-dev libjpeg-dev libpng-dev libpq-dev libgdal-dev \ - software-properties-common g++ \ - zlib1g-dev libgeos-dev libproj-dev \ - sqlite3 spatialite-bin libsqlite3-mod-spatialite \ - wkhtmltopdf && \ - apt clean +RUN apt update \ + && apt install -y --no-install-recommends gdal-bin \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean COPY ./requirements.txt /app/requirements.txt diff --git a/backend/requirements.txt b/backend/requirements.txt index cc5fe59..545b176 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,10 +1,10 @@ -Django==3.2.8 -psycopg2-binary>=2.8 -djangorestframework==3.12.4 -django-cors-headers==3.10.0 -openpyxl==3.0.9 -requests==2.26.0 +Django==3.2.17 +psycopg2-binary>=2.9.5 +djangorestframework==3.14.0 +django-cors-headers==3.13.0 +openpyxl==3.1.0 +requests==2.28.2 gunicorn==20.1.0 -pandas==1.3.4 +pandas==1.5.3 pdfkit==1.0.0 -PyPDF2==1.26.0 \ No newline at end of file +PyPDF2==1.28.6 diff --git a/backend/right_tree/api/data/fixtures/001_eco_regions.json b/backend/right_tree/api/data/fixtures/001_eco_regions.json index 7663a85..bef4072 100644 --- a/backend/right_tree/api/data/fixtures/001_eco_regions.json +++ b/backend/right_tree/api/data/fixtures/001_eco_regions.json @@ -3,7 +3,7 @@ "model": "api.ecologicalregion", "pk": 1, "fields": { - "name": "Aorrangi" + "name": "Aorangi" } }, { @@ -549,7 +549,7 @@ "model": "api.ecologicalregion", "pk": 79, "fields": { - "name": "Whatkatane" + "name": "Whakatane" } } -] \ No newline at end of file +] diff --git a/backend/right_tree/api/data/resources/plant_data.xlsx b/backend/right_tree/api/data/resources/plant_data.xlsx new file mode 100644 index 0000000..8e60638 Binary files /dev/null and b/backend/right_tree/api/data/resources/plant_data.xlsx differ diff --git a/backend/right_tree/api/management/commands/loadshapefiles.py b/backend/right_tree/api/management/commands/loadshapefiles.py index 0800034..dc77fa8 100644 --- a/backend/right_tree/api/management/commands/loadshapefiles.py +++ b/backend/right_tree/api/management/commands/loadshapefiles.py @@ -1,7 +1,9 @@ from django.core.management.base import BaseCommand from django.contrib.gis.utils import LayerMapping +from glob import iglob from pathlib import Path +from zipfile import ZipFile, is_zipfile import right_tree.api.data from right_tree.api.models import SoilLayer, EcologicalDistrictLayer, ChristchurchRegion @@ -32,15 +34,24 @@ christchurchregion_mapping = { 'geom': 'MULTIPOLYGON', } -# Shapefiles -soillayer_shp = Path(right_tree.api.data.__file__).resolve().parent / 'resources' / 'fundamental_soil_layers' / 'fundamental-soil-layers-new-zealand-soil-classification.shp' -ecologicaldistrictlayer_shp = Path(right_tree.api.data.__file__).resolve().parent / 'resources' / 'ecological_districts' / 'DOC_EcologicalDistricts_2021_08_02.shp' -christchurchregion_shp = Path(right_tree.api.data.__file__).resolve().parent / 'resources' / 'chch_zone' / 'Greater_Christchurch_Area.shp' +resources_path = Path(right_tree.api.data.__file__).resolve().parent / "resources" + +soillayer_shp = resources_path / "fundamental-soil-layers-new-zealand-soil-classification.shp" +ecologicaldistrictlayer_shp = resources_path / "Ecological_Districts.shp" +christchurchregion_shp = resources_path / "Greater_Christchurch_Area.shp" + class Command(BaseCommand): help = 'Ingests the shapefile data for ecological regions and soil layers.' def handle(self, *args, **options): + query = str(resources_path / "*.zip") + sources = [ZipFile(path) for path in iglob(query) if is_zipfile(path)] + + for zf in sources: + zf.extractall(resources_path) + zf.close() + self.stdout.write('Loading soil layers...') soil_lm = LayerMapping(SoilLayer, soillayer_shp, soillayer_mapping, transform=False) soil_lm.save(strict=True) diff --git a/backend/right_tree/settings.py b/backend/right_tree/settings.py index 1d9119b..75e1c6d 100644 --- a/backend/right_tree/settings.py +++ b/backend/right_tree/settings.py @@ -27,7 +27,7 @@ SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", 'django-insecure-5t05qc2&14xuot4lgs# # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.getenv('DJANGO_DEBUG_MODE', '') != 'False' -# os.getenv("ALLOWED_HOSTS", "").split(","), +# os.getenv("ALLOWED_HOSTS", "").split(","), ALLOWED_HOSTS = [BASE_URL, "localhost"] @@ -85,11 +85,11 @@ WSGI_APPLICATION = 'right_tree.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': os.getenv("RIGHTTREE_DB", "postgres"), - 'USER': os.getenv("RIGHTTREE_DB_USER", "postgres"), - 'PASSWORD': os.getenv("RIGHTTREE_DB_PASSWORD", "postgres"), - 'HOST': "postgres", - 'PORT': 5432, + 'NAME': os.getenv("DATABASE_NAME", "righttree"), + 'USER': os.getenv("DATABASE_USER", "righttree"), + 'PASSWORD': os.getenv("DATABASE_PASSWORD", "righttree"), + 'HOST': os.getenv("DATABASE_HOST", "postgres"), + 'PORT': int(os.getenv("DATABASE_PORT", 5432)), } } @@ -139,13 +139,6 @@ STATIC_ROOT = os.path.join(PROJECT_DIR, 'staticfiles') DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' -frontend_url = os.getenv('FRONTEND_BASE_URL', 'localhost:3000') - -CORS_ALLOWED_ORIGINS = [ - f"https://{frontend_url}", - f"http://{frontend_url}" -] - CORS_ALLOW_HEADERS = [ 'access-control-allow-origin' ] diff --git a/create_database.sql b/create_database.sql index 710c1b2..b943594 100644 --- a/create_database.sql +++ b/create_database.sql @@ -1,6 +1,11 @@ CREATE DATABASE righttree; +CREATE USER righttree; +ALTER USER righttree WITH PASSWORD 'righttree'; \c righttree CREATE EXTENSION IF NOT EXISTS postgis; GRANT ALL ON geometry_columns TO PUBLIC; GRANT ALL ON spatial_ref_sys TO PUBLIC; + +ALTER DATABASE righttree OWNER TO righttree; +GRANT ALL PRIVILEGES ON DATABASE righttree TO righttree; diff --git a/docker-compose.yaml b/docker-compose.yaml index d8146c0..1fb0964 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,53 +5,86 @@ volumes: name: righttree-postgres-data services: - backend: - restart: unless-stopped - build: - context: backend - dockerfile: Dockerfile - container_name: backend + + backend_migrate: + restart: on-failure + image: right-tree + container_name: backend_migrate depends_on: - postgres volumes: - ./backend:/app - env_file: .env - command: bash -c "./manage.py runserver 0.0.0.0:8000" + user: "$UID:$GID" + environment: + DATABASE_NAME: righttree + DATABASE_USER: righttree + DATABASE_PASSWORD: righttree + DATABASE_HOST: postgres + command: + - bash + - -c + - python manage.py makemigrations --noinput && python manage.py migrate --noinput + + backend: + restart: unless-stopped + image: right-tree + container_name: backend + depends_on: + - postgres + - backend_migrate + volumes: + - ./backend:/app + user: "$UID:$GID" + expose: + - "8000" + environment: + LINZ_API_KEY: 3aa06ba7bb2949a9b23ba2c8ac315e2b + DATABASE_NAME: righttree + DATABASE_USER: righttree + DATABASE_PASSWORD: righttree + DATABASE_HOST: postgres + command: + - gunicorn + - --reload + - --bind=0.0.0.0:8000 + - --timeout=300 + - right_tree.wsgi frontend: - image: node:16-alpine3.11 + image: node:16-bullseye restart: unless-stopped container_name: frontend volumes: - ./frontend:/app working_dir: /app + user: "$UID:$GID" ports: - - "3000:3000" - command: sh -c "npm install; npm start" + - 3000:3000 + command: + - npm + - start postgres: - image: postgis/postgis:13-3.0 + image: postgis/postgis:13-3.1 restart: unless-stopped container_name: postgres volumes: - righttree-postgres-data:/var/lib/postgresql/data - ./create_database.sql:/docker-entrypoint-initdb.d/create_database.sql ports: - - "5432:5432" + - 5432:5432 environment: - - POSTGRES_DB=${POSTGRES_DB} - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + POSTGRES_PASSWORD: postgres nginx: + image: nginx:1.23.3 + restart: unless-stopped container_name: nginx - image: nginx depends_on: - - postgres - backend - frontend volumes: - - ./nginx.conf:/etc/nginx/nginx.conf - - ./backend/right_tree/staticfiles:/etc/nginx/html/staticfiles + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ./backend/right_tree/staticfiles:/etc/nginx/html/staticfiles:ro ports: - "9000:80" diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 35766ef..469ee53 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -4898,6 +4898,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/ansi-html": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz", @@ -21357,9 +21368,11 @@ } }, "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -27174,6 +27187,13 @@ "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", "requires": { "type-fest": "^0.21.3" + }, + "dependencies": { + "type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==" + } } }, "ansi-html": { @@ -39934,9 +39954,11 @@ "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==" }, "type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==" + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "optional": true, + "peer": true }, "type-is": { "version": "1.6.18",