From 625c637f9940eccc75e2386f22679a32ab2d9de5 Mon Sep 17 00:00:00 2001 From: Dana Lambert Date: Thu, 4 Nov 2021 14:11:50 +1300 Subject: [PATCH] Add zones to plant fixture creation and extract spreadsheet constants --- .../commands/_spreadsheet_constants.py | 65 +++++++++++++++++++ .../commands/_spreadsheet_helpers.py | 40 +++++++++++- .../commands/createplantfixtures.py | 64 +++--------------- 3 files changed, 112 insertions(+), 57 deletions(-) create mode 100644 backend/right_tree/api/management/commands/_spreadsheet_constants.py diff --git a/backend/right_tree/api/management/commands/_spreadsheet_constants.py b/backend/right_tree/api/management/commands/_spreadsheet_constants.py new file mode 100644 index 0000000..f211ebc --- /dev/null +++ b/backend/right_tree/api/management/commands/_spreadsheet_constants.py @@ -0,0 +1,65 @@ +from pathlib import Path + +import right_tree.api.data +from ._spreadsheet_helpers import * +from right_tree.api.models import EcologicalRegion, SoilOrder, SoilVariant, ToleranceLevel, Zone + +# Mapping adjustments between the shapefile ecological regions and those in the spreadsheet +ECO_REGION_ADJUSTMENTS = { + "Whakatane": "Whatkatane", + "North West Nelson": "North-west Nelson", + "Aorangi": "Aorrangi", + "Mackenzie": "MacKenzie", + "Southland Hills": "Southland Foothills", + "Sounds Wellington": "Sounds-Wellington" +} + +# Relevant columns and information used to retrieve information from the spreadsheet +PLANT_COLS = { + 'SCIENTIFIC NAME': {"str": "name", "expected_type": str, "max_length": 50}, + 'MAX HT': {"str": "maxheight", "expected_type": float}, + 'SPACING': {"str": "spacing", "expected_type": float}, + 'COMMON NAME': {"str": "commonname", "expected_type": str, "null_allowed": True, "max_length": 200}, + 'SYNONYM': {"str": "synonym", "expected_type": str, "null_allowed": True, "max_length": 200}, + 'ECOLOGICAL REGION': {"str": "region", "expected_type": list, "model_name": "ecological_regions"}, + 'SOIL ORDER': {"str": "soilorder", "expected_type": list, "model_name": "soil_order"}, + 'Wet': {"str": "wet", "expected_type": list, "model_name": "soil_variants"}, + 'Mesic': {"str": "mesic", "expected_type": list, "model_name": "soil_variants"}, + 'Dry': {"str": "dry", "expected_type": list, "model_name": "soil_variants"}, + 'Water': {"str": "water", "expected_type": int, "model_name": "water_tolerance"}, + 'Drought': {"str": "drought", "expected_type": int, "model_name": "drought_tolerance"}, + 'Frost': {"str": "frost", "expected_type": int, "model_name": "frost_tolerance"}, + 'Salinity': {"str": "salinity", "expected_type": int, "model_name": "salinity_tolerance"}, + 'ES': {"str": "purpose", "expected_type": str, "null_allowed": True}, + 'STAGE': {"str": "stage", "expected_type": int}, + 'GrowthForm': {"str": "growthform", "expected_type": str, "model_name": "growth_form", "null_allowed": True, "max_length": 50} +} + +# Spreadsheet constants +SPREADSHEET_FILENAME = 'plant_data.xlsx' +DATA_START_COL = 3 +DATA_START_ROW = 6 +INFO_HEADER_ROW = 4 + +# Site col and row contants +SITE_DATA_START_COL = 25 +SITE_DATA_STOP_COL = 157 +HABITAT_ROW = 2 +ZONE_NAME_ROW = 3 +ZONE_VARIANT_ROW = 4 +ZONE_REFINED_VARIANT_ROW = 5 + +# Data directory path +DATA_DIR_PATH = Path(right_tree.api.data.__file__).resolve().parent + +# Mappings between values in the spreadsheet and primary key values in the database +ECO_REGION_PK_MAPPING = get_pk_mapping(EcologicalRegion) +SOIL_ORDER_PK_MAPPING = get_pk_mapping(SoilOrder) +SOIL_VARIANT_PK_MAPPING = get_pk_mapping(SoilVariant) +TOLERANCE_PK_MAPPING = get_pk_mapping(ToleranceLevel, "level") +ZONE_PK_MAPPING = get_zone_pk_mapping(Zone) + +# Spreadsheet and corresponding value to column index mappings +SPREADSHEET = get_spreadsheet(DATA_DIR_PATH, SPREADSHEET_FILENAME) +INFO_COL_INDEXES = get_col_mappings(SPREADSHEET, DATA_START_COL, SITE_DATA_START_COL-1, INFO_HEADER_ROW) +ZONE_COL_INDEXES = get_zone_col_mappings(SPREADSHEET, SITE_DATA_START_COL, SITE_DATA_STOP_COL, ZONE_NAME_ROW, ZONE_REFINED_VARIANT_ROW) diff --git a/backend/right_tree/api/management/commands/_spreadsheet_helpers.py b/backend/right_tree/api/management/commands/_spreadsheet_helpers.py index 719ceb5..aadc167 100644 --- a/backend/right_tree/api/management/commands/_spreadsheet_helpers.py +++ b/backend/right_tree/api/management/commands/_spreadsheet_helpers.py @@ -1,4 +1,5 @@ from openpyxl import load_workbook +from right_tree.api.models import Zone def get_pk_mapping(object, mapping_key="name"): @@ -11,17 +12,52 @@ def get_pk_mapping(object, mapping_key="name"): return pk_mapping -def get_col_mappings(sheet, start_col, row_index): +def get_zone_pk_mapping(zone_model): + """ Maps the string instance (unique) of a zone to its corresponding primary key. + """ + zone_pk_mapping = {} + for instance in zone_model.objects.all(): + zone_pk_mapping[str(instance)] = instance.pk + + return zone_pk_mapping + + +def get_col_mappings(sheet, start_col, stop_col, row_index): """ Returns a dictionary that maps a spreadsheet cell value to a corresponding column index. """ col_mappings = {} - for row in sheet.iter_rows(min_col=start_col, min_row=row_index, max_row=row_index, values_only=True): + for row in sheet.iter_rows(min_col=start_col, max_col=stop_col, min_row=row_index, max_row=row_index, values_only=True): for i, col_name in enumerate(row): col_mappings[col_name] = i return col_mappings +def get_zone_col_mappings(sheet, start_col, stop_col, start_row, stop_row): + """ Returns a dictionary that maps a spreadsheet zone string object to a column index. + """ + zone_col_mappings = {} + current_zone = current_variant = current_refined_variant = None + for i, col in enumerate(sheet.iter_cols(min_col=start_col, max_col=stop_col, min_row=start_row, max_row=stop_row, values_only=True)): + zone_name, zone_variant, zone_refined_variant = col + + if zone_name is not None: + current_zone = zone_name + current_variant = current_refined_variant = None + + if zone_variant is not None: + current_variant = zone_variant + current_refined_variant = None + + current_refined_variant = zone_refined_variant if zone_refined_variant is not None else current_refined_variant + + zone_obj = Zone(name=current_zone, variant=current_variant, + refined_variant=current_refined_variant) + zone_col_mappings[str(zone_obj)] = i + start_col + + return zone_col_mappings + + def get_pk_list_from_str(values_str, pk_mapping, fixes={}): """ Given a list of comma separated values from the spreadsheet. Returns a list of primary keys that correspond to the relevant values with any given mapping fixes applied. diff --git a/backend/right_tree/api/management/commands/createplantfixtures.py b/backend/right_tree/api/management/commands/createplantfixtures.py index 181d1ff..f34d15e 100644 --- a/backend/right_tree/api/management/commands/createplantfixtures.py +++ b/backend/right_tree/api/management/commands/createplantfixtures.py @@ -1,62 +1,8 @@ from django.core.management.base import BaseCommand import json -from pathlib import Path -import right_tree.api.data -from ._spreadsheet_helpers import * -from right_tree.api.models import EcologicalRegion, SoilOrder, SoilVariant, ToleranceLevel - -# Mapping adjustments between the shapefile ecological regions and those in the spreadsheet -ECO_REGION_ADJUSTMENTS = { - "Whakatane": "Whatkatane", - "North West Nelson": "North-west Nelson", - "Aorangi": "Aorrangi", - "Mackenzie": "MacKenzie", - "Southland Hills": "Southland Foothills", - "Sounds Wellington": "Sounds-Wellington" -} - -# Relevant columns and information used to retrieve information from the spreadsheet -PLANT_COLS = { - 'SCIENTIFIC NAME': {"str": "name", "expected_type": str, "max_length": 50}, - 'MAX HT': {"str": "maxheight", "expected_type": float}, - 'SPACING': {"str": "spacing", "expected_type": float}, - 'COMMON NAME': {"str": "commonname", "expected_type": str, "null_allowed": True, "max_length": 200}, - 'SYNONYM': {"str": "synonym", "expected_type": str, "null_allowed": True, "max_length": 200}, - 'ECOLOGICAL REGION': {"str": "region", "expected_type": list, "model_name": "ecological_regions"}, - 'SOIL ORDER': {"str": "soilorder", "expected_type": list, "model_name": "soil_order"}, - 'Wet': {"str": "wet", "expected_type": list, "model_name": "soil_variants"}, - 'Mesic': {"str": "mesic", "expected_type": list, "model_name": "soil_variants"}, - 'Dry': {"str": "dry", "expected_type": list, "model_name": "soil_variants"}, - 'Water': {"str": "water", "expected_type": int, "model_name": "water_tolerance"}, - 'Drought': {"str": "drought", "expected_type": int, "model_name": "drought_tolerance"}, - 'Frost': {"str": "frost", "expected_type": int, "model_name": "frost_tolerance"}, - 'Salinity': {"str": "salinity", "expected_type": int, "model_name": "salinity_tolerance"}, - 'ES': {"str": "purpose", "expected_type": str, "null_allowed": True}, - 'STAGE': {"str": "stage", "expected_type": int}, - 'GrowthForm': {"str": "growthform", "expected_type": str, "model_name": "growth_form", "null_allowed": True, "max_length": 50} -} - -# Spreadsheet constants -SPREADSHEET_FILENAME = 'plant_data.xlsx' -DATA_START_COL = 3 -DATA_START_ROW = 6 -INFO_HEADER_ROW = 4 - -# Data directory path -DATA_DIR_PATH = Path(right_tree.api.data.__file__).resolve().parent - -# Mappings between values in the spreadsheet and primary key values in the database -ECO_REGION_PK_MAPPING = get_pk_mapping(EcologicalRegion) -SOIL_ORDER_PK_MAPPING = get_pk_mapping(SoilOrder) -SOIL_VARIANT_PK_MAPPING = get_pk_mapping(SoilVariant) -TOLERANCE_PK_MAPPING = get_pk_mapping(ToleranceLevel, "level") - -# Spreadsheet and corresponding value to column index mappings -SPREADSHEET = get_spreadsheet(DATA_DIR_PATH, SPREADSHEET_FILENAME) -INFO_COL_INDEXES = get_col_mappings( - SPREADSHEET, DATA_START_COL, INFO_HEADER_ROW) +from ._spreadsheet_constants import * # Template for the plant json to add as an entry for the fixtures PLANT_JSON_TEMPLATE = { @@ -136,6 +82,14 @@ def get_plant_json_from_row(row_data): return {} + # Add zone spreadsheet data to the plant json fields. + zone_list = [] + for zone, zone_index in ZONE_COL_INDEXES.items(): + if row_data[zone_index-DATA_START_COL] != None: + zone_list.append(ZONE_PK_MAPPING[zone]) + plant_json_fields['zones'] = zone_list + + # Create a plant json object using the defined template plant_json = PLANT_JSON_TEMPLATE.copy() plant_json["fields"] = plant_json_fields