Mirri 5.1.2

This commit is contained in:
2023-06-10 14:49:33 +01:00
parent e2278fd509
commit 322ed203d8
102 changed files with 1978 additions and 3356 deletions
Binary file not shown.
Binary file not shown.
View File
Binary file not shown.
Binary file not shown.
Binary file not shown.
+3
View File
@@ -0,0 +1,3 @@
from .error import Entity, Error
from .error_message import ErrorMessage
from .error_log import ErrorLog
+119
View File
@@ -0,0 +1,119 @@
from typing import Optional
from .error_message import ErrorMessage
class Entity():
"""Entity information
Args:
acronym: acronym of the entity. Must be a 3-characters captalized string
"""
def __init__(self, acronym: str) -> None:
self.acronym = acronym
def __str__(self) -> str:
return f"Entity {self.acronym}: {self.name}"
@property
def _acronyms(self) -> list:
return [
func
for func in dir(self)
if func.isupper() and
callable(getattr(self, func)) and
not func.startswith("__")
]
@property
def _names(self) -> dict:
return {acr: getattr(self, acr)() for acr in self._acronyms}
@property
def name(self) -> str:
try:
return self._names[self.acronym]
except KeyError:
raise KeyError(f'Unknown acronym {self.acronym}.')
@property
def acronym(self) -> str:
return self._acronym
@acronym.setter
def acronym(self, acronym: str) -> None:
self._acronym = acronym
def EFS(self) -> str:
return 'Excel File Structure'
def GMD(self) -> str:
return 'Growth Media'
def GOD(self) -> str:
return 'Geographic Origin'
def LID(self) -> str:
return 'Literature'
def STD(self) -> str:
return 'Strains'
def GID(self) -> str:
return 'Genomic Information'
def OTD(self) -> str:
return 'Ontobiotope'
def UCT(self) -> str:
return 'Uncategorized'
class Error():
"""Error information
Args:
message (str): Error message
entity (Entity, optional): Entity related to the error. If None will default to Uncategorized. Defaults to None.
data (str, optional): Data used for sorting the messages. Defaults to None.
"""
def __init__(self, code: str, pk: Optional[str] = None, data: Optional[str] = None) -> None:
self.code = code.upper()
self.pk = pk
self.data = data
def __str__(self):
return f"Error {self._code}: {self.message}"
@property
def code(self) -> str:
return self._code
@code.setter
def code(self, code: str) -> None:
self._code = code.upper()
@property
def pk(self) -> Optional[str]:
return self._pk
@pk.setter
def pk(self, pk: Optional[str] = None) -> None:
self._pk = pk
@property
def data(self) -> Optional[str]:
return self._data
@data.setter
def data(self, data: Optional[str]):
self._data = data
@property
def entity(self) -> Entity:
return Entity(self.code[:3])
@property
def message(self) -> str:
return ErrorMessage(self.code, self.pk, self.data).message
+77
View File
@@ -0,0 +1,77 @@
from typing import Optional, Union
from datetime import datetime
from .error import Error
class ErrorLog():
def __init__(self, input_filename: str, cc: Optional[str] = None, date: Optional[Union[str, datetime]] = None, limit: int = 100):
"""
Logger for Error instances.
Args:
input_filename (str): name of the file to be logged
cc (str, optional): name of the curator. Defaults to None.
date (str, optional): date (e.g. created, last modified) associated with the file. Useful for versioning. Defaults to None.
limit (int, optional): limit of errors to print to the report. Defaults to 100.
"""
self._input_filename = input_filename
self._cc = cc
self._date = date
self._errors = {}
self.limit = limit
self._counter = 0
def __str__(self) -> str:
output = f"""Error Log for file {self._input_filename}\nENTITY | CODE | MESSAGE"""
for acronym, error_list in self.get_errors().items():
for error in error_list:
output += f"\n{acronym:6} | {error.code:6} | {error.message[:100]}"
return output
@property
def input_filename(self) -> str:
return self._input_filename
@input_filename.setter
def input_filename(self, input_filename: str) -> None:
self._input_filename = input_filename
@property
def cc(self) -> Optional[str]:
return self._cc
@cc.setter
def cc(self, cc: Optional[str]) -> None:
self._cc = cc
@property
def date(self) -> Optional[Union[str, datetime]]:
return self._date
@date.setter
def date(self, date: Optional[Union[str, datetime]] = None) -> None:
if isinstance(date, str):
self._date = datetime.strptime(date, r'%d-%m-%Y')
else:
self._date = date
def get_errors(self) -> dict:
"""
Get all errors
Returns:
dict: Error intances grouped by entity acronym.
"""
return self._errors
def add_error(self, error: Error) -> None:
"""
Add an error.
Args:
error (Error): Error instance.
"""
if error.entity.acronym not in self._errors:
self._errors[error.entity.acronym] = [error]
else:
self._errors[error.entity.acronym].append(error)
+552
View File
@@ -0,0 +1,552 @@
from typing import Optional
class ErrorMessage():
"""Error message
Args:
code (str): Error code.
pk (str | optional): The instance's primary key that triggered the error. Defaults to None.
value (str | optional): The instance's value that triggered the error. Defaults to None.
"""
def __init__(self, code: str, pk: Optional[str] = None, value: Optional[str] = None):
self.code = code.upper()
self.pk = pk
self.value = value
@property
def _codes(self) -> list:
return [
func
for func in dir(self)
if func.isupper() and
callable(getattr(self, func)) and
not func.startswith("__")
]
@property
def _messages(self) -> dict:
return {code: getattr(self, code) for code in self._codes}
@property
def message(self) -> str:
if not self._validate_code():
raise ValueError(f"{self.code} not found")
return self._messages[self.code]()
@property
def code(self) -> str:
return self._code
@code.setter
def code(self, code: str) -> None:
self._code = code.upper()
def _validate_code(self) -> bool:
return self.code in self._codes
@property
def pk(self) -> str:
return self._pk
@pk.setter
def pk(self, pk: str) -> None:
self._pk = pk
@property
def value(self) -> str:
return self._value
@value.setter
def value(self, value: str) -> None:
self._value = value
"""
Excel File Structure Error Codes
"""
def EXL00(self):
return f"The provided file '{self.pk}' is not an excel(xlsx) file"
def EFS01(self):
return "The 'Growth media' sheet is missing. Please check the provided excel template."
def EFS02(self):
return "The 'Geographic origin' sheet is missing. Please check the provided excel template."
def EFS03(self):
return "The 'Literature' sheet is missing. Please check the provided excel template."
def EFS04(self):
return "The 'Sexual state' sheet is missing. Please check the provided excel template."
def EFS05(self):
return "The 'Strains' sheet is missing. Please check the provided excel template."
def EFS06(self):
return "The 'Ontobiotope' sheet is missing. Please check the provided excel template."
def EFS07(self):
return "The 'Markers' sheet is missing. Please check the provided excel template."
def EFS08(self):
return "The 'Genomic information' sheet is missing. Please check the provided excel template."
def EFS09(self):
return "The 'Version' sheet is missing. Please check the provided excel template."
"""
Growth Media Error Codes
"""
def GMD01(self):
return "The 'Acronym' column is a mandatory field in the Growth Media sheet."
def GMD02(self):
return "The 'Acronym' column is empty or has missing values."
def GMD03(self):
return "The 'Description' column is a mandatory field in the Growth Media sheet. The column can not be empty."
def GMD04(self):
return f"The 'Description' for growth media with Acronym {self.pk} is missing."
"""
Geographic Origin Error Codes
"""
def GOD01(self):
return "The 'ID' column is a mandatory field in the Geographic Origin sheet."
def GOD02(self):
return "The 'ID' column is empty or has missing values."
def GOD03(self):
return "The 'Country' column is a mandatory field in the Geographic Origin sheet. The column can not be empty."
def GOD04(self):
return f"The 'Country' for geographic origin with ID {self.pk} is missing."
def GOD05(self):
return f"The 'Country' for geographic origin with ID {self.pk} is incorrect."
def GOD06(self):
return f"The 'Locality' column is a mandatory field in the Geographic Origin sheet. The column can not be empty."
def GOD07(self):
return f"The 'Locality' for geographic origin with ID {self.pk} is missing."
"""
Literature Error Codes
"""
def LID01(self):
return "The 'ID' column is a mandatory field in the Literature sheet."
def LID02(self):
return "The 'ID' column empty or missing values."
def LID03(self):
return "The 'Full reference' column is a mandatory field in the Literature sheet. The column can not be empty."
#def LID04(self):
#return f"The 'Full reference' for literature with ID {self.pk} is missing."
def LID05(self):
return "The 'Authors' column is a mandatory field in the Literature sheet. The column can not be empty."
#def LID06(self):
#return f"The 'Authors' for literature with ID {self.pk} is missing."
def LID07(self):
return "The 'Title' column is a mandatory field in the Literature sheet. The column can not be empty."
#def LID08(self):
#return f"The 'Title' for literature with ID {self.pk} is missing."
def LID09(self):
return "The 'Journal' column is a mandatory field in the Literature sheet. The column can not be empty."
#def LID10(self):
#return f"The 'Journal' for literature with ID {self.pk} is missing."
def LID11(self):
return "The 'Year' column is a mandatory field in the Literature sheet. The column can not be empty."
def LID12(self,):
return f"The 'Year' for literature with ID {self.pk} is missing."
def LID13(self):
return "The 'Volume' column is a mandatory field in the Literature sheet. The column can not be empty."
def LID14(self):
return f"The 'Volume' for literature with ID {self.pk} is missing."
def LID15(self):
return "The 'First page' column is a mandatory field. The column can not be empty."
def LID16(self):
return f"The 'First page' for literature with ID {self.pk} is missing."
def LID17(self):
return( f"There are four types of ways to fill in the 'Literature' sheet.",
"1st- Columns 'ID' and 'DOI' must be obrigatory.",
"2nd-Columns 'ID' and 'PMID' are obrigatory.",
"3rd-Columns 'ID' and 'Full reference' are obrigatory.",
"In the alternative of these three types of forms not being filled in, we have:",
"4th-Columns 'ID', 'Authors', 'Title', 'Journal', 'Year', 'Volume', 'First page'.")
def LID18(self):
return "The 'PMID' column is a mandatory field. The column can not be empty."
#def LID19(self):
#return f"PMID for literature with ID {self.pk} is missing."
def LID20(self):
return "The 'DOI' column is a mandatory field. The column can not be empty."
#def LID21(self):
#return f"DOI for literature with ID {self.pk} is missing."
"""
Strains Error Codes
"""
def STD01(self):
return "The 'accessionNumber' column is a mandatory field in the Strains sheet."
def STD02(self):
return "The 'accessionNumber' column is empty or has missing values."
def STD03(self):
return f"The 'accessionNumber' must be unique. The '{self.value}' is repeated."
def STD04(self):
return (f"The 'accessionNumber' {self.pk} is not according to the specification."
" The value must be of the format '<Sequence of characters> <sequence of characters>'.")
def STD05(self):
return f"The 'useRestrictions' column is a mandatory field in the Strains Sheet. The column can not be empty."
def STD06(self):
return f"The 'useRestrictions' for strain with accessionNumber {self.pk} is missing."
def STD07(self):
return (f"The 'useRestrictions' for strain with accessionNumber {self.pk} is not according to the specification."
f" Your value is {self.value} and the accepted values are 1, 2, 3.")
def STD08(self):
return f"The 'nagoyaConditions' column is a mandatory field in the Strains Sheet. The column can not be empty."
def STD09(self):
return f"The 'nagoyaConditions' for strain with accessionNumber {self.pk} is missing."
def STD10(self):
return (f"The 'nagoyaConditions' for strain with accessionNumber {self.pk} is not according to the specification."
f" Your value is {self.value} and the accepted values are 1, 2, 3.")
def STD11(self):
return (f"The 'registeredCollection' for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 1, 2, 3.")
def STD12(self):
return "The 'riskGroup' column is a mandatory field in the Strains Sheet. The column can not be empty."
def STD13(self):
return f"The 'riskGroup' for strain with accessionNumber {self.pk} is missing."
def STD14(self):
return (f"The 'riskGroup' for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 1, 2, 3, 4.")
def STD15(self):
return (f"The 'dualUse' for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 1, 2.")
def STD16(self):
return (f"The “euQuarantine” for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 1, 2.")
def STD17(self):
return f"The 'organismType' column is a mandatory field in the Strains Sheet. The column can not be empty."
def STD18(self):
return f"The 'organismType' for strain with accessionNumber {self.pk} is missing."
def STD19(self):
return (f"The 'organismType' for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 'Algae', 'Archaea', 'Bacteria', 'Cyanobacteria', "
"'Filamentous Fungi', 'Phage', 'Plasmid', 'Virus', 'Yeast', 1, 2, 3, 4, 5, 6, 7, 8, 9.")
def STD20(self):
return f"The 'speciesName' column is a mandatory field in the Strains Sheet. The column can not be empty."
def STD21(self):
return f"The 'speciesName' for strain with accessionNumber {self.pk} is missing."
def STD22(self):
return f"The 'speciesName' for strain with accessionNumber {self.pk} is incorrect."
def STD23(self):
return (f"The 'hybrid' for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 1, 2.")
def STD24(self):
return (f"The 'depositHistory' for strain with accessionNumber {self.pk} is incorrect."
"The field includes entries separated by '<' meaning 'received from'."
"Entries may include persons or CCs. The name of the CC should be followed by"
"the month, when available, and year of the acquisition. Between parentheses,"
"the strain designation or CC numbers and/or a name can also be entered when "
"a name change has occurred.")
def STD25(self):
return (f"The 'depositDate' for strain with accessionNumber {self.pk} is incorrect."
" The allowed formats are 'YYYY-MM-DD', 'YYYYMMDD', 'YYYYMM', and 'YYYY'.")
def STD26(self):
return (f"The 'accessionDate' for strain with accessionNumber {self.pk} is incorrect."
" The allowed formats are 'YYYY-MM-DD', 'YYYYMMDD', 'YYYYMM', and 'YYYY'.")
def STD27(self):
return (f"The 'collectionDate' for strain with accessionNumber {self.pk} is incorrect."
" The allowed formats are 'YYYY-MM-DD', 'YYYYMMDD', 'YYYYMM', and 'YYYY'.")
def STD28(self):
return (f"The 'isolationDate' for strain with accessionNumber {self.pk} is incorrect."
" The allowed formats are 'YYYY-MM-DD', 'YYYYMMDD', 'YYYYMM', and 'YYYY'.")
def STD29(self):
return (f"The 'temperatureGrowthRange' for strain with accessionNumber {self.pk} is incorrect."
" It must have two decimal numbers separated by ','")
def STD30(self):
return f"The 'temperatureGrowthRange' column is a mandatory field in the Strains Sheet. The column can not be empty."
def STD31(self):
return f"The 'temperatureGrowthRange' for strain with accessionNumber {self.pk} is missing."
def STD32(self):
return (f"The 'temperatureGrowthRange' for strain with accessionNumber {self.pk} is incorrect."
" It must have two decimal numbers separated by ','.")
def STD33(self):
return ("The 'recommendedTemperature' column is a mandatory field in the Strains Sheet. The column can not be empty.")
def STD34(self):
return f"The 'recommendedTemperature' for strain with accessionNumber {self.pk} is missing."
def STD35(self):
return f"The value of 'recommendedTemperature' for strain with accessionNumber {self.pk} is not in the Growth Media Sheet."
def STD36(self):
return f"The 'supplyForms' column is a mandatory field in the Strains Sheet. The column can not be empty."
def STD37(self):
return f"The 'supplyForms' for strain with accessionNumber {self.pk} is missing."
def STD38(self):
return f"The value of 'supplyForms' for strain with accessionNumber {self.pk} is not in the Forms of Supply Sheet."
def STD39(self):
return (f"The 'geographicCoordinates' column for strain with accessionNumber {self.pk} is incorrect."
"The allowed formats are two, three or four decimal numbers separated by ','. Moreover, the first number must be."
"between [-90, 90], the second between [-180, 180], and the third and fourth refers to the precision and altitude, defined by decimal numbers."
"Put a question mark for lack of precision or altitude when one of them is missing. Leave the values blank when both are missing. ")
def STD40(self):
return (f"The 'country' column for strain with accessionNumber {self.pk} is incorrect."
"The allowed formats are one decimal number between [-200, 8000].")
def STD54(self):
return (f"The 'country'column is a mandatory field in the Strains Sheet. The column can not be empty.")
def STD55(self):
return (f"The 'country' for strain with accessionNumber {self.pk} is missing.")
def STD41(self):
return f"The value of 'ontobiotopeTerms' for strain with accessionNumber {self.pk} is not in the Ontobiotope Sheet."
def STD42(self):
return (f"The 'gmo' for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 1, 2")
def STD43(self):
return (f"The 'sexualState' for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 'Mata', 'Matalpha', 'Mata/Matalpha', "
"'Matb', 'Mata/Matb', 'MTLa', 'MTLalpha', 'MTLa/MTLalpha', 'MAT1-1', 'MAT1-2', 'MAT1', 'MAT2', 'MT+', 'MT-'")
def STD44(self):
return (f"The 'ploidy' for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 0, 1, 2, 3, 4, 9")
def STD45(self):
msg = f"At least one of the values '{self.value}' of the literature field for strain {self.pk} are not in the literature sheet. "
msg += "If the those values are Pubmed ids or DOIs, please ignore this messsage"
return msg
def STD46(self):
return (f"The 'geographicOrigin' for strain with accessionNumber {self.pk} is not according to specification."
f"The 'geographicOrigin' column must consist of the ID's associated with the Geographic origin sheet.")
def STD47(self):
return "The 'country' column is a mandatory field in the Strains sheet."
def STD48(self):
return "The 'country' column is empty or has missing values."
def STD49(self):
return (f"The “qps” for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 1, 2.")
def STD50(self):
return (f"The “axenicCulture” for strain with accessionNumber {self.pk} is not according to specification."
f" Your value is {self.value} and the accepted values are 'Axenic', 'Not axenic'.")
def STD51(self):
return f"The 'mirriAccessionNumber' must be unique. The '{self.pk}' is repeated."
def STD52(self):
return (f"The 'mirriAccessionNumber' for strain with accessionNumber {self.pk} is incorrect."
" It must have the expression MIRRI followed by 7 digits")
def STD53(self):
return (f"The 'siteLinks' for strain with accessionNumber {self.pk} is incorrect."
" The displayed expression it should be composed of: site name ';' website url." )
def STD56(self):
return (f"The 'siteLinks' for strain with accessionNumber {self.pk} is incorrect."
" The url must be valid. " )
def STD57(self):
return (f"The 'country' for strain with accessionNumber {self.pk} is incorrect."
"This information must be expressed by using the ISO-3166 standard for country"
"codes. The preferred set is ISO 3166-1 alpha-2 (two letters code), but ISO 3166-"
"1 alpha-3 (three letters code) is also accepted. Former country codes must"
"follow standards part three ISO 3166-3 (four letters code). Only one code can"
"be included." )
def STD58(self):
return (f"The 'mtaFile' for strain with accessionNumber {self.pk} is incorrect."
" The url must be valid. " )
def STD59(self):
return (f"The 'absFile' for strain with accessionNumber {self.pk} is incorrect."
"The displayed expression it should be composed of: name ';' website url."
"When only one URL is provided, the title may be omitted. In this case, the URL"
"will be shown in clear to users." )
def STD60(self):
return (f"The 'absFile' for strain with accessionNumber {self.pk} is incorrect."
" The url must be valid. ")
def STD61(self):
return (f"The 'sequenceLiterature' for strain with accessionNumber {self.pk} is incorrect."
"Numeric identifiers separated by a semicolon ';'.")
def STD62(self):
return (f"The 'plasmidCollections' for strain with accessionNumber {self.pk} is incorrect."
"It should include the name of the plasmid followed by the CC number in"
"parentheses. More than one plasmid can be reported, separated by ';'. "
"Plasmid names should be provided as free text."
"CC numbers should be composed by the CC acronym followed by a number"
"separated by a space'. Numeric identifiers separated by a semicolon ';'.")
def STD63(self):
return (f"The 'otherCollectionNumbers' for strain with accessionNumber {self.pk} is incorrect."
" The value must be of the format '<Sequence of characters> <sequence of characters>'.")
def STD64(self):
return (f"The 'type' for strain with accessionNumber {self.pk} is incorrect."
f"Your value is {self.value} and the accepted values are 1, 2.")
def STD64(self):
return (f"The 'status' for strain with accessionNumber {self.pk} is incorrect."
"When the type equal 2 the status must contain the type,"
"if the type equal 2 the status must have a null value.")
def STD65(self):
return (f"The 'status' for strain with accessionNumber {self.pk} is incorrect."
"The structure should be 'type of <character string>.")
def STD68(self):
return (f"The 'geographicOrigin'column is a mandatory field in the Strains Sheet. The column can not be empty.")
def STD69(self):
return (f"The 'geographicOrigin' for strain with accessionNumber {self.pk} is missing.")
"""
Genomic Information Error Codes
"""
def GID01(self):
return f"The 'Strain accessionNumber' (Strain AN) column is a mandatory field in the Genomic Information Sheet."
def GID02(self):
return f"The 'Strain accessionNumber' (Strain AN) column is empty or has missing values."
def GID03(self):
return f"The value of 'Strain accessionNumber' (Strain AN) {self.value} is not in the Strains sheet."
def GID04(self):
return f"The 'Marker' column is a mandatory field in the Genomic Information Sheet. The column can not be empty."
def GID05(self):
return f"The 'Marker' for genomic information with Strain AN {self.pk} is missing."
def GID06(self):
return f"The value of 'Marker' {self.value} is not in the Markers sheet."
def GID07(self):
return f"The 'INSDC AN' column is a mandatory field in the Genomic Information Sheet. The column can not be empty."
def GID08(self):
return f"The 'INSDC AN' for genomic information with Strain AN {self.pk} is missing."
def GID09(self):
return f"The 'INSDC AN' for genomic information with Strain AN {self.pk} is incorrect."
def GID10(self):
return (f"The 'Sequence' for genomic information with Strain AN {self.pk} is incorrect."
" It must be a sequence of 'G', 'T', 'A', 'C' characteres of any length and without white spaces.")
def GID11(self):
return (f"The 'Sequence' for genomic information with Strain AN {self.pk} is incorrect."
"An INSDC accession number is an alphanumeric"
"code made by a fixed number of letters followed by a fixed number of digits,"
"without any separation. For sequences, the code is currently made of two"
"letters followed by six numbers.")
"""
Ontobiotope Error Codes
"""
def CTR01(self):
return "The 'Version' columns is a mandatory field in the Version Sheet."
def CTR02(self):
return "The 'Version' columns is empty or has missing values."
def CTR03(self):
return "The 'Date' columns is a mandatory field in the Control Sheet."
def CTR04(self):
return "The 'Date' columns is empty or has missing values."
def CTR05(self):
return f"The version {self.value} is the only one to be used."
"""
Ontobiotope Error Codes
"""
def OTD01(self):
return "The 'ID' columns is a mandatory field in the Ontobiotope Sheet."
def OTD02(self):
return "The 'ID' columns is empty or has missing values."
#def OTD03(self):
return "The 'Name' columns is a mandatory field in the Ontobiotope Sheet. The column can not be empty."
#def OTD04(self):
return f"The 'Name' for ontobiotope with ID {self.pk} is missing."
+671
View File
@@ -0,0 +1,671 @@
import re
from pathlib import Path
from io import BytesIO
from zipfile import BadZipfile
from datetime import datetime
from calendar import monthrange
import requests
from openpyxl import load_workbook
import pycountry
from mirri.io.parsers.excel import workbook_sheet_reader, get_all_cell_data_from_sheet
from mirri.validation.error_logging import ErrorLog, Error
from mirri.validation.tags import (CHOICES, COLUMNS, COORDINATES, CROSSREF, CROSSREF_NAME, DATE,
ERROR_CODE, FIELD, MANDATORY, MATCH,
MISSING, MULTIPLE, NAGOYA, REGEXP, ROW_VALIDATION, SEPARATOR, TAXON,
TYPE, UNIQUE, VALIDATION, VALUES, BIBLIO, DOMINIO,URL_DOMINIO, ISO, URL_TITLE,JUST_URL,TITLE,
HISTORY,NAGOYA1, VERSION)
from mirri.settings import LOCATIONS, SUBTAXAS
from mirri.settings_v1 import LOCATIONS, SUBTAXAS
from mirri.validation.validation_conf_12052023 import version_config
def validate_mirri_excel(fhand, version="", date=""):
configuration = version_config.get(version)
if configuration is None:
raise NotImplementedError("Unsupported version: " + version)
configuration["date"] = date or configuration.get("date")
if configuration["date"] != "12/05/2023":
raise ValueError("Invalid date. Expected: 12/05/2023")
return validate_excel(fhand, configuration)
def version(value , validation_conf=None):
if value is None:
return True
try:
for version in version_config:
if value == version :
return True
except:
return False
def validate_country_code(value,validation_conf=None):
if value is None:
return True
try:
if pycountry.countries.get(alpha_2=value) or pycountry.countries.get(alpha_3=value) or pycountry.historic_countries.get(alpha_4 = value):
return True
except:
return False
def validate_excel(fhand, configuration):
validation_conf = configuration['sheet_schema']
cross_ref_conf = configuration['cross_ref_conf']
in_memory_sheet_conf = configuration['keep_sheets_in_memory']
excel_name = Path(fhand.name).stem
error_log = ErrorLog(excel_name)
try:
workbook = load_workbook(filename=BytesIO(
fhand.read()), read_only=True, data_only=True)
except (BadZipfile, IOError):
error = Error('EXL00', fhand.name, fhand.name)
error_log.add_error(error)
return error_log
# excel structure errors
structure_errors = list(validate_excel_structure(workbook, validation_conf))
if structure_errors:
for error in structure_errors:
error = Error(error[ERROR_CODE], pk=error['id'],
data=error['value'])
error_log.add_error(error)
return error_log
crossrefs = get_all_crossrefs(workbook, cross_ref_conf)
in_memory_sheets = get_all_in_memory_sheet(workbook, in_memory_sheet_conf)
content_errors = validate_content(workbook, validation_conf,
crossrefs, in_memory_sheets)
for error in content_errors:
# if error[ERROR_CODE] == 'STD43':
# continue
error = Error(error[ERROR_CODE], pk=error['id'], data=error['value'])
error_log.add_error(error)
return error_log
def validate_excel_structure(workbook, validation_conf):
for sheet_name, sheet_conf in validation_conf.items():
mandatory = sheet_conf.get(VALIDATION, {}).get(TYPE, None)
mandatory = mandatory == MANDATORY
error_code = sheet_conf.get(VALIDATION, {}).get(ERROR_CODE, False)
try:
sheet = workbook[sheet_name]
except KeyError:
sheet = None
if sheet is None:
if mandatory:
yield {'id': None, 'sheet': sheet_name, 'field': None,
'error_code': error_code, 'value': None}
continue
headers = _get_sheet_headers(sheet)
for column in sheet_conf.get(COLUMNS):
field = column[FIELD]
for step in column.get(VALIDATION, []):
if step[TYPE] == MANDATORY and field not in headers:
yield {'id': None, 'sheet': sheet_name, 'field': field,
'error_code': step[ERROR_CODE], 'value': None}
def _get_sheet_headers(sheet):
first_row = next(sheet.iter_rows(min_row=1, max_row=1))
return [c.value for c in first_row]
def _get_values_from_columns(workbook, sheet_name, columns):
indexed_values = {}
for row in workbook_sheet_reader(workbook, sheet_name):
for col in columns:
indexed_values[str(row.get(col))] = ""
return indexed_values
def get_all_crossrefs(workbook, cross_refs_names):
crossrefs = {}
for ref_name, columns in cross_refs_names.items():
if columns:
crossrefs[ref_name] = _get_values_from_columns(workbook, ref_name,
columns)
else:
try:
crossrefs[ref_name] = get_all_cell_data_from_sheet(workbook, ref_name)
except ValueError as error:
if 'sheet is missing' in str(error):
crossrefs[ref_name] = []
else:
raise
return crossrefs
def get_all_in_memory_sheet(workbook, in_memory_sheet_conf):
in_memory_sheets = {}
for sheet_conf in in_memory_sheet_conf:
sheet_name = sheet_conf['sheet_name']
indexed_by = sheet_conf['indexed_by']
rows = workbook_sheet_reader(workbook, sheet_name)
indexed_rows = {row[indexed_by]: row for row in rows}
in_memory_sheets[sheet_name] = indexed_rows
return in_memory_sheets
def validate_content(workbook, validation_conf, crossrefs, in_memory_sheets):
for sheet_name in validation_conf.keys():
sheet_conf = validation_conf[sheet_name]
sheet_id_column = sheet_conf['id_field']
shown_values = {}
row_validation_steps = sheet_conf.get(ROW_VALIDATION, None)
for row in workbook_sheet_reader(workbook, sheet_name):
id_ = row.get(sheet_id_column, None)
if id_ is None:
error_code = _get_missing_row_id_error(sheet_id_column,
sheet_conf)
yield {'id': id_, 'sheet': sheet_name,
'field': sheet_id_column,
'error_code': error_code, 'value': None}
continue
do_have_cell_error = False
for column in sheet_conf[COLUMNS]:
label = column[FIELD]
validation_steps = column.get(VALIDATION, None)
value = row.get(label, None)
if validation_steps:
error_code = validate_cell(value, validation_steps,
crossrefs, shown_values, label)
if error_code is not None:
do_have_cell_error = True
yield {'id': id_, 'sheet': sheet_name, 'field': label,
'error_code': error_code, 'value': value}
if not do_have_cell_error and row_validation_steps:
error_code = validate_row(
row, row_validation_steps, in_memory_sheets)
if error_code is not None:
yield {'id': id_, 'sheet': sheet_name, 'field': 'row',
'error_code': error_code, 'value': 'row'}
def _get_missing_row_id_error(sheet_id_column, sheet_conf):
error_code = None
for column in sheet_conf[COLUMNS]:
if column[FIELD] == sheet_id_column:
error_code = [step[ERROR_CODE]
for step in column[VALIDATION] if step[TYPE] == MISSING][0]
return error_code
def validate_row(row, validation_steps, in_memory_sheets):
for validation_step in validation_steps:
kind = validation_step[TYPE]
error_code = validation_step[ERROR_CODE]
if kind == NAGOYA:
if not is_valid_nagoya_v20200601(row, in_memory_sheets):
return error_code
if not is_valid_nagoya_v12052023(row, in_memory_sheets):
return error_code
elif kind == BIBLIO:
if not is_valid_pub(row):
return error_code
elif kind == NAGOYA1:
if not is_valid_nago(row):
return error_code
else:
msg = f'{kind} is not a recognized row validation type method'
raise NotImplementedError(msg)
def validate_cell(value, validation_steps, crossrefs, shown_values, label):
for step_conf in validation_steps:
if step_conf[TYPE] == MANDATORY:
continue
step_conf['crossrefs_pointer'] = crossrefs
step_conf['shown_values'] = shown_values
step_conf['label'] = label
error_code = validate_value(value, step_conf)
if error_code is not None:
return error_code
def is_valid_pub(row):
pub_id = row.get('ID', None)
pub_pmid = row.get('PMID', None)
pub_doi = row.get('DOI', None)
title = row.get('Title', None)
full_reference = row.get('Full reference', None)
authors = row.get('Authors', None)
journal = row.get('Journal', None)
year = row.get('Year', None)
volumen = row.get('Volume', None)
first_page = row.get('First page', None)
book_title = row.get('Book title', None)
editors = row.get('Editors', None)
publishers = row.get('Publishers', None)
if (pub_id != None and pub_doi != None) or (pub_id != None and pub_pmid != None) or (pub_id != None and full_reference != None) or (pub_id != None and authors != None and title != None and journal != None and year != None and volumen != None and first_page != None) :
return True
is_journal = bool(title)
# if (is_journal and (not authors or not journal or not not year or
# not volumen or not first_page)):
# return False
#if (not is_journal and (not authors or not year or
# not editors or not publishers or not book_title)):
# return False
return False
def is_valid_nago(row):
if not row:
return True
status = row.get("status", None)
type = row.get("type", None)
regex = r'^[a-zA-Z\s.\'-]+$'
if status != None and type != None:
if (re.match(regex, status) and type==1):
return False
if (type == 2 and status is None):
return False
return True
def parsee_mirri_excel(row, in_memory_sheets, version=""):
if version == "20200601":
return is_valid_nagoya_v20200601 (row, in_memory_sheets)
elif version == "12052023":
return is_valid_nagoya_v12052023 (row, in_memory_sheets)
else:
raise NotImplementedError("Only versions 20200601 and 12052023 are implemented")
def is_valid_nagoya_v20200601(row, in_memory_sheets): # sourcery skip: return-identity
location_index = row.get('Geographic origin', None)
if location_index is None:
country = None
else:
geo_origin = in_memory_sheets[LOCATIONS].get(location_index, {})
country = geo_origin.get('Country', None)
_date = row.get("Date of collection", None)
if _date is None:
_date = row.get("Date of isolation", None)
if _date is None:
_date = row.get("Date of deposit", None)
if _date is None:
_date = row.get("Date of inclusion in the catalogue", None)
if _date is not None:
year = _date.year if isinstance(_date, datetime) else int(str(_date)[:4])
else:
year = None
if year is not None and year >= 2014 and country is None:
return False
return True
def is_valid_nagoya_v12052023(row, in_memory_sheets): # sourcery skip: return-identity
location_index = row.get('geographicOrigin', None)
if location_index is None:
country = None
else:
geo_origin = in_memory_sheets[LOCATIONS].get(location_index, {})
country = geo_origin.get('Country', None)
_date = row.get("collectionDate", None)
if _date is None:
_date = row.get("isolationDate", None)
if _date is None:
_date = row.get("depositDate", None)
if _date is None:
_date = row.get("accessionDate", None)
if _date is not None:
year = _date.year if isinstance(_date, datetime) else int(str(_date)[:4])
else:
year = None
if year is not None and year >= 2014 and country is None:
return False
return True
def is_valid_regex(value, validation_conf):
if value is None:
return True
value = str(value)
regexp = validation_conf[MATCH]
multiple = validation_conf.get(MULTIPLE, False)
separator = validation_conf.get(SEPARATOR, None)
values = [v.strip() for v in value.split(
separator)] if multiple else [value]
for value in values:
matches_regexp = re.fullmatch(regexp, value)
if not matches_regexp:
return False
return True
def is_valid_crossrefs(value, validation_conf):
crossref_name = validation_conf[CROSSREF_NAME]
crossrefs = validation_conf['crossrefs_pointer']
choices = crossrefs[crossref_name]
if value is None or not choices:
return True
value = str(value)
multiple = validation_conf.get(MULTIPLE, False)
separator = validation_conf.get(SEPARATOR, None)
if value is None:
return True
if multiple:
values = [v.strip() for v in value.split(separator)]
else:
values = [value.strip()]
return all(value in choices for value in values)
def is_valid_choices(value, validation_conf):
if value is None:
return True
choices = validation_conf[VALUES]
multiple = validation_conf.get(MULTIPLE, False)
separator = validation_conf.get(SEPARATOR, None)
if multiple:
values = [v.strip() for v in str(value).split(separator)]
else:
values = [str(value).strip()]
sorted_values = sorted(values)
if sorted_values != values:
return False
return all(value in choices for value in values)
def is_valid_date(value, validation_conf):
if value is None:
return True
if isinstance(value, datetime):
year = value.year
month = value.month
day = value.day
elif isinstance(value, int):
year = value
month = None
day = None
elif isinstance(value, str):
value = value.replace('-', '')
value = value.replace('/', '')
month = None
day = None
try:
year = int(value[: 4])
if len(value) >= 6:
month = int(value[4: 6])
if len(value) >= 8:
day = int(value[6: 8])
except (IndexError, TypeError, ValueError):
return False
else:
return False
if year < 1700 or year > datetime.now().year:
return False
if month is not None:
if month < 1 or month > 13:
return False
if day is not None and (day < 1 or day > monthrange(year, month)[1]):
return False
return True
def is_valid_dominio(value, validation_conf=None):
if value is None:
return True
try:
items = [i.strip() for i in value.split(";")]
if len(items) >1:
for i in range(0, len(items),2):
nameSite = str(items[i])
urlSite = str(items[i+1])
dominio = urlSite.split(".")[-2]
if nameSite.lower() != dominio:
return False
return True
except:
return False
def is_valid_title(value, validation_conf=None):
if value is None:
return True
try:
items = [i.strip() for i in value.split(";")]
if len(items) >1:
for i in range(0, len(items),2):
nameSite = (items[i])
urlSite = str(items[i+1])
regex = r'^(http|https):\/\/[a-z0-9\-\.]+\.[a-z]{2,}([/a-z0-9\-\.]*)*$'
if re.match(regex, nameSite) or isinstance(nameSite, int) or nameSite == '':
return False
return True
except:
return False
def is_valid_url_title(value, validation_conf=None):
if value is None:
return True
try:
items = [i.strip() for i in value.split(";")]
if len(items) ==1:
urlSite = str(items[0])
response = requests.head(urlSite)
if response.status_code != 200:
return False
else:
items = [i.strip() for i in value.split(";")]
for i in range(0, len(items),2):
nameSite = (items[i])
urlSite = str(items[i+1])
response = requests.head(urlSite)
if response.status_code != 200:
return False
return True
except:
return False
def is_valid_url_dominio(value, validation_conf=None):
if value is None:
return True
try:
items = [i.strip() for i in value.split(";")]
for i in range(0, len(items),2):
nameSite = str(items[i])
urlSite = str(items[i+1])
response = requests.head(urlSite)
if response.status_code != 200:
return False
return True
except:
return False
def is_valid_just_url(value, validation_conf=None):
if value is None:
return True
try:
items = [i.strip() for i in value.split(";")]
for i in items:
nameSite = str(items[0])
response = requests.head(i)
if response.status_code != 200:
return False
return True
except:
return False
def is_valid_history(value, validation_conf=None):
if value is None:
return True
try:
items = [i.strip() for i in value.split("<")]
for i in items:
regex1 = r'^[a-zA-Z &,;.''-]+, ((19|20)\d{2})'
regex2 = r'^[a-zA-Z &,;.''-]+, [a-zA-Z &,;.''-] (19|20)\d{2}\s\([a-zA-Z &,;.''-]+\)'
regex3 = r'^[a-zA-Z &,;.''-]+\, [a-zA-Z &,;.''-]'
regex4 = r'^[a-zA-Z &,;.''-]+, (19|20)\d{2}\s\([a-zA-Z .''-,;&]+\)'
regex5 = r'^[a-zA-Z &,;.''-]+, \([a-zA-Z &,;.''-]+\) (19|20)\d{2}'
if re.match(regex1, i):
return True
elif re.match(regex2, i):
return True
elif re.match(regex3, i):
return True
elif re.match(regex4, i):
return True
elif re.match(regex5, i):
return True
else:
return False
except:
return False
def is_valid_coords(value, validation_conf=None):
if value is None:
return True
try:
regex1 = r'^-?(90(\.0+)?|[1-8]?\d(\.\d+)?);-?(180(\.0+)?|((1[0-7]\d)|(\d{1,2}))(\.\d+)?)$'
regex2 = r'^-?(90(\.0+)?|[1-8]?\d(\.\d+)?);-?(180(\.0+)?|((1[0-7]\d)|(\d{1,2}))(\.\d+)?);(\d+\.\d+|\?);(\d+\.\d+|\?)$|^(\d+\.\d+|\?)$|^;$'
if not re.match(regex1, value) and not re.match(regex2, value):
return False
return True
except:
return False
def is_valid_missing(value, validation_conf=None):
return value is not None
def is_valid_taxon(value, validation_conf=None):
multiple = validation_conf.get(MULTIPLE, False)
separator = validation_conf.get(SEPARATOR, ';')
value = value.split(separator) if multiple else [value]
for taxon in value:
taxon = taxon.strip()
if not _is_valid_taxon(taxon):
return False
return True
def _is_valid_taxon(value):
value = value.strip()
if not value:
return True
items = re.split(r" +", value)
genus = items[0]
if len(items) > 1:
species = items[1]
if species in ("sp", "spp", ".sp", "sp."):
return False
if len(items) > 2:
for index in range(0, len(items[2:]), 2):
rank = SUBTAXAS.get(items[index + 2], None)
if rank is None:
print(value)
return False
return True
def is_valid_unique(value, validation_conf):
if not value:
return True
label = validation_conf['label']
shown_values = validation_conf['shown_values']
if label not in shown_values:
shown_values[label] = {}
already_in_file = shown_values[label]
if value in already_in_file:
return False
# NOTE: what's the use of this?
# What is the expected format for value and shown_values?
shown_values[label][value] = None
return True
def is_valid_file(path):
try:
with path.open("rb") as fhand:
error_log = validate_mirri_excel(fhand)
if "EXL" in error_log.get_errors():
return False
except:
return False
return True
VALIDATION_FUNCTIONS = {
MISSING: is_valid_missing,
REGEXP: is_valid_regex,
CHOICES: is_valid_choices,
CROSSREF: is_valid_crossrefs,
DATE: is_valid_date,
COORDINATES: is_valid_coords,
TAXON: is_valid_taxon,
TITLE: is_valid_title,
DOMINIO: is_valid_dominio,
URL_TITLE: is_valid_url_title,
URL_DOMINIO: is_valid_url_dominio,
JUST_URL: is_valid_just_url,
ISO: validate_country_code,
HISTORY: is_valid_history,
VERSION: version,
UNIQUE: is_valid_unique}
def validate_value(value, step_conf):
kind = step_conf[TYPE]
try:
is_value_valid = VALIDATION_FUNCTIONS[kind]
except KeyError:
msg = f'This validation type {kind} is not implemented'
raise NotImplementedError(msg)
error_code = step_conf[ERROR_CODE]
if not is_value_valid(value, step_conf):
return error_code
+35
View File
@@ -0,0 +1,35 @@
MANDATORY = "mandatory"
REGEXP = "regexp"
CHOICES = "choices"
CROSSREF = 'crossref'
CROSSREF_NAME = 'crossref_name'
MISSING = "missing"
VALIDATION = 'validation'
ERROR_CODE = 'error_code'
FIELD = 'field'
MULTIPLE = 'multiple'
TYPE = 'type'
COLUMNS = 'columns'
SOURCE = "sources"
SEPARATOR = "separator"
MATCH = 'match'
VALUES = 'values'
DATE = 'date'
COORDINATES = 'coord'
COORDINATES1 = 'coord1'
NUMBER = 'number'
TAXON = 'taxon'
UNIQUE = 'unique'
ROW_VALIDATION = 'row_validation'
NAGOYA = 'nagoya'
BIBLIO = 'bibliography'
DOMINIO= 'is_valid_dominio'
TITLE= 'is_valid_title'
URL_DOMINIO = 'urll_valid_dominio'
URL_TITLE= 'is_valid_url_title'
ISO = 'validate_country_code'
JUST_URL= 'is_valid_just_url'
HISTORY= 'is_valid_history'
MEU='is_valid_crossrefs_meu'
NAGOYA1 = 'nayoga1'
VERSION = 'version'
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env python
import pandas as pd
import sys
from pathlib import Path
import warnings
warnings.simplefilter("ignore")
from mirri.validation.excel_validator import validate_mirri_excel
def main():
path = Path(sys.argv[1])
version = str(sys.argv[2])
date = str(sys.argv[3])
try:
error_log = validate_mirri_excel(path.open("rb"), version=version, date=date)
except NotImplementedError as e:
print(e)
for errors in error_log.get_errors().values():
for error in errors:
print(error.pk, error.message, error.code)
if __name__ == "__main__":
main()
+675
View File
@@ -0,0 +1,675 @@
from mirri.validation.tags import (CHOICES, COLUMNS, COORDINATES, CROSSREF, CROSSREF_NAME, DATE,
ERROR_CODE, FIELD, MANDATORY, MATCH,
MISSING, MULTIPLE, NAGOYA, REGEXP, ROW_VALIDATION, SEPARATOR, TAXON, TYPE,
UNIQUE,VERSION,
VALIDATION, VALUES, BIBLIO, DOMINIO, URL_DOMINIO,ISO, JUST_URL, URL_TITLE, TITLE, HISTORY,NAGOYA1)
from mirri.settings import (ONTOBIOTOPE, LOCATIONS, GROWTH_MEDIA, GENOMIC_INFO,
STRAINS, LITERATURE_SHEET, SEXUAL_STATE_SHEET, MARKERS, CONTROL_SHEET,)
# GEOGRAPHIC_ORIGIN
# SEXUAL_STATE_SHEET,
# RESOURCE_TYPES_VALUES,
# FORM_OF_SUPPLY_SHEET,
# PLOIDY_SHEET)
STRAIN_FIELDS = [
{
FIELD: "accessionNumber",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: 'STD01'},
{TYPE: UNIQUE, ERROR_CODE: 'STD03'},
{TYPE: MISSING, ERROR_CODE: "STD02"},
{TYPE: REGEXP, MATCH: "[^ ]* [^ ]*", ERROR_CODE: "STD04"}
]
},
{
FIELD: "useRestrictions",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD05"},
{TYPE: MISSING, ERROR_CODE: "STD06"},
{TYPE: CHOICES, VALUES: ["1", "2", "3"],
MULTIPLE: False, ERROR_CODE: "STD07"}
]
},
{
FIELD: "mirriAccessionNumber",
VALIDATION: [
{TYPE: UNIQUE, ERROR_CODE: 'STD51'},
{TYPE: REGEXP, MATCH: "^MIRRI[0-9]{7}$", ERROR_CODE: "STD52"},
],
},
{
FIELD: "nagoyaConditions",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD08"},
{TYPE: MISSING, ERROR_CODE: "STD09"},
{TYPE: CHOICES, VALUES: ["1", "2", "3"],
MULTIPLE: False, ERROR_CODE: "STD10"}
]
},
{
FIELD: "absFile",
VALIDATION: [
{TYPE: TITLE, ERROR_CODE: "STD59"},
{TYPE: URL_TITLE, ERROR_CODE: "STD60",
MULTIPLE: True, SEPARATOR: ";"},
],
},
{
FIELD: "siteLinks",
VALIDATION: [
{TYPE: DOMINIO, ERROR_CODE: "STD53",
MULTIPLE: False, SEPARATOR: ";"},
{TYPE: URL_DOMINIO, ERROR_CODE: "STD56",
MULTIPLE: False, SEPARATOR: ";"},
],
},
{
FIELD: "mtaFile",
VALIDATION: [
{TYPE: JUST_URL, ERROR_CODE: "STD58",
MULTIPLE: True, SEPARATOR: ";"},
],
},
{
FIELD: "otherCollectionNumbers",
VALIDATION: [
{TYPE: REGEXP, MATCH: "([^ ]* [^ ]*)(; [^ ]* [^ ]*)*$", ERROR_CODE: "STD63"},
#{TYPE: CROSSREF, CROSSREF_NAME: "Strains", ERROR_CODE: "STD64"},
]
},
{
FIELD: "registeredCollection",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD11"}
]
},
{
FIELD: "type",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"], ERROR_CODE: "STD64"},
]
},
{
FIELD: "riskGroup",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD12"},
{TYPE: MISSING, ERROR_CODE: "STD13"},
{TYPE: CHOICES, VALUES: ["1", "2", "3", "4"],
MULTIPLE: False, ERROR_CODE: "STD14"}
]
},
{
FIELD: "dualUse",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD15"}
]
},
{
FIELD: "euQuarantine",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD16"}
]
},
{
FIELD: "axenicCulture",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["Axenic", "Not axenic"],
ERROR_CODE: "STD50"}
]
},
{
FIELD: "organismType",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD17"},
{TYPE: MISSING, ERROR_CODE: "STD18"},
{TYPE: CHOICES, VALUES: ["Algae", "Archaea", "Bacteria",
"Cyanobacteria", "Filamentous Fungi", "Filamentous fungi",
"Yeast", "Microalgae",
"1", "2", "3", "4", "5", "6", "7"],
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD19"}
]
},
{
FIELD: "speciesName",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD20"},
{TYPE: MISSING, ERROR_CODE: "STD21"},
{TYPE: TAXON, ERROR_CODE: "STD22", MULTIPLE: True,
SEPARATOR: ';'}
]
},
{
FIELD: "infrasubspecificNames",
VALIDATION: []
},
{
FIELD: "taxonomyComments",
VALIDATION: []
},
{
FIELD: "hybrid",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD23"}
]
},
{
FIELD: "status",
VALIDATION: [
{TYPE: REGEXP, MATCH: "^(type of|neotype of|holotype of |epitype of) ([a-zA-Z .'-]+)$", ERROR_CODE: "STD65"},
]
},
{
FIELD: "depositHistory",
VALIDATION: [
{TYPE: HISTORY, ERROR_CODE: 'STD24'},
]
},
{
FIELD: "depositor",
VALIDATION: []
},
{
FIELD: "depositDate",
VALIDATION: [
{TYPE: DATE, ERROR_CODE: "STD25"},
]
},
{
FIELD: "accessionDate",
VALIDATION: [
{TYPE: DATE, ERROR_CODE: "STD26"},
]
},
{
FIELD: "collector",
VALIDATION: []
},
{
FIELD: "substrate",
VALIDATION: []
},
{
FIELD: "temperatureGrowthRange",
VALIDATION: [
{TYPE: REGEXP, "match": r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?',
ERROR_CODE: "STD29", MULTIPLE: True, SEPARATOR: ";"}
]
},
{
FIELD: "recommendedTemperature",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD30"},
{TYPE: MISSING, ERROR_CODE: "STD31"},
{TYPE: REGEXP, "match": r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?',
ERROR_CODE: "STD32",
MULTIPLE: True, SEPARATOR: ";"}
]
},
{
FIELD: "supplyForms",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD36"},
{TYPE: MISSING, ERROR_CODE: "STD37"},
{TYPE: CHOICES, VALUES: ['Agar', 'Cryo', 'Dry Ice', 'Liquid Culture Medium',
'Lyo', 'Oil', 'Water'],
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD38"}
]
},
{
FIELD: "otherDenomination",
VALIDATION: []
},
{
FIELD: "geographicCoordinates",
VALIDATION: [
{TYPE: COORDINATES, ERROR_CODE: "STD39"},
]
},
{
# value can be in the cell or in another sheet. Don't configure this
FIELD: "geographicOrigin",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD68"},
{TYPE: MISSING, ERROR_CODE: "STD69"},
{TYPE: CROSSREF, CROSSREF_NAME: "Geographic origin", ERROR_CODE: "STD46"},
]
},
{
FIELD: "isolationHabitat",
VALIDATION: []
},
{
FIELD: "ontobiotopeTerms",
VALIDATION: [
{TYPE: CROSSREF, CROSSREF_NAME: "Ontobiotope",
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD41"}
]
},
{
FIELD: "qps",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD49"}
]
},
{
FIELD: "gmo",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD42"}
]
},
{
FIELD: "gmoConstruction",
VALIDATION: []
},
{
FIELD: "mutant",
VALIDATION: []
},
{
FIELD: "genotype",
VALIDATION: []
},
{
FIELD: "Plant pathogenicity code",
VALIDATION: []
},
{
FIELD: "sexualState",
VALIDATION: [
{TYPE: CROSSREF, CROSSREF_NAME: SEXUAL_STATE_SHEET,
ERROR_CODE: "STD43"}
# {TYPE: CHOICES, VALUES: ["Mata", "Matalpha", "Mata/Matalpha",
# "Matb", "Mata/Matb", "MTLa", "MTLalpha", "MTLa/MTLalpha",
# "MAT1-1", "MAT1-2", "MAT1", "MAT2", "MT+", "MT-"],
# ERROR_CODE: "STD43"}
]
},
{
FIELD: "ploidy",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2", "3", "4", "5", "9"],
ERROR_CODE: "STD44"}
]
},
{
FIELD: "plasmids",
VALIDATION: []
},
{
FIELD: "plasmidCollections",
VALIDATION: [
{TYPE: REGEXP, MATCH: "([a-zA-Z .'-]+)\(([a-zA-Z .'-]+) (\d+)\)(; ([a-zA-Z .'-]+)\(([a-zA-Z .'-]+) (\d+)\))*$",
ERROR_CODE: "STD62"}
]
},
{
# value can be in the cell or in another sheet. Don't configure this
FIELD: "identificationLiterature",
VALIDATION: [
{TYPE: CROSSREF, CROSSREF_NAME: LITERATURE_SHEET,
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD45"}
]
},
{
FIELD: "pathogenicity",
VALIDATION: []
},
{
FIELD: "enzymes",
VALIDATION: []
},
{
FIELD: "metabolites",
VALIDATION: []
},
{
FIELD: "applications",
VALIDATION: []
},
{
FIELD: "remarks",
VALIDATION: []
},
{
FIELD: "sequenceLiterature",
VALIDATION: [
{TYPE: REGEXP, MATCH: "^\d+(; \d+)*$", ERROR_CODE: "STD61"},
]
},
{
FIELD: "recommendedMedium",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD33"},
{TYPE: MISSING, ERROR_CODE: "STD34"},
{TYPE: CROSSREF, CROSSREF_NAME: "Growth media",
MULTIPLE: True, SEPARATOR: "/", ERROR_CODE: "STD35"}
]
},
{
FIELD: "country",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD54"},
{TYPE: MISSING, ERROR_CODE: "STD55"},
{TYPE: ISO, ERROR_CODE: "STD57"},
#{TYPE: CROSSREF, CROSSREF_NAME: COUNTRY_CODES_SHEET, ERROR_CODE: "STD57"}
]
},
]
SHEETS_SCHEMA = {
LOCATIONS: {
"acronym": "GOD",
"id_field": "ID",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS02"},
COLUMNS: [
{
FIELD: "ID",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GOD01"},
{TYPE: MISSING, ERROR_CODE: "GOD02"},
]
},
{
FIELD: "Country",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GOD03"},
{TYPE: MISSING, ERROR_CODE: "GOD04"},
]
},
{
FIELD: "Region",
VALIDATION: []
},
{
FIELD: "City",
VALIDATION: []
},
{
FIELD: "Locality",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GOD06"},
{TYPE: MISSING, ERROR_CODE: "GOD07"}
]
}
],
},
GROWTH_MEDIA: {
"acronym": "GMD",
"id_field": "Acronym",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS01"},
COLUMNS: [
{
FIELD: "Acronym",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GMD01"},
{TYPE: MISSING, ERROR_CODE: "GMD02"}
]
},
{
FIELD: "Description",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GMD03"},
{TYPE: MISSING, ERROR_CODE: "GMD04"}
]
},
{
FIELD: "Full description",
VALIDATION: []
},
],
},
GENOMIC_INFO: {
"acronym": "GID",
"id_field": "Strain AN",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS08"},
COLUMNS: [
{
FIELD: "Strain AN",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GID01"},
{TYPE: MISSING, ERROR_CODE: "GID02"},
{TYPE: CROSSREF, CROSSREF_NAME: "Strains",
ERROR_CODE: "GID03"},
]
},
{
FIELD: "Marker",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GID04"},
{TYPE: MISSING, ERROR_CODE: "GID05"},
{TYPE: CROSSREF, CROSSREF_NAME: MARKERS, ERROR_CODE: "GID06"}
]
},
{
FIELD: "INSDC AN",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GID07"},
{TYPE: MISSING, ERROR_CODE: "GID08"},
{TYPE: REGEXP, MATCH: "^[A-Z]{2}[0-9]{6}$", ERROR_CODE: "GID11"},
]
},
{
FIELD: "Sequence",
VALIDATION: []
},
],
},
STRAINS: {
"acronym": "STD",
'id_field': 'accessionNumber',
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS05"},
ROW_VALIDATION: [
#{TYPE: NAGOYA, ERROR_CODE: "STD46"},
{TYPE: NAGOYA1, ERROR_CODE: "STD64"}
],
COLUMNS: STRAIN_FIELDS,
},
LITERATURE_SHEET: {
"acronym": "LID",
'id_field': 'ID',
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS03"},
ROW_VALIDATION: [
{TYPE: BIBLIO, ERROR_CODE: 'LID17'}
],
COLUMNS: [
{
FIELD: "ID",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID01"},
{TYPE: MISSING, ERROR_CODE: "LID02"},
]
},
{
FIELD: "PMID",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID18"},
]
},
{
FIELD: "DOI",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID20"},
]
},
{
FIELD: "Full reference",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID03"},
]
},
{
FIELD: "Authors",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID05"},
]
},
{
FIELD: "Title",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID07"},
]
},
{
FIELD: "Journal",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID09"},
]
},
{
FIELD: "Year",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID11"},
]
},
{
FIELD: "Volume",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID13"},
]
},
{
FIELD: "Issue",
VALIDATION: []
},
{
FIELD: "First page",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID15"},
]
},
{
FIELD: "Last page",
VALIDATION: []
},
{
FIELD: "Book title",
VALIDATION: []
},
{
FIELD: "Editors",
VALIDATION: []
},
{
FIELD: "Publisher",
VALIDATION: []
}
],
},
# SEXUAL_STATE_SHEET: {"acronym": "SSD", COLUMNS: []},
# RESOURCE_TYPES_VALUES: {"acronym": "RTD", COLUMNS: []},
# FORM_OF_SUPPLY_SHEET: {"acronym": "FSD", COLUMNS: []},
# PLOIDY_SHEET: {"acronym": "PLD", COLUMNS: []},
ONTOBIOTOPE: {
"acronym": "OTD",
"id_field": "ID",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS06"},
COLUMNS: [
{
FIELD: "ID",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "OTD01"},
{TYPE: MISSING, ERROR_CODE: "OTD02"},
]
},
{
FIELD: "Name",
VALIDATION: []
},
]
},
CONTROL_SHEET: {
"acronym": "CTR",
"id_field": "Version",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS09"},
COLUMNS: [
{
FIELD: "Version",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "CTR01"},
{TYPE: MISSING, ERROR_CODE: "CTR02"},
{TYPE: VERSION, ERROR_CODE: "CTR05"},
]
},
{
FIELD: "Date",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "CTR03"},
{TYPE: MISSING, ERROR_CODE: "CTR04"},
]
},
]
},
MARKERS: {
"acronym": "MKD",
"id_field": "Acronym",
COLUMNS: [
{
FIELD: "Acronym",
VALIDATION: []
},
{
FIELD: "Marker",
VALIDATION: []
},
],
},
}
CROSS_REF_CONF = {
ONTOBIOTOPE: ['ID'],
LITERATURE_SHEET: ['ID', 'DOI', 'PMID', 'Full reference', 'Authors', 'Title', 'Journal', 'Year', 'Volume', 'First page'],
LOCATIONS: ['ID', 'Locality'],
GROWTH_MEDIA: ['Acronym'],
STRAINS: ["accessionNumber"],
SEXUAL_STATE_SHEET: [],
MARKERS: ["Acronym"],
}
MIRRI_12052023_VALLIDATION_CONF = {
'sheet_schema': SHEETS_SCHEMA,
'cross_ref_conf': CROSS_REF_CONF,
'keep_sheets_in_memory': [
{'sheet_name': LOCATIONS, 'indexed_by': 'Locality'}]
}
version_config = {
'5.1.2': MIRRI_12052023_VALLIDATION_CONF,
'date': '12/05/2023'
}
+545
View File
@@ -0,0 +1,545 @@
from mirri.validation.tags import (CHOICES, COLUMNS, COORDINATES, CROSSREF, CROSSREF_NAME, DATE,
ERROR_CODE, FIELD, MANDATORY, MATCH,
MISSING, MULTIPLE, NAGOYA, NUMBER, REGEXP, ROW_VALIDATION, SEPARATOR, TAXON, TYPE,
UNIQUE,
VALIDATION, VALUES, BIBLIO)
from mirri.settings_v1 import (ONTOBIOTOPE, LOCATIONS, GROWTH_MEDIA, GENOMIC_INFO,
STRAINS, LITERATURE_SHEET, SEXUAL_STATE_SHEET, MARKERS)
# GEOGRAPHIC_ORIGIN
# SEXUAL_STATE_SHEET,
# RESOURCE_TYPES_VALUES,
# FORM_OF_SUPPLY_SHEET,
# PLOIDY_SHEET)
STRAIN_FIELDS = [
{
FIELD: "Accession number",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: 'STD01'},
{TYPE: UNIQUE, ERROR_CODE: 'STD03'},
{TYPE: MISSING, ERROR_CODE: "STD02"},
{TYPE: REGEXP, MATCH: "[^ ]* [^ ]*", ERROR_CODE: "STD04"}
]
},
{
FIELD: "Restrictions on use",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD05"},
{TYPE: MISSING, ERROR_CODE: "STD06"},
{TYPE: CHOICES, VALUES: ["1", "2", "3"],
MULTIPLE: False, ERROR_CODE: "STD07"}
]
},
{
FIELD: "Nagoya protocol restrictions and compliance conditions",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD08"},
{TYPE: MISSING, ERROR_CODE: "STD09"},
{TYPE: CHOICES, VALUES: ["1", "2", "3"],
MULTIPLE: False, ERROR_CODE: "STD10"}
]
},
{
FIELD: "ABS related files",
VALIDATION: [],
},
{
FIELD: "MTA file",
VALIDATION: [],
},
{
FIELD: "Other culture collection numbers",
# VALIDATION: [
# {TYPE: REGEXP, "match": "[^ ]* [^ ]*", ERROR_CODE: "STD07",
# MULTIPLE: True, SEPARATOR: ";"}
# ]
},
{
FIELD: "Strain from a registered collection",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD11"}
]
},
{
FIELD: "Risk Group",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD12"},
{TYPE: MISSING, ERROR_CODE: "STD13"},
{TYPE: CHOICES, VALUES: ["1", "2", "3", "4"],
MULTIPLE: False, ERROR_CODE: "STD14"}
]
},
{
FIELD: "Dual use",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD15"}
]
},
{
FIELD: "Quarantine in Europe",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD16"}
]
},
{
FIELD: "Organism type",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD17"},
{TYPE: MISSING, ERROR_CODE: "STD18"},
{TYPE: CHOICES, VALUES: ["Algae", "Archaea", "Bacteria",
"Cyanobacteria", "Filamentous Fungi",
"Phage", "Plasmid", "Virus", "Yeast",
"1", "2", "3", "4", "5", "6", "7", "8", "9"],
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD19"}
]
},
{
FIELD: "Taxon name",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD20"},
{TYPE: MISSING, ERROR_CODE: "STD21"},
{TYPE: TAXON, ERROR_CODE: "STD22", MULTIPLE: True,
SEPARATOR: ';'}
]
},
{
FIELD: "Infrasubspecific names",
},
{
FIELD: "Comment on taxonomy",
},
{
FIELD: "Interspecific hybrid",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD23"}
]
},
{
FIELD: "Status",
},
{
FIELD: "History of deposit",
VALIDATION: [
# {TYPE: REGEXP, "match": "[^ ]* [^ ]*", ERROR_CODE: "STD24", # modify the regex
# MULTIPLE: True, SEPARATOR: ";"}
]
},
{
FIELD: "Depositor"
},
{
FIELD: "Date of deposit",
VALIDATION: [
{TYPE: DATE, ERROR_CODE: "STD25"},
]
},
{
FIELD: "Date of inclusion in the catalogue",
VALIDATION: [
{TYPE: DATE, ERROR_CODE: "STD26"},
]
},
{
FIELD: "Collected by",
},
{
FIELD: "Date of collection",
VALIDATION: [
{TYPE: DATE, ERROR_CODE: "STD27"},
]
},
{
FIELD: "Isolated by",
},
{
FIELD: "Date of isolation",
VALIDATION: [
{TYPE: DATE, ERROR_CODE: "STD28"},
]
},
{
FIELD: "Substrate/host of isolation",
},
{
FIELD: "Tested temperature growth range",
VALIDATION: [
{TYPE: REGEXP, "match": r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?',
ERROR_CODE: "STD29", MULTIPLE: True, SEPARATOR: ";"}
]
},
{
FIELD: "Recommended growth temperature",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD30"},
{TYPE: MISSING, ERROR_CODE: "STD31"},
{TYPE: REGEXP, "match": r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?',
ERROR_CODE: "STD32",
MULTIPLE: True, SEPARATOR: ";"}
]
},
{
FIELD: "Recommended medium for growth",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD33"},
{TYPE: MISSING, ERROR_CODE: "STD34"},
{TYPE: CROSSREF, CROSSREF_NAME: "Growth media",
MULTIPLE: True, SEPARATOR: "/", ERROR_CODE: "STD35"}
]
},
{
FIELD: "Form of supply",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "STD36"},
{TYPE: MISSING, ERROR_CODE: "STD37"},
{TYPE: CHOICES, VALUES: ['Agar', 'Cryo', 'Dry Ice', 'Liquid Culture Medium',
'Lyo', 'Oil', 'Water'],
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD38"}
]
},
{
FIELD: "Other denomination",
},
{
FIELD: "Coordinates of geographic origin",
VALIDATION: [
{TYPE: COORDINATES, ERROR_CODE: "STD39"},
]
},
{
FIELD: "Altitude of geographic origin",
VALIDATION: [
{TYPE: NUMBER, 'max': 8000, 'min': -200, ERROR_CODE: "STD40"},
]
},
{
# value can be in the cell or in another sheet. Don't configure this
FIELD: "Geographic origin",
},
{
FIELD: "Isolation habitat",
},
{
FIELD: "Ontobiotope term for the isolation habitat",
VALIDATION: [
{TYPE: CROSSREF, CROSSREF_NAME: "Ontobiotope",
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD41"}
]
},
{
FIELD: "GMO",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["1", "2"],
ERROR_CODE: "STD42"}
]
},
{
FIELD: "GMO construction information",
},
{
FIELD: "Mutant information",
},
{
FIELD: "Genotype",
},
{
FIELD: "Sexual state",
VALIDATION: [
{TYPE: CROSSREF, CROSSREF_NAME: SEXUAL_STATE_SHEET,
ERROR_CODE: "STD43"}
# {TYPE: CHOICES, VALUES: ["Mata", "Matalpha", "Mata/Matalpha",
# "Matb", "Mata/Matb", "MTLa", "MTLalpha", "MTLa/MTLalpha",
# "MAT1-1", "MAT1-2", "MAT1", "MAT2", "MT+", "MT-"],
# ERROR_CODE: "STD43"}
]
},
{
FIELD: "Ploidy",
VALIDATION: [
{TYPE: CHOICES, VALUES: ["0", "1", "2", "3", "4", "9"],
ERROR_CODE: "STD44"}
]
},
{
FIELD: "Plasmids",
},
{
FIELD: "Plasmids collections fields",
},
{
# value can be in the cell or in another sheet. Don't configure this
FIELD: "Literature",
VALIDATION: [
{TYPE: CROSSREF, CROSSREF_NAME: LITERATURE_SHEET,
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD45"}
]
},
{
FIELD: "Plant pathogenicity code",
},
{
FIELD: "Pathogenicity",
},
{
FIELD: "Enzyme production",
},
{
FIELD: "Production of metabolites",
},
{
FIELD: "Applications",
},
{
FIELD: "Remarks"
},
{
FIELD: "Literature linked to the sequence/genome",
},
]
SHEETS_SCHEMA = {
LOCATIONS: {
"acronym": "GOD",
"id_field": "ID",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS02"},
COLUMNS: [
{
FIELD: "ID",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GOD01"},
{TYPE: MISSING, ERROR_CODE: "GOD02"},
]
},
{
FIELD: "Country",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GOD03"},
{TYPE: MISSING, ERROR_CODE: "GOD04"}
]
},
{
FIELD: "Region",
VALIDATION: []
},
{
FIELD: "City",
VALIDATION: []
},
{
FIELD: "Locality",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GOD06"},
{TYPE: MISSING, ERROR_CODE: "GOD07"}
]
}
],
},
GROWTH_MEDIA: {
"acronym": "GMD",
"id_field": "Acronym",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS01"},
COLUMNS: [
{
FIELD: "Acronym",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GMD01"},
{TYPE: MISSING, ERROR_CODE: "GMD02"}
]
},
{
FIELD: "Description",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GMD03"},
{TYPE: MISSING, ERROR_CODE: "GMD04"}
]
},
{
FIELD: "Full description",
VALIDATION: []
},
],
},
GENOMIC_INFO: {
"acronym": "GID",
"id_field": "Strain AN",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS08"},
COLUMNS: [
{
FIELD: "Strain AN",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GID01"},
{TYPE: MISSING, ERROR_CODE: "GID02"},
{TYPE: CROSSREF, CROSSREF_NAME: "Strains",
ERROR_CODE: "GID03"},
]
},
{
FIELD: "Marker",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GID04"},
{TYPE: MISSING, ERROR_CODE: "GID05"},
{TYPE: CROSSREF, CROSSREF_NAME: MARKERS, ERROR_CODE: "GID06"}
]
},
{
FIELD: "INSDC AN",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "GID07"},
{TYPE: MISSING, ERROR_CODE: "GID08"},
]
},
{
FIELD: "Sequence",
VALIDATION: []
},
],
},
STRAINS: {
"acronym": "STD",
'id_field': 'Accession number',
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS05"},
ROW_VALIDATION: [
{TYPE: NAGOYA, ERROR_CODE: "STD46"},
],
COLUMNS: STRAIN_FIELDS,
},
LITERATURE_SHEET: {
"acronym": "LID",
'id_field': 'ID',
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS03"},
ROW_VALIDATION: [
{TYPE: BIBLIO, ERROR_CODE: 'LID17'}
],
COLUMNS: [
{
FIELD: "ID",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID01"},
{TYPE: MISSING, ERROR_CODE: "LID02"},
]
},
{
FIELD: "Full reference",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID03"},
]
},
{
FIELD: "Authors",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID05"},
]
},
{
FIELD: "Title",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID07"},
]
},
{
FIELD: "Journal",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID09"},
]
},
{
FIELD: "Year",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID11"},
]
},
{
FIELD: "Volume",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID13"},
]
},
{
FIELD: "Issue",
VALIDATION: []
},
{
FIELD: "First page",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "LID15"},
{TYPE: MISSING, ERROR_CODE: "LID16"},
]
},
{
FIELD: "Last page",
VALIDATION: []
},
{
FIELD: "Book title",
VALIDATION: []
},
{
FIELD: "Editors",
VALIDATION: []
},
{
FIELD: "Publisher",
VALIDATION: []
}
],
},
# SEXUAL_STATE_SHEET: {"acronym": "SSD", COLUMNS: []},
# RESOURCE_TYPES_VALUES: {"acronym": "RTD", COLUMNS: []},
# FORM_OF_SUPPLY_SHEET: {"acronym": "FSD", COLUMNS: []},
# PLOIDY_SHEET: {"acronym": "PLD", COLUMNS: []},
ONTOBIOTOPE: {
"acronym": "OTD",
"id_field": "ID",
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS06"},
COLUMNS: [
{
FIELD: "ID",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "OTD01"},
{TYPE: MISSING, ERROR_CODE: "OTD02"},
]
},
{
FIELD: "Name",
VALIDATION: [
{TYPE: MANDATORY, ERROR_CODE: "OTD03"},
{TYPE: MISSING, ERROR_CODE: "OTD04"},
]
},
]
},
MARKERS: {
"acronym": "MKD",
"id_field": "Acronym",
COLUMNS: [
{
FIELD: "Acronym",
VALIDATION: []
},
{
FIELD: "Marker",
VALIDATION: []
},
],
},
}
CROSS_REF_CONF = {
ONTOBIOTOPE: ['ID', 'Name'],
LITERATURE_SHEET: ['ID'],
LOCATIONS: ['Locality'],
GROWTH_MEDIA: ['Acronym'],
STRAINS: ["Accession number"],
SEXUAL_STATE_SHEET: [],
MARKERS: ["Acronym"],
}
MIRRI_20200601_VALLIDATION_CONF = {
'sheet_schema': SHEETS_SCHEMA,
'cross_ref_conf': CROSS_REF_CONF,
'keep_sheets_in_memory': [
{'sheet_name': LOCATIONS, 'indexed_by': 'Locality'}]
}