First import
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
from mirri import rgetattr
|
||||
|
||||
|
||||
def validate_strain(strain, version='20200601'):
|
||||
if version == '20200601':
|
||||
return _validate_strain_v20200601(strain)
|
||||
raise NotImplementedError('Only v20200601 is implemented')
|
||||
|
||||
|
||||
def _validate_strain_v20200601(strain):
|
||||
mandatory_attrs = [{'label': 'Accession Number', 'attr': 'id.strain_id'},
|
||||
{'label': 'Nagoya protocol', 'attr': 'nagoya_protocol'},
|
||||
{'label': 'Restriction on use', 'attr': 'restriction_on_use'},
|
||||
{'label': 'Risk group', 'attr': 'risk_group'},
|
||||
{'label': 'Organism type', 'attr': 'taxonomy.organism_type'},
|
||||
{'label': 'Taxon name', 'attr': 'taxonomy.long_name'},
|
||||
{'label': 'Recommended temperature to growth', 'attr': 'growth.recommended_temp'},
|
||||
{'label': 'Recommended media', 'attr': 'growth.recommended_media'},
|
||||
{'label': 'Form of supply', 'attr': 'form_of_supply'},
|
||||
{'label': 'Country', 'attr': 'collect.location.country'}]
|
||||
|
||||
errors = []
|
||||
|
||||
for mandatory in mandatory_attrs:
|
||||
value = rgetattr(strain, mandatory['attr'])
|
||||
if value is None:
|
||||
errors.append(f"{mandatory['label']} is mandatory field")
|
||||
|
||||
if not is_valid_nagoya(strain):
|
||||
errors.append('Not compliant wih nagoya protocol requirements')
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def is_valid_nagoya(strain):
|
||||
# nagoya_requirements
|
||||
_date = strain.collect.date
|
||||
if _date is None:
|
||||
_date = strain.isolation.date
|
||||
if _date is None:
|
||||
_date = strain.deposit.date
|
||||
if _date is None:
|
||||
_date = strain.catalog_inclusion_date
|
||||
# print(_date)
|
||||
year = None if _date is None else _date._year
|
||||
|
||||
if year is not None and year >= 2014 and strain.collect.location.country is None:
|
||||
return False
|
||||
|
||||
return True
|
||||
@@ -0,0 +1,3 @@
|
||||
from .error import Entity, Error
|
||||
from .error_message import ErrorMessage
|
||||
from .error_log import ErrorLog
|
||||
@@ -0,0 +1,119 @@
|
||||
from typing import Optional
|
||||
from .error_message import ErrorMessage
|
||||
|
||||
|
||||
class Entity():
|
||||
"""Entity information
|
||||
|
||||
Args:
|
||||
acronym: acronym of the entity. Must be a 3-characters captalized string
|
||||
"""
|
||||
|
||||
def __init__(self, acronym: str) -> None:
|
||||
self.acronym = acronym
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"Entity {self.acronym}: {self.name}"
|
||||
|
||||
@property
|
||||
def _acronyms(self) -> list:
|
||||
return [
|
||||
func
|
||||
for func in dir(self)
|
||||
if func.isupper() and
|
||||
callable(getattr(self, func)) and
|
||||
not func.startswith("__")
|
||||
]
|
||||
|
||||
@property
|
||||
def _names(self) -> dict:
|
||||
return {acr: getattr(self, acr)() for acr in self._acronyms}
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
try:
|
||||
return self._names[self.acronym]
|
||||
except KeyError:
|
||||
raise KeyError(f'Unknown acronym {self.acronym}.')
|
||||
|
||||
@property
|
||||
def acronym(self) -> str:
|
||||
return self._acronym
|
||||
|
||||
@acronym.setter
|
||||
def acronym(self, acronym: str) -> None:
|
||||
self._acronym = acronym
|
||||
|
||||
def EFS(self) -> str:
|
||||
return 'Excel File Structure'
|
||||
|
||||
def GMD(self) -> str:
|
||||
return 'Growth Media'
|
||||
|
||||
def GOD(self) -> str:
|
||||
return 'Geographic Origin'
|
||||
|
||||
def LID(self) -> str:
|
||||
return 'Literature'
|
||||
|
||||
def STD(self) -> str:
|
||||
return 'Strains'
|
||||
|
||||
def GID(self) -> str:
|
||||
return 'Genomic Information'
|
||||
|
||||
def OTD(self) -> str:
|
||||
return 'Ontobiotope'
|
||||
|
||||
def UCT(self) -> str:
|
||||
return 'Uncategorized'
|
||||
|
||||
|
||||
class Error():
|
||||
"""Error information
|
||||
|
||||
Args:
|
||||
message (str): Error message
|
||||
entity (Entity, optional): Entity related to the error. If None will default to Uncategorized. Defaults to None.
|
||||
data (str, optional): Data used for sorting the messages. Defaults to None.
|
||||
"""
|
||||
|
||||
def __init__(self, code: str, pk: Optional[str] = None, data: Optional[str] = None) -> None:
|
||||
self.code = code.upper()
|
||||
self.pk = pk
|
||||
self.data = data
|
||||
|
||||
def __str__(self):
|
||||
return f"Error {self._code}: {self.message}"
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
return self._code
|
||||
|
||||
@code.setter
|
||||
def code(self, code: str) -> None:
|
||||
self._code = code.upper()
|
||||
|
||||
@property
|
||||
def pk(self) -> Optional[str]:
|
||||
return self._pk
|
||||
|
||||
@pk.setter
|
||||
def pk(self, pk: Optional[str] = None) -> None:
|
||||
self._pk = pk
|
||||
|
||||
@property
|
||||
def data(self) -> Optional[str]:
|
||||
return self._data
|
||||
|
||||
@data.setter
|
||||
def data(self, data: Optional[str]):
|
||||
self._data = data
|
||||
|
||||
@property
|
||||
def entity(self) -> Entity:
|
||||
return Entity(self.code[:3])
|
||||
|
||||
@property
|
||||
def message(self) -> str:
|
||||
return ErrorMessage(self.code, self.pk, self.data).message
|
||||
@@ -0,0 +1,77 @@
|
||||
from typing import Optional, Union
|
||||
from datetime import datetime
|
||||
from .error import Error
|
||||
|
||||
|
||||
class ErrorLog():
|
||||
def __init__(self, input_filename: str, cc: Optional[str] = None, date: Optional[Union[str, datetime]] = None, limit: int = 100):
|
||||
"""
|
||||
Logger for Error instances.
|
||||
|
||||
Args:
|
||||
input_filename (str): name of the file to be logged
|
||||
cc (str, optional): name of the curator. Defaults to None.
|
||||
date (str, optional): date (e.g. created, last modified) associated with the file. Useful for versioning. Defaults to None.
|
||||
limit (int, optional): limit of errors to print to the report. Defaults to 100.
|
||||
"""
|
||||
self._input_filename = input_filename
|
||||
self._cc = cc
|
||||
self._date = date
|
||||
self._errors = {}
|
||||
self.limit = limit
|
||||
self._counter = 0
|
||||
|
||||
def __str__(self) -> str:
|
||||
output = f"""Error Log for file {self._input_filename}\nENTITY | CODE | MESSAGE"""
|
||||
for acronym, error_list in self.get_errors().items():
|
||||
for error in error_list:
|
||||
output += f"\n{acronym:6} | {error.code:6} | {error.message[:100]}"
|
||||
return output
|
||||
|
||||
@property
|
||||
def input_filename(self) -> str:
|
||||
return self._input_filename
|
||||
|
||||
@input_filename.setter
|
||||
def input_filename(self, input_filename: str) -> None:
|
||||
self._input_filename = input_filename
|
||||
|
||||
@property
|
||||
def cc(self) -> Optional[str]:
|
||||
return self._cc
|
||||
|
||||
@cc.setter
|
||||
def cc(self, cc: Optional[str]) -> None:
|
||||
self._cc = cc
|
||||
|
||||
@property
|
||||
def date(self) -> Optional[Union[str, datetime]]:
|
||||
return self._date
|
||||
|
||||
@date.setter
|
||||
def date(self, date: Optional[Union[str, datetime]] = None) -> None:
|
||||
if isinstance(date, str):
|
||||
self._date = datetime.strptime(date, r'%d-%m-%Y')
|
||||
else:
|
||||
self._date = date
|
||||
|
||||
def get_errors(self) -> dict:
|
||||
"""
|
||||
Get all errors
|
||||
|
||||
Returns:
|
||||
dict: Error intances grouped by entity acronym.
|
||||
"""
|
||||
return self._errors
|
||||
|
||||
def add_error(self, error: Error) -> None:
|
||||
"""
|
||||
Add an error.
|
||||
|
||||
Args:
|
||||
error (Error): Error instance.
|
||||
"""
|
||||
if error.entity.acronym not in self._errors:
|
||||
self._errors[error.entity.acronym] = [error]
|
||||
else:
|
||||
self._errors[error.entity.acronym].append(error)
|
||||
@@ -0,0 +1,408 @@
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ErrorMessage():
|
||||
"""Error message
|
||||
|
||||
Args:
|
||||
code (str): Error code.
|
||||
pk (str | optional): The instance's primary key that triggered the error. Defaults to None.
|
||||
value (str | optional): The instance's value that triggered the error. Defaults to None.
|
||||
"""
|
||||
|
||||
def __init__(self, code: str, pk: Optional[str] = None, value: Optional[str] = None):
|
||||
self.code = code.upper()
|
||||
self.pk = pk
|
||||
self.value = value
|
||||
|
||||
@property
|
||||
def _codes(self) -> list:
|
||||
return [
|
||||
func
|
||||
for func in dir(self)
|
||||
if func.isupper() and
|
||||
callable(getattr(self, func)) and
|
||||
not func.startswith("__")
|
||||
]
|
||||
|
||||
@property
|
||||
def _messages(self) -> dict:
|
||||
return {code: getattr(self, code) for code in self._codes}
|
||||
|
||||
@property
|
||||
def message(self) -> str:
|
||||
if not self._validate_code():
|
||||
raise ValueError(f"{self.code} not found")
|
||||
return self._messages[self.code]()
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
return self._code
|
||||
|
||||
@code.setter
|
||||
def code(self, code: str) -> None:
|
||||
self._code = code.upper()
|
||||
|
||||
def _validate_code(self) -> bool:
|
||||
return self.code in self._codes
|
||||
|
||||
@property
|
||||
def pk(self) -> str:
|
||||
return self._pk
|
||||
|
||||
@pk.setter
|
||||
def pk(self, pk: str) -> None:
|
||||
self._pk = pk
|
||||
|
||||
@property
|
||||
def value(self) -> str:
|
||||
return self._value
|
||||
|
||||
@value.setter
|
||||
def value(self, value: str) -> None:
|
||||
self._value = value
|
||||
|
||||
"""
|
||||
Excel File Structure Error Codes
|
||||
"""
|
||||
|
||||
def EXL00(self):
|
||||
return f"The provided file '{self.pk}' is not an excel(xlsx) file"
|
||||
|
||||
def EFS01(self):
|
||||
return "The 'Growth media' sheet is missing. Please check the provided excel template."
|
||||
|
||||
def EFS02(self):
|
||||
return "The 'Geographic origin' sheet is missing. Please check the provided excel template."
|
||||
|
||||
def EFS03(self):
|
||||
return "The 'Literature' sheet is missing. Please check the provided excel template."
|
||||
|
||||
def EFS04(self):
|
||||
return "The 'Sexual state' sheet is missing. Please check the provided excel template."
|
||||
|
||||
def EFS05(self):
|
||||
return "The 'Strains' sheet is missing. Please check the provided excel template."
|
||||
|
||||
def EFS06(self):
|
||||
return "The 'Ontobiotope' sheet is missing. Please check the provided excel template."
|
||||
|
||||
def EFS07(self):
|
||||
return "The 'Markers' sheet is missing. Please check the provided excel template."
|
||||
|
||||
def EFS08(self):
|
||||
return "The 'Genomic information' sheet is missing. Please check the provided excel template."
|
||||
|
||||
"""
|
||||
Growth Media Error Codes
|
||||
"""
|
||||
|
||||
def GMD01(self):
|
||||
return "The 'Acronym' column is a mandatory field in the Growth Media sheet."
|
||||
|
||||
def GMD02(self):
|
||||
return "The 'Acronym' column is empty or has missing values."
|
||||
|
||||
def GMD03(self):
|
||||
return "The 'Description' column is a mandatory field in the Growth Media sheet. The column can not be empty."
|
||||
|
||||
def GMD04(self):
|
||||
return f"The 'Description' for growth media with Acronym {self.pk} is missing."
|
||||
|
||||
"""
|
||||
Geographic Origin Error Codes
|
||||
"""
|
||||
|
||||
def GOD01(self):
|
||||
return "The 'ID' column is a mandatory field in the Geographic Origin sheet."
|
||||
|
||||
def GOD02(self):
|
||||
return "The 'ID' column is empty or has missing values."
|
||||
|
||||
def GOD03(self):
|
||||
return "The 'Country' column is a mandatory field in the Geographic Origin sheet. The column can not be empty."
|
||||
|
||||
def GOD04(self):
|
||||
return f"The 'Country' for geographic origin with ID {self.pk} is missing."
|
||||
|
||||
def GOD05(self):
|
||||
return f"The 'Country' for geographic origin with ID {self.pk} is incorrect."
|
||||
|
||||
def GOD06(self):
|
||||
return f"The 'Locality' column is a mandatory field in the Geographic Origin sheet. The column can not be empty."
|
||||
|
||||
def GOD07(self):
|
||||
return f"The 'Locality' for geographic origin with ID {self.pk} is missing."
|
||||
|
||||
"""
|
||||
Literature Error Codes
|
||||
"""
|
||||
|
||||
def LID01(self):
|
||||
return "The 'ID' column is a mandatory field in the Literature sheet."
|
||||
|
||||
def LID02(self):
|
||||
return "The 'ID' column empty or missing values."
|
||||
|
||||
def LID03(self):
|
||||
return "The 'Full reference' column is a mandatory field in the Literature sheet. The column can not be empty."
|
||||
|
||||
def LID04(self):
|
||||
return f"The 'Full reference' for literature with ID {self.pk} is missing."
|
||||
|
||||
def LID05(self):
|
||||
return "The 'Authors' column is a mandatory field in the Literature sheet. The column can not be empty."
|
||||
|
||||
def LID06(self):
|
||||
return f"The 'Authors' for literature with ID {self.pk} is missing."
|
||||
|
||||
def LID07(self):
|
||||
return "The 'Title' column is a mandatory field in the Literature sheet. The column can not be empty."
|
||||
|
||||
def LID08(self):
|
||||
return f"The 'Title' for literature with ID {self.pk} is missing."
|
||||
|
||||
def LID09(self):
|
||||
return "The 'Journal' column is a mandatory field in the Literature sheet. The column can not be empty."
|
||||
|
||||
def LID10(self):
|
||||
return f"The 'Journal' for literature with ID {self.pk} is missing."
|
||||
|
||||
def LID11(self):
|
||||
return "The 'Year' column is a mandatory field in the Literature sheet. The column can not be empty."
|
||||
|
||||
def LID12(self,):
|
||||
return f"The 'Year' for literature with ID {self.pk} is missing."
|
||||
|
||||
def LID13(self):
|
||||
return "The 'Volume' column is a mandatory field in the Literature sheet. The column can not be empty."
|
||||
|
||||
def LID14(self):
|
||||
return f"The 'Volume' for literature with ID {self.pk} is missing."
|
||||
|
||||
def LID15(self):
|
||||
return "The 'First page' column is a mandatory field. The column can not be empty."
|
||||
|
||||
def LID16(self):
|
||||
return f"The 'First page' for literature with ID {self.pk} is missing."
|
||||
|
||||
def LID17(self):
|
||||
msg = 'If journal; Title, Authors, journal, year and first page are required'
|
||||
msg += 'If Book; Book Title, Authors, Year, Editors, Publishers'
|
||||
return msg
|
||||
|
||||
"""
|
||||
Strains Error Codes
|
||||
"""
|
||||
|
||||
def STD01(self):
|
||||
return "The 'Accession number' column is a mandatory field in the Strains sheet."
|
||||
|
||||
def STD02(self):
|
||||
return "The 'Accession number' column is empty or has missing values."
|
||||
|
||||
def STD03(self):
|
||||
return f"The 'Accesion number' must be unique. The '{self.value}' is repeated."
|
||||
|
||||
def STD04(self):
|
||||
return (f"The 'Accession number' {self.pk} is not according to the specification."
|
||||
" The value must be of the format '<Sequence of characters> <sequence of characters>'.")
|
||||
|
||||
def STD05(self):
|
||||
return f"The 'Restriction on use' column is a mandatory field in the Strains Sheet. The column can not be empty."
|
||||
|
||||
def STD06(self):
|
||||
return f"The 'Restriction on use' for strain with Accession Number {self.pk} is missing."
|
||||
|
||||
def STD07(self):
|
||||
return (f"The 'Restriction on use' for strain with Accession Number {self.pk} is not according to the specification."
|
||||
f" Your value is {self.value} and the accepted values are 1, 2, 3.")
|
||||
|
||||
def STD08(self):
|
||||
return f"The 'Nagoya protocol restrictions and compliance conditions' column is a mandatory field in the Strains Sheet. The column can not be empty."
|
||||
|
||||
def STD09(self):
|
||||
return f"The 'Nagoya protocol restrictions and compliance conditions' for strain with Accession Number {self.pk} is missing."
|
||||
|
||||
def STD10(self):
|
||||
return (f"The 'Nagoya protocol restrictions and compliance conditions' for strain with Accession Number {self.pk} is not according to the specification."
|
||||
f" Your value is {self.value} and the accepted values are 1, 2, 3.")
|
||||
|
||||
def STD11(self):
|
||||
return (f"The 'Strain from a registered collection' for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 1, 2, 3.")
|
||||
|
||||
def STD12(self):
|
||||
return "The 'Risk group' column is a mandatory field in the Strains Sheet. The column can not be empty."
|
||||
|
||||
def STD13(self):
|
||||
return f"The 'Risk group' for strain with Accession Number {self.pk} is missing."
|
||||
|
||||
def STD14(self):
|
||||
return (f"The 'Risk group' for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 1, 2, 3, 4.")
|
||||
|
||||
def STD15(self):
|
||||
return (f"The 'Dual use' for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 1, 2.")
|
||||
|
||||
def STD16(self):
|
||||
return (f"The “Quarantine in europe” for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 1, 2.")
|
||||
|
||||
def STD17(self):
|
||||
return f"The 'Organism type' column is a mandatory field in the Strains Sheet. The column can not be empty."
|
||||
|
||||
def STD18(self):
|
||||
return f"The 'Organism type' for strain with Accession Number {self.pk} is missing."
|
||||
|
||||
def STD19(self):
|
||||
return (f"The 'Organism type' for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 'Algae', 'Archaea', 'Bacteria', 'Cyanobacteria', "
|
||||
"'Filamentous Fungi', 'Phage', 'Plasmid', 'Virus', 'Yeast', 1, 2, 3, 4, 5, 6, 7, 8, 9.")
|
||||
|
||||
def STD20(self):
|
||||
return f"The 'Taxon name' column is a mandatory field in the Strains Sheet. The column can not be empty."
|
||||
|
||||
def STD21(self):
|
||||
return f"The 'Taxon name' for strain with Accession Number {self.pk} is missing."
|
||||
|
||||
def STD22(self):
|
||||
return f"The 'Taxon name' for strain with Accession Number {self.pk} is incorrect."
|
||||
|
||||
def STD23(self):
|
||||
return (f"The 'Interspecific hybrid' for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 1, 2.")
|
||||
|
||||
def STD24(self):
|
||||
return f"The 'History of deposit' for strain with Accession Number {self.pk} is incorrect."
|
||||
|
||||
def STD25(self):
|
||||
return (f"The 'Date of deposit' for strain with Accession Number {self.pk} is incorrect."
|
||||
" The allowed formats are 'YYYY-MM-DD', 'YYYYMMDD', 'YYYYMM', and 'YYYY'.")
|
||||
|
||||
def STD26(self):
|
||||
return (f"The 'Date of inclusion in the catalogue' for strain with Accession Number {self.pk} is incorrect."
|
||||
" The allowed formats are 'YYYY-MM-DD', 'YYYYMMDD', 'YYYYMM', and 'YYYY'.")
|
||||
|
||||
def STD27(self):
|
||||
return (f"The 'Date of collection' for strain with Accession Number {self.pk} is incorrect."
|
||||
" The allowed formats are 'YYYY-MM-DD', 'YYYYMMDD', 'YYYYMM', and 'YYYY'.")
|
||||
|
||||
def STD28(self):
|
||||
return (f"The 'Date of isolation' for strain with Accession Number {self.pk} is incorrect."
|
||||
" The allowed formats are 'YYYY-MM-DD', 'YYYYMMDD', 'YYYYMM', and 'YYYY'.")
|
||||
|
||||
def STD29(self):
|
||||
return (f"The 'Tested temperature growth range' for strain with Accession Number {self.pk} is incorrect."
|
||||
" It must have two decimal numbers separated by ','")
|
||||
|
||||
def STD30(self):
|
||||
return f"The 'Recommended growth temperature' column is a mandatory field in the Strains Sheet. The column can not be empty."
|
||||
|
||||
def STD31(self):
|
||||
return f"The 'Recommended growth temperature' for strain with Accession Number {self.pk} is missing."
|
||||
|
||||
def STD32(self):
|
||||
return (f"The 'Recommended growth temperature' for strain with Accession Number {self.pk} is incorrect."
|
||||
" It must have two decimal numbers separated by ','.")
|
||||
|
||||
def STD33(self):
|
||||
return f"The 'Recommended medium for growth' column is a mandatory field in the Strains Sheet. The column can not be empty."
|
||||
|
||||
def STD34(self):
|
||||
return f"The 'Recommended medium for growth' for strain with Accession Number {self.pk} is missing."
|
||||
|
||||
def STD35(self):
|
||||
return f"The value of 'Recommended medium for growth' for strain with Accession Number {self.pk} is not in the Growth Media Sheet."
|
||||
|
||||
def STD36(self):
|
||||
return f"The 'Forms of supply' column is a mandatory field in the Strains Sheet. The column can not be empty."
|
||||
|
||||
def STD37(self):
|
||||
return f"The 'Forms of supply' for strain with Accession Number {self.pk} is missing."
|
||||
|
||||
def STD38(self):
|
||||
return f"The value of 'Forms of supply' for strain with Accession Number {self.pk} is not in the Forms of Supply Sheet."
|
||||
|
||||
def STD39(self):
|
||||
return (f"The 'Coordinates of geographic origin' column for strain with Accession Number {self.pk} is incorrect."
|
||||
"The allowed formats are two or three decimal numbers separated by ','. Moreover, the first number must be"
|
||||
"between [-90, 90], the second between [-180, 180], and the third, if provided, can assume any value.")
|
||||
|
||||
def STD40(self):
|
||||
return (f"The 'Altitude of geographic origin' column for strain with Accession Number {self.pk} is incorrect."
|
||||
"The allowed formats are one decimal number between [-200, 8000].")
|
||||
|
||||
def STD41(self):
|
||||
return f"The value of 'Ontobiotope term for the isolation habitat' for strain with Accession Number {self.pk} is not in the Ontobiotope Sheet."
|
||||
|
||||
def STD42(self):
|
||||
return (f"The 'GMO' for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 1, 2")
|
||||
|
||||
def STD43(self):
|
||||
return (f"The 'Sexual State' for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 'Mata', 'Matalpha', 'Mata/Matalpha', "
|
||||
"'Matb', 'Mata/Matb', 'MTLa', 'MTLalpha', 'MTLa/MTLalpha', 'MAT1-1', 'MAT1-2', 'MAT1', 'MAT2', 'MT+', 'MT-'")
|
||||
|
||||
def STD44(self):
|
||||
return (f"The 'Ploidy' for strain with Accession Number {self.pk} is not according to specification."
|
||||
f" Your value is {self.value} and the accepted values are 0, 1, 2, 3, 4, 9")
|
||||
|
||||
def STD45(self):
|
||||
msg = f"At least one of the values '{self.value}' of the literature field for strain {self.pk} are not in the literature sheet. "
|
||||
msg += "If the those values are Pubmed ids or DOIs, please ignore this messsage"
|
||||
return msg
|
||||
|
||||
|
||||
"""
|
||||
Genomic Information Error Codes
|
||||
"""
|
||||
|
||||
def GID01(self):
|
||||
return f"The 'Strain Acession Number' (Strain AN) column is a mandatory field in the Genomic Information Sheet."
|
||||
|
||||
def GID02(self):
|
||||
return f"The 'Strain Acession Number' (Strain AN) column is empty or has missing values."
|
||||
|
||||
def GID03(self):
|
||||
return f"The value of 'Strain Acession Number' (Strain AN) {self.value} is not in the Strains sheet."
|
||||
|
||||
def GID04(self):
|
||||
return f"The 'Marker' column is a mandatory field in the Genomic Information Sheet. The column can not be empty."
|
||||
|
||||
def GID05(self):
|
||||
return f"The 'Marker' for genomic information with Strain AN {self.pk} is missing."
|
||||
|
||||
def GID06(self):
|
||||
return f"The 'Marker' for genomic information with Strain AN {self.pk} is incorrect."
|
||||
|
||||
def GID07(self):
|
||||
return f"The 'INSDC AN' column is a mandatory field in the Genomic Information Sheet. The column can not be empty."
|
||||
|
||||
def GID08(self):
|
||||
return f"The 'INSDC AN' for genomic information with Strain AN {self.pk} is missing."
|
||||
|
||||
def GID09(self):
|
||||
return f"The 'INSDC AN' for genomic information with Strain AN {self.pk} is incorrect."
|
||||
|
||||
def GID10(self):
|
||||
return (f"The 'Sequence' for genomic information with Strain AN {self.pk} is incorrect."
|
||||
" It must be a sequence of 'G', 'T', 'A', 'C' characteres of any length and without white spaces.")
|
||||
|
||||
"""
|
||||
Ontobiotope Error Codes
|
||||
"""
|
||||
|
||||
def OTD01(self):
|
||||
return "The 'ID' columns is a mandatory field in the Ontobiotope Sheet."
|
||||
|
||||
def OTD02(self):
|
||||
return "The 'ID' columns is empty or has missing values."
|
||||
|
||||
def OTD03(self):
|
||||
return "The 'Name' columns is a mandatory field in the Ontobiotope Sheet. The column can not be empty."
|
||||
|
||||
def OTD04(self):
|
||||
return f"The 'Name' for ontobiotope with ID {self.pk} is missing."
|
||||
@@ -0,0 +1,483 @@
|
||||
import re
|
||||
from pathlib import Path
|
||||
from io import BytesIO
|
||||
from zipfile import BadZipfile
|
||||
from datetime import datetime
|
||||
from calendar import monthrange
|
||||
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from mirri.io.parsers.excel import workbook_sheet_reader, get_all_cell_data_from_sheet
|
||||
from mirri.validation.error_logging import ErrorLog, Error
|
||||
from mirri.validation.tags import (CHOICES, COLUMNS, COORDINATES, CROSSREF, CROSSREF_NAME, DATE,
|
||||
ERROR_CODE, FIELD, MANDATORY, MATCH,
|
||||
MISSING, MULTIPLE, NAGOYA, NUMBER, REGEXP, ROW_VALIDATION, SEPARATOR, TAXON,
|
||||
TYPE, UNIQUE, VALIDATION, VALUES, BIBLIO)
|
||||
from mirri.settings import LOCATIONS, SUBTAXAS
|
||||
from mirri.validation.validation_conf_20200601 import MIRRI_20200601_VALLIDATION_CONF
|
||||
|
||||
|
||||
def validate_mirri_excel(fhand, version="20200601"):
|
||||
if version == "20200601":
|
||||
configuration = MIRRI_20200601_VALLIDATION_CONF
|
||||
else:
|
||||
raise NotImplementedError("Only version20200601 is implemented")
|
||||
|
||||
return validate_excel(fhand, configuration)
|
||||
|
||||
|
||||
def validate_excel(fhand, configuration):
|
||||
validation_conf = configuration['sheet_schema']
|
||||
cross_ref_conf = configuration['cross_ref_conf']
|
||||
in_memory_sheet_conf = configuration['keep_sheets_in_memory']
|
||||
excel_name = Path(fhand.name).stem
|
||||
error_log = ErrorLog(excel_name)
|
||||
|
||||
try:
|
||||
workbook = load_workbook(filename=BytesIO(
|
||||
fhand.read()), read_only=True, data_only=True)
|
||||
except (BadZipfile, IOError):
|
||||
error = Error('EXL00', fhand.name, fhand.name)
|
||||
error_log.add_error(error)
|
||||
return error_log
|
||||
|
||||
# excel structure errors
|
||||
structure_errors = list(validate_excel_structure(workbook, validation_conf))
|
||||
if structure_errors:
|
||||
for error in structure_errors:
|
||||
error = Error(error[ERROR_CODE], pk=error['id'],
|
||||
data=error['value'])
|
||||
error_log.add_error(error)
|
||||
|
||||
return error_log
|
||||
|
||||
crossrefs = get_all_crossrefs(workbook, cross_ref_conf)
|
||||
in_memory_sheets = get_all_in_memory_sheet(workbook, in_memory_sheet_conf)
|
||||
content_errors = validate_content(workbook, validation_conf,
|
||||
crossrefs, in_memory_sheets)
|
||||
|
||||
for error in content_errors:
|
||||
# if error[ERROR_CODE] == 'STD43':
|
||||
# continue
|
||||
error = Error(error[ERROR_CODE], pk=error['id'], data=error['value'])
|
||||
|
||||
error_log.add_error(error)
|
||||
return error_log
|
||||
|
||||
|
||||
def validate_excel_structure(workbook, validation_conf):
|
||||
for sheet_name, sheet_conf in validation_conf.items():
|
||||
mandatory = sheet_conf.get(VALIDATION, {}).get(TYPE, None)
|
||||
mandatory = mandatory == MANDATORY
|
||||
|
||||
error_code = sheet_conf.get(VALIDATION, {}).get(ERROR_CODE, False)
|
||||
try:
|
||||
sheet = workbook[sheet_name]
|
||||
except KeyError:
|
||||
sheet = None
|
||||
|
||||
if sheet is None:
|
||||
if mandatory:
|
||||
yield {'id': None, 'sheet': sheet_name, 'field': None,
|
||||
'error_code': error_code, 'value': None}
|
||||
continue
|
||||
|
||||
headers = _get_sheet_headers(sheet)
|
||||
for column in sheet_conf.get(COLUMNS):
|
||||
field = column[FIELD]
|
||||
for step in column.get(VALIDATION, []):
|
||||
if step[TYPE] == MANDATORY and field not in headers:
|
||||
yield {'id': None, 'sheet': sheet_name, 'field': field,
|
||||
'error_code': step[ERROR_CODE], 'value': None}
|
||||
|
||||
|
||||
def _get_sheet_headers(sheet):
|
||||
first_row = next(sheet.iter_rows(min_row=1, max_row=1))
|
||||
return [c.value for c in first_row]
|
||||
|
||||
|
||||
def _get_values_from_columns(workbook, sheet_name, columns):
|
||||
indexed_values = {}
|
||||
for row in workbook_sheet_reader(workbook, sheet_name):
|
||||
for col in columns:
|
||||
indexed_values[str(row.get(col))] = ""
|
||||
|
||||
return indexed_values
|
||||
|
||||
|
||||
def get_all_crossrefs(workbook, cross_refs_names):
|
||||
crossrefs = {}
|
||||
for ref_name, columns in cross_refs_names.items():
|
||||
if columns:
|
||||
crossrefs[ref_name] = _get_values_from_columns(workbook, ref_name,
|
||||
columns)
|
||||
else:
|
||||
try:
|
||||
crossrefs[ref_name] = get_all_cell_data_from_sheet(workbook, ref_name)
|
||||
except ValueError as error:
|
||||
if 'sheet is missing' in str(error):
|
||||
crossrefs[ref_name] = []
|
||||
else:
|
||||
raise
|
||||
|
||||
return crossrefs
|
||||
|
||||
|
||||
def get_all_in_memory_sheet(workbook, in_memory_sheet_conf):
|
||||
in_memory_sheets = {}
|
||||
for sheet_conf in in_memory_sheet_conf:
|
||||
sheet_name = sheet_conf['sheet_name']
|
||||
indexed_by = sheet_conf['indexed_by']
|
||||
rows = workbook_sheet_reader(workbook, sheet_name)
|
||||
indexed_rows = {row[indexed_by]: row for row in rows}
|
||||
in_memory_sheets[sheet_name] = indexed_rows
|
||||
|
||||
return in_memory_sheets
|
||||
|
||||
|
||||
def validate_content(workbook, validation_conf, crossrefs, in_memory_sheets):
|
||||
for sheet_name in validation_conf.keys():
|
||||
sheet_conf = validation_conf[sheet_name]
|
||||
sheet_id_column = sheet_conf['id_field']
|
||||
shown_values = {}
|
||||
row_validation_steps = sheet_conf.get(ROW_VALIDATION, None)
|
||||
for row in workbook_sheet_reader(workbook, sheet_name):
|
||||
id_ = row.get(sheet_id_column, None)
|
||||
if id_ is None:
|
||||
error_code = _get_missing_row_id_error(sheet_id_column,
|
||||
sheet_conf)
|
||||
yield {'id': id_, 'sheet': sheet_name,
|
||||
'field': sheet_id_column,
|
||||
'error_code': error_code, 'value': None}
|
||||
continue
|
||||
do_have_cell_error = False
|
||||
for column in sheet_conf[COLUMNS]:
|
||||
label = column[FIELD]
|
||||
validation_steps = column.get(VALIDATION, None)
|
||||
value = row.get(label, None)
|
||||
if validation_steps:
|
||||
error_code = validate_cell(value, validation_steps,
|
||||
crossrefs, shown_values, label)
|
||||
if error_code is not None:
|
||||
do_have_cell_error = True
|
||||
yield {'id': id_, 'sheet': sheet_name, 'field': label,
|
||||
'error_code': error_code, 'value': value}
|
||||
|
||||
if not do_have_cell_error and row_validation_steps:
|
||||
error_code = validate_row(
|
||||
row, row_validation_steps, in_memory_sheets)
|
||||
if error_code is not None:
|
||||
yield {'id': id_, 'sheet': sheet_name, 'field': 'row',
|
||||
'error_code': error_code, 'value': 'row'}
|
||||
|
||||
|
||||
def _get_missing_row_id_error(sheet_id_column, sheet_conf):
|
||||
error_code = None
|
||||
for column in sheet_conf[COLUMNS]:
|
||||
if column[FIELD] == sheet_id_column:
|
||||
error_code = [step[ERROR_CODE]
|
||||
for step in column[VALIDATION] if step[TYPE] == MISSING][0]
|
||||
return error_code
|
||||
|
||||
|
||||
def validate_row(row, validation_steps, in_memory_sheets):
|
||||
for validation_step in validation_steps:
|
||||
kind = validation_step[TYPE]
|
||||
error_code = validation_step[ERROR_CODE]
|
||||
if kind == NAGOYA:
|
||||
if not is_valid_nagoya(row, in_memory_sheets):
|
||||
return error_code
|
||||
elif kind == BIBLIO:
|
||||
if not is_valid_pub(row):
|
||||
return error_code
|
||||
else:
|
||||
msg = f'{kind} is not a recognized row validation type method'
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
|
||||
def validate_cell(value, validation_steps, crossrefs, shown_values, label):
|
||||
|
||||
for step_conf in validation_steps:
|
||||
if step_conf[TYPE] == MANDATORY:
|
||||
continue
|
||||
step_conf['crossrefs_pointer'] = crossrefs
|
||||
step_conf['shown_values'] = shown_values
|
||||
step_conf['label'] = label
|
||||
error_code = validate_value(value, step_conf)
|
||||
|
||||
if error_code is not None:
|
||||
return error_code
|
||||
|
||||
|
||||
def is_valid_pub(row):
|
||||
title = row.get('Title', None)
|
||||
full_reference = row.get('Full reference', None)
|
||||
authors = row.get('Authors', None)
|
||||
journal = row.get('Journal', None)
|
||||
year = row.get('Year', None)
|
||||
volumen = row.get('Volumen', None)
|
||||
first_page = row.get('First page', None)
|
||||
book_title = row.get('Book title', None)
|
||||
editors = row.get('Editors', None)
|
||||
publishers = row.get('Publishers', None)
|
||||
|
||||
if full_reference:
|
||||
return True
|
||||
is_journal = bool(title)
|
||||
|
||||
if (is_journal and (not authors or not journal or not not year or
|
||||
not volumen or not first_page)):
|
||||
return False
|
||||
if (not is_journal and (not authors or not year or
|
||||
not editors or not publishers or not book_title)):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_nagoya(row, in_memory_sheets): # sourcery skip: return-identity
|
||||
location_index = row.get('Geographic origin', None)
|
||||
if location_index is None:
|
||||
country = None
|
||||
else:
|
||||
geo_origin = in_memory_sheets[LOCATIONS].get(location_index, {})
|
||||
country = geo_origin.get('Country', None)
|
||||
|
||||
_date = row.get("Date of collection", None)
|
||||
if _date is None:
|
||||
_date = row.get("Date of isolation", None)
|
||||
if _date is None:
|
||||
_date = row.get("Date of deposit", None)
|
||||
if _date is None:
|
||||
_date = row.get("Date of inclusion in the catalogue", None)
|
||||
if _date is not None:
|
||||
year = _date.year if isinstance(_date, datetime) else int(str(_date)[:4])
|
||||
else:
|
||||
year = None
|
||||
|
||||
if year is not None and year >= 2014 and country is None:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_regex(value, validation_conf):
|
||||
if value is None:
|
||||
return True
|
||||
value = str(value)
|
||||
regexp = validation_conf[MATCH]
|
||||
multiple = validation_conf.get(MULTIPLE, False)
|
||||
separator = validation_conf.get(SEPARATOR, None)
|
||||
|
||||
values = [v.strip() for v in value.split(
|
||||
separator)] if multiple else [value]
|
||||
|
||||
for value in values:
|
||||
matches_regexp = re.fullmatch(regexp, value)
|
||||
if not matches_regexp:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_crossrefs(value, validation_conf):
|
||||
crossref_name = validation_conf[CROSSREF_NAME]
|
||||
crossrefs = validation_conf['crossrefs_pointer']
|
||||
choices = crossrefs[crossref_name]
|
||||
if value is None or not choices:
|
||||
return True
|
||||
value = str(value)
|
||||
|
||||
multiple = validation_conf.get(MULTIPLE, False)
|
||||
separator = validation_conf.get(SEPARATOR, None)
|
||||
if value is None:
|
||||
return True
|
||||
if multiple:
|
||||
values = [v.strip() for v in value.split(separator)]
|
||||
else:
|
||||
values = [value.strip()]
|
||||
|
||||
return all(value in choices for value in values)
|
||||
|
||||
|
||||
def is_valid_choices(value, validation_conf):
|
||||
if value is None:
|
||||
return True
|
||||
choices = validation_conf[VALUES]
|
||||
multiple = validation_conf.get(MULTIPLE, False)
|
||||
separator = validation_conf.get(SEPARATOR, None)
|
||||
|
||||
if multiple:
|
||||
values = [v.strip() for v in str(value).split(separator)]
|
||||
else:
|
||||
values = [str(value).strip()]
|
||||
|
||||
return all(value in choices for value in values)
|
||||
|
||||
|
||||
def is_valid_date(value, validation_conf):
|
||||
if value is None:
|
||||
return True
|
||||
if isinstance(value, datetime):
|
||||
year = value.year
|
||||
month = value.month
|
||||
day = value.day
|
||||
elif isinstance(value, int):
|
||||
year = value
|
||||
month = None
|
||||
day = None
|
||||
elif isinstance(value, str):
|
||||
value = value.replace('-', '')
|
||||
value = value.replace('/', '')
|
||||
month = None
|
||||
day = None
|
||||
try:
|
||||
year = int(value[: 4])
|
||||
if len(value) >= 6:
|
||||
month = int(value[4: 6])
|
||||
if len(value) >= 8:
|
||||
day = int(value[6: 8])
|
||||
|
||||
except (IndexError, TypeError, ValueError):
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
|
||||
if year < 1700 or year > datetime.now().year:
|
||||
return False
|
||||
if month is not None:
|
||||
if month < 1 or month > 13:
|
||||
return False
|
||||
if day is not None and (day < 1 or day > monthrange(year, month)[1]):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_coords(value, validation_conf=None):
|
||||
# sourcery skip: return-identity
|
||||
if value is None:
|
||||
return True
|
||||
try:
|
||||
items = [i.strip() for i in value.split(";")]
|
||||
latitude = float(items[0])
|
||||
longitude = float(items[1])
|
||||
if len(items) > 2:
|
||||
precision = float(items[2])
|
||||
if latitude < -90 or latitude > 90:
|
||||
return False
|
||||
if longitude < -180 or longitude > 180:
|
||||
return False
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def is_valid_missing(value, validation_conf=None):
|
||||
return value is not None
|
||||
|
||||
|
||||
def is_valid_number(value, validation_conf):
|
||||
if value is None:
|
||||
return True
|
||||
try:
|
||||
value = float(value)
|
||||
except TypeError:
|
||||
return False
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
_max = validation_conf.get('max', None)
|
||||
_min = validation_conf.get('min', None)
|
||||
if (_max is not None and value > _max) or (_min is not None and value < _min):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_taxon(value, validation_conf=None):
|
||||
multiple = validation_conf.get(MULTIPLE, False)
|
||||
separator = validation_conf.get(SEPARATOR, ';')
|
||||
|
||||
value = value.split(separator) if multiple else [value]
|
||||
for taxon in value:
|
||||
taxon = taxon.strip()
|
||||
if not _is_valid_taxon(taxon):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_valid_taxon(value):
|
||||
value = value.strip()
|
||||
if not value:
|
||||
return True
|
||||
|
||||
items = re.split(r" +", value)
|
||||
genus = items[0]
|
||||
|
||||
if len(items) > 1:
|
||||
species = items[1]
|
||||
if species in ("sp", "spp", ".sp", "sp."):
|
||||
return False
|
||||
|
||||
if len(items) > 2:
|
||||
for index in range(0, len(items[2:]), 2):
|
||||
rank = SUBTAXAS.get(items[index + 2], None)
|
||||
if rank is None:
|
||||
print(value)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_unique(value, validation_conf):
|
||||
label = validation_conf['label']
|
||||
shown_values = validation_conf['shown_values']
|
||||
if label not in shown_values:
|
||||
shown_values[label] = {}
|
||||
|
||||
already_in_file = shown_values[label]
|
||||
if value in already_in_file:
|
||||
return False
|
||||
|
||||
# NOTE: what's the use of this?
|
||||
# What is the expected format for value and shown_values?
|
||||
shown_values[label][value] = None
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_file(path):
|
||||
try:
|
||||
with path.open("rb") as fhand:
|
||||
error_log = validate_mirri_excel(fhand)
|
||||
if "EXL" in error_log.get_errors():
|
||||
return False
|
||||
except:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
VALIDATION_FUNCTIONS = {
|
||||
MISSING: is_valid_missing,
|
||||
REGEXP: is_valid_regex,
|
||||
CHOICES: is_valid_choices,
|
||||
CROSSREF: is_valid_crossrefs,
|
||||
DATE: is_valid_date,
|
||||
COORDINATES: is_valid_coords,
|
||||
NUMBER: is_valid_number,
|
||||
TAXON: is_valid_taxon,
|
||||
UNIQUE: is_valid_unique}
|
||||
|
||||
|
||||
def validate_value(value, step_conf):
|
||||
kind = step_conf[TYPE]
|
||||
try:
|
||||
is_value_valid = VALIDATION_FUNCTIONS[kind]
|
||||
except KeyError:
|
||||
msg = f'This validation type {kind} is not implemented'
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
error_code = step_conf[ERROR_CODE]
|
||||
|
||||
if not is_value_valid(value, step_conf):
|
||||
return error_code
|
||||
@@ -0,0 +1,24 @@
|
||||
MANDATORY = "mandatory"
|
||||
REGEXP = "regexp"
|
||||
CHOICES = "choices"
|
||||
CROSSREF = 'crossref'
|
||||
CROSSREF_NAME = 'crossref_name'
|
||||
MISSING = "missing"
|
||||
VALIDATION = 'validation'
|
||||
ERROR_CODE = 'error_code'
|
||||
FIELD = 'field'
|
||||
MULTIPLE = 'multiple'
|
||||
TYPE = 'type'
|
||||
COLUMNS = 'columns'
|
||||
SOURCE = "sources"
|
||||
SEPARATOR = "separator"
|
||||
MATCH = 'match'
|
||||
VALUES = 'values'
|
||||
DATE = 'date'
|
||||
COORDINATES = 'coord'
|
||||
NUMBER = 'number'
|
||||
TAXON = 'taxon'
|
||||
UNIQUE = 'unique'
|
||||
ROW_VALIDATION = 'row_validation'
|
||||
NAGOYA = 'nagoya'
|
||||
BIBLIO = 'bibliography'
|
||||
@@ -0,0 +1,548 @@
|
||||
from mirri.validation.tags import (CHOICES, COLUMNS, COORDINATES, CROSSREF, CROSSREF_NAME, DATE,
|
||||
ERROR_CODE, FIELD, MANDATORY, MATCH,
|
||||
MISSING, MULTIPLE, NAGOYA, NUMBER, REGEXP, ROW_VALIDATION, SEPARATOR, TAXON, TYPE,
|
||||
UNIQUE,
|
||||
VALIDATION, VALUES, BIBLIO)
|
||||
from mirri.settings import (GEOGRAPHIC_ORIGIN, ONTOBIOTOPE, LOCATIONS, GROWTH_MEDIA, GENOMIC_INFO,
|
||||
STRAINS, LITERATURE_SHEET, SEXUAL_STATE_SHEET)
|
||||
# MARKERS,
|
||||
# SEXUAL_STATE_SHEET,
|
||||
# RESOURCE_TYPES_VALUES,
|
||||
# FORM_OF_SUPPLY_SHEET,
|
||||
# PLOIDY_SHEET)
|
||||
|
||||
|
||||
STRAIN_FIELDS = [
|
||||
{
|
||||
FIELD: "Accession number",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: 'STD01'},
|
||||
{TYPE: UNIQUE, ERROR_CODE: 'STD03'},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD02"},
|
||||
{TYPE: REGEXP, MATCH: "[^ ]* [^ ]*", ERROR_CODE: "STD04"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Restrictions on use",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "STD05"},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD06"},
|
||||
{TYPE: CHOICES, VALUES: ["1", "2", "3"],
|
||||
MULTIPLE: False, ERROR_CODE: "STD07"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Nagoya protocol restrictions and compliance conditions",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "STD08"},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD09"},
|
||||
{TYPE: CHOICES, VALUES: ["1", "2", "3"],
|
||||
MULTIPLE: False, ERROR_CODE: "STD10"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "ABS related files",
|
||||
VALIDATION: [],
|
||||
},
|
||||
{
|
||||
FIELD: "MTA file",
|
||||
VALIDATION: [],
|
||||
},
|
||||
{
|
||||
FIELD: "Other culture collection numbers",
|
||||
# VALIDATION: [
|
||||
# {TYPE: REGEXP, "match": "[^ ]* [^ ]*", ERROR_CODE: "STD07",
|
||||
# MULTIPLE: True, SEPARATOR: ";"}
|
||||
# ]
|
||||
},
|
||||
{
|
||||
FIELD: "Strain from a registered collection",
|
||||
VALIDATION: [
|
||||
{TYPE: CHOICES, VALUES: ["1", "2"],
|
||||
ERROR_CODE: "STD11"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Risk Group",
|
||||
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "STD12"},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD13"},
|
||||
{TYPE: CHOICES, VALUES: ["1", "2", "3", "4"],
|
||||
MULTIPLE: False, ERROR_CODE: "STD14"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Dual use",
|
||||
VALIDATION: [
|
||||
{TYPE: CHOICES, VALUES: ["1", "2"],
|
||||
ERROR_CODE: "STD15"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Quarantine in Europe",
|
||||
VALIDATION: [
|
||||
{TYPE: CHOICES, VALUES: ["1", "2"],
|
||||
ERROR_CODE: "STD16"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Organism type",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "STD17"},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD18"},
|
||||
{TYPE: CHOICES, VALUES: ["Algae", "Archaea", "Bacteria",
|
||||
"Cyanobacteria", "Filamentous Fungi",
|
||||
"Phage", "Plasmid", "Virus", "Yeast",
|
||||
"1", "2", "3", "4", "5", "6", "7", "8", "9"],
|
||||
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD19"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Taxon name",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "STD20"},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD21"},
|
||||
{TYPE: TAXON, ERROR_CODE: "STD22", MULTIPLE: True,
|
||||
SEPARATOR: ';'}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Infrasubspecific names",
|
||||
},
|
||||
{
|
||||
FIELD: "Comment on taxonomy",
|
||||
},
|
||||
{
|
||||
FIELD: "Interspecific hybrid",
|
||||
VALIDATION: [
|
||||
{TYPE: CHOICES, VALUES: ["1", "2"],
|
||||
ERROR_CODE: "STD23"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Status",
|
||||
},
|
||||
{
|
||||
FIELD: "History of deposit",
|
||||
VALIDATION: [
|
||||
# {TYPE: REGEXP, "match": "[^ ]* [^ ]*", ERROR_CODE: "STD24", # modify the regex
|
||||
# MULTIPLE: True, SEPARATOR: ";"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Depositor"
|
||||
},
|
||||
{
|
||||
FIELD: "Date of deposit",
|
||||
VALIDATION: [
|
||||
{TYPE: DATE, ERROR_CODE: "STD25"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Date of inclusion in the catalogue",
|
||||
VALIDATION: [
|
||||
{TYPE: DATE, ERROR_CODE: "STD26"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Collected by",
|
||||
},
|
||||
{
|
||||
FIELD: "Date of collection",
|
||||
VALIDATION: [
|
||||
{TYPE: DATE, ERROR_CODE: "STD27"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Isolated by",
|
||||
},
|
||||
{
|
||||
FIELD: "Date of isolation",
|
||||
VALIDATION: [
|
||||
{TYPE: DATE, ERROR_CODE: "STD28"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Substrate/host of isolation",
|
||||
},
|
||||
{
|
||||
FIELD: "Tested temperature growth range",
|
||||
VALIDATION: [
|
||||
{TYPE: REGEXP, "match": r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?',
|
||||
ERROR_CODE: "STD29", MULTIPLE: True, SEPARATOR: ";"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Recommended growth temperature",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "STD30"},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD31"},
|
||||
{TYPE: REGEXP, "match": r'[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?',
|
||||
ERROR_CODE: "STD32",
|
||||
MULTIPLE: True, SEPARATOR: ";"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Recommended medium for growth",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "STD33"},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD34"},
|
||||
{TYPE: CROSSREF, CROSSREF_NAME: "Growth media",
|
||||
MULTIPLE: True, SEPARATOR: "/", ERROR_CODE: "STD35"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Form of supply",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "STD36"},
|
||||
{TYPE: MISSING, ERROR_CODE: "STD37"},
|
||||
{TYPE: CHOICES, VALUES: ['Agar', 'Cryo', 'Dry Ice', 'Liquid Culture Medium',
|
||||
'Lyo', 'Oil', 'Water'],
|
||||
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD38"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Other denomination",
|
||||
},
|
||||
{
|
||||
FIELD: "Coordinates of geographic origin",
|
||||
VALIDATION: [
|
||||
{TYPE: COORDINATES, ERROR_CODE: "STD39"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Altitude of geographic origin",
|
||||
VALIDATION: [
|
||||
{TYPE: NUMBER, 'max': 8000, 'min': -200, ERROR_CODE: "STD40"},
|
||||
]
|
||||
},
|
||||
{
|
||||
# value can be in the cell or in another sheet. Don't configure this
|
||||
FIELD: "Geographic origin",
|
||||
},
|
||||
{
|
||||
FIELD: "Isolation habitat",
|
||||
},
|
||||
{
|
||||
FIELD: "Ontobiotope term for the isolation habitat",
|
||||
VALIDATION: [
|
||||
{TYPE: CROSSREF, CROSSREF_NAME: "Ontobiotope",
|
||||
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD41"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "GMO",
|
||||
VALIDATION: [
|
||||
{TYPE: CHOICES, VALUES: ["1", "2"],
|
||||
ERROR_CODE: "STD42"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "GMO construction information",
|
||||
},
|
||||
{
|
||||
FIELD: "Mutant information",
|
||||
},
|
||||
{
|
||||
FIELD: "Genotype",
|
||||
},
|
||||
{
|
||||
FIELD: "Sexual state",
|
||||
VALIDATION: [
|
||||
{TYPE: CROSSREF, CROSSREF_NAME: SEXUAL_STATE_SHEET,
|
||||
ERROR_CODE: "STD43"}
|
||||
# {TYPE: CHOICES, VALUES: ["Mata", "Matalpha", "Mata/Matalpha",
|
||||
# "Matb", "Mata/Matb", "MTLa", "MTLalpha", "MTLa/MTLalpha",
|
||||
# "MAT1-1", "MAT1-2", "MAT1", "MAT2", "MT+", "MT-"],
|
||||
# ERROR_CODE: "STD43"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Ploidy",
|
||||
VALIDATION: [
|
||||
{TYPE: CHOICES, VALUES: ["0", "1", "2", "3", "4", "9"],
|
||||
ERROR_CODE: "STD44"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Plasmids",
|
||||
},
|
||||
{
|
||||
FIELD: "Plasmids collections fields",
|
||||
},
|
||||
{
|
||||
# value can be in the cell or in another sheet. Don't configure this
|
||||
FIELD: "Literature",
|
||||
VALIDATION: [
|
||||
{TYPE: CROSSREF, CROSSREF_NAME: LITERATURE_SHEET,
|
||||
MULTIPLE: True, SEPARATOR: ";", ERROR_CODE: "STD45"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Plant pathogenicity code",
|
||||
},
|
||||
{
|
||||
FIELD: "Pathogenicity",
|
||||
},
|
||||
{
|
||||
FIELD: "Enzyme production",
|
||||
},
|
||||
{
|
||||
FIELD: "Production of metabolites",
|
||||
},
|
||||
{
|
||||
FIELD: "Applications",
|
||||
},
|
||||
{
|
||||
FIELD: "Remarks"
|
||||
},
|
||||
{
|
||||
FIELD: "Literature linked to the sequence/genome",
|
||||
},
|
||||
]
|
||||
SHEETS_SCHEMA = {
|
||||
LOCATIONS: {
|
||||
"acronym": "GOD",
|
||||
"id_field": "ID",
|
||||
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS02"},
|
||||
COLUMNS: [
|
||||
{
|
||||
FIELD: "ID",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "GOD01"},
|
||||
{TYPE: MISSING, ERROR_CODE: "GOD02"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Country",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "GOD03"},
|
||||
{TYPE: MISSING, ERROR_CODE: "GOD04"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Region",
|
||||
VALIDATION: []
|
||||
},
|
||||
{
|
||||
FIELD: "City",
|
||||
VALIDATION: []
|
||||
},
|
||||
{
|
||||
FIELD: "Locality",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "GOD06"},
|
||||
{TYPE: MISSING, ERROR_CODE: "GOD07"}
|
||||
]
|
||||
}
|
||||
],
|
||||
},
|
||||
GROWTH_MEDIA: {
|
||||
"acronym": "GMD",
|
||||
"id_field": "Acronym",
|
||||
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS01"},
|
||||
COLUMNS: [
|
||||
{
|
||||
FIELD: "Acronym",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "GMD01"},
|
||||
{TYPE: MISSING, ERROR_CODE: "GMD02"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Description",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "GMD03"},
|
||||
{TYPE: MISSING, ERROR_CODE: "GMD04"}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Full description",
|
||||
VALIDATION: []
|
||||
},
|
||||
],
|
||||
},
|
||||
GENOMIC_INFO: {
|
||||
"acronym": "GID",
|
||||
"id_field": "Strain AN",
|
||||
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS08"},
|
||||
COLUMNS: [
|
||||
{
|
||||
FIELD: "Strain AN",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "GID01"},
|
||||
{TYPE: MISSING, ERROR_CODE: "GID02"},
|
||||
{TYPE: CROSSREF, CROSSREF_NAME: "Strains",
|
||||
ERROR_CODE: "GID03"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Marker",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "GID04"},
|
||||
{TYPE: MISSING, ERROR_CODE: "GID05"},
|
||||
{TYPE: CHOICES, ERROR_CODE: "GID06",
|
||||
VALUES: ['16S rRNA', 'ACT', 'CaM', 'EF-1α', 'ITS',
|
||||
'LSU', 'RPB1', 'RPB2', 'TUBB']}
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "INSDC AN",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "GID07"},
|
||||
{TYPE: MISSING, ERROR_CODE: "GID08"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Sequence",
|
||||
VALIDATION: []
|
||||
},
|
||||
],
|
||||
},
|
||||
STRAINS: {
|
||||
"acronym": "STD",
|
||||
'id_field': 'Accession number',
|
||||
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS05"},
|
||||
ROW_VALIDATION: [
|
||||
{TYPE: NAGOYA, ERROR_CODE: "STRXXX"},
|
||||
],
|
||||
COLUMNS: STRAIN_FIELDS,
|
||||
},
|
||||
LITERATURE_SHEET: {
|
||||
"acronym": "LID",
|
||||
'id_field': 'ID',
|
||||
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS03"},
|
||||
ROW_VALIDATION: [
|
||||
{TYPE: BIBLIO, ERROR_CODE: 'LID17'}
|
||||
],
|
||||
COLUMNS: [
|
||||
{
|
||||
FIELD: "ID",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "LID01"},
|
||||
{TYPE: MISSING, ERROR_CODE: "LID02"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Full reference",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "LID03"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Authors",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "LID05"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Title",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "LID07"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Journal",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "LID09"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Year",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "LID11"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Volume",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "LID13"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Issue",
|
||||
VALIDATION: []
|
||||
},
|
||||
{
|
||||
FIELD: "First page",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "LID15"},
|
||||
{TYPE: MISSING, ERROR_CODE: "LID16"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Last page",
|
||||
VALIDATION: []
|
||||
},
|
||||
{
|
||||
FIELD: "Book title",
|
||||
VALIDATION: []
|
||||
},
|
||||
{
|
||||
FIELD: "Editors",
|
||||
VALIDATION: []
|
||||
},
|
||||
{
|
||||
FIELD: "Publisher",
|
||||
VALIDATION: []
|
||||
}
|
||||
],
|
||||
},
|
||||
# SEXUAL_STATE_SHEET: {"acronym": "SSD", COLUMNS: []},
|
||||
# RESOURCE_TYPES_VALUES: {"acronym": "RTD", COLUMNS: []},
|
||||
# FORM_OF_SUPPLY_SHEET: {"acronym": "FSD", COLUMNS: []},
|
||||
# PLOIDY_SHEET: {"acronym": "PLD", COLUMNS: []},
|
||||
ONTOBIOTOPE: {
|
||||
"acronym": "OTD",
|
||||
"id_field": "ID",
|
||||
VALIDATION: {TYPE: MANDATORY, ERROR_CODE: "EFS06"},
|
||||
COLUMNS: [
|
||||
{
|
||||
FIELD: "ID",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "OTD01"},
|
||||
{TYPE: MISSING, ERROR_CODE: "OTD02"},
|
||||
]
|
||||
},
|
||||
{
|
||||
FIELD: "Name",
|
||||
VALIDATION: [
|
||||
{TYPE: MANDATORY, ERROR_CODE: "OTD03"},
|
||||
{TYPE: MISSING, ERROR_CODE: "OTD04"},
|
||||
]
|
||||
},
|
||||
]
|
||||
},
|
||||
# MARKERS: {
|
||||
# "acronym": "MKD",
|
||||
# "id_field": "",
|
||||
# COLUMNS: [
|
||||
# {
|
||||
# FIELD: "Acronym",
|
||||
# VALIDATION: []
|
||||
# },
|
||||
# {
|
||||
# FIELD: "Marker",
|
||||
# VALIDATION: []
|
||||
# },
|
||||
# ],
|
||||
# },
|
||||
}
|
||||
|
||||
CROSS_REF_CONF = {
|
||||
ONTOBIOTOPE: ['ID', 'Name'],
|
||||
LITERATURE_SHEET: ['ID'],
|
||||
LOCATIONS: ['Locality'],
|
||||
GROWTH_MEDIA: ['Acronym'],
|
||||
STRAINS: ["Accession number"],
|
||||
SEXUAL_STATE_SHEET: []
|
||||
|
||||
}
|
||||
|
||||
MIRRI_20200601_VALLIDATION_CONF = {
|
||||
'sheet_schema': SHEETS_SCHEMA,
|
||||
'cross_ref_conf': CROSS_REF_CONF,
|
||||
'keep_sheets_in_memory': [
|
||||
{'sheet_name': LOCATIONS, 'indexed_by': 'Locality'}]
|
||||
}
|
||||
Reference in New Issue
Block a user