forked from MIRRI/mirri_utils
Mirri 5.1.2
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
from mirri.biolomics.remote.endoint_names import (SEQUENCE_WS, STRAIN_WS,
|
||||
GROWTH_MEDIUM_WS, TAXONOMY_WS,
|
||||
COUNTRY_WS, ONTOBIOTOPE_WS,
|
||||
BIBLIOGRAPHY_WS)
|
||||
from mirri.biolomics.remote.rest_client import BiolomicsClient
|
||||
from mirri.biolomics.serializers.sequence import (
|
||||
serialize_to_biolomics as sequence_to_biolomics,
|
||||
serialize_from_biolomics as sequence_from_biolomics)
|
||||
from mirri.biolomics.serializers.strain import (
|
||||
serialize_to_biolomics as strain_to_biolomics,
|
||||
serialize_from_biolomics as strain_from_biolomics)
|
||||
|
||||
from mirri.biolomics.serializers.growth_media import (
|
||||
serialize_to_biolomics as growth_medium_to_biolomics,
|
||||
serialize_from_biolomics as growth_medium_from_biolomics)
|
||||
from mirri.biolomics.serializers.taxonomy import (
|
||||
serialize_from_biolomics as taxonomy_from_biolomics)
|
||||
from mirri.biolomics.serializers.locality import (
|
||||
serialize_from_biolomics as country_from_biolomics)
|
||||
from mirri.biolomics.serializers.ontobiotope import (
|
||||
serialize_from_biolomics as ontobiotope_from_biolomics)
|
||||
from mirri.biolomics.serializers.bibliography import (
|
||||
serializer_from_biolomics as bibliography_from_biolomics,
|
||||
serializer_to_biolomics as bibliography_to_biolomics
|
||||
)
|
||||
from pprint import pprint
|
||||
|
||||
|
||||
class BiolomicsMirriClient:
|
||||
_conf = {
|
||||
SEQUENCE_WS: {
|
||||
'serializers': {'to': sequence_to_biolomics,
|
||||
'from': sequence_from_biolomics},
|
||||
'endpoint': 'WS Sequences'},
|
||||
STRAIN_WS: {
|
||||
'serializers': {'to': strain_to_biolomics,
|
||||
'from': strain_from_biolomics},
|
||||
'endpoint': 'WS Strains'},
|
||||
GROWTH_MEDIUM_WS: {
|
||||
'serializers': {'from': growth_medium_from_biolomics,
|
||||
'to': growth_medium_to_biolomics},
|
||||
'endpoint': 'WS Growth media'},
|
||||
TAXONOMY_WS: {
|
||||
'serializers': {'from': taxonomy_from_biolomics},
|
||||
'endpoint': 'WS Taxonomy'},
|
||||
COUNTRY_WS: {
|
||||
'serializers': {'from': country_from_biolomics},
|
||||
'endpoint': 'WS Locality'},
|
||||
ONTOBIOTOPE_WS: {
|
||||
'serializers': {'from': ontobiotope_from_biolomics},
|
||||
'endpoint': 'WS Ontobiotope'},
|
||||
BIBLIOGRAPHY_WS: {
|
||||
'serializers': {'from': bibliography_from_biolomics,
|
||||
'to': bibliography_to_biolomics},
|
||||
'endpoint': 'WS Bibliography'
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self, server_url, api_version, client_id, client_secret, username,
|
||||
password, website_id=1, verbose=False):
|
||||
_client = BiolomicsClient(server_url, api_version, client_id,
|
||||
client_secret, username, password,
|
||||
website_id=website_id, verbose=verbose)
|
||||
|
||||
self.client = _client
|
||||
self.schemas = self.client.get_schemas()
|
||||
self.allowed_fields = self.client.allowed_fields
|
||||
self._transaction_created_ids = None
|
||||
self._in_transaction = False
|
||||
self._verbose = verbose
|
||||
|
||||
def _initialize_transaction_storage(self):
|
||||
if self._in_transaction:
|
||||
msg = 'Can not initialize transaction if already in a transaction'
|
||||
raise RuntimeError(msg)
|
||||
self._transaction_created_ids = []
|
||||
|
||||
def _add_created_to_transaction_storage(self, response, entity_name):
|
||||
if not self._in_transaction:
|
||||
msg = 'Can not add ids to transaction storage if not in a transaction'
|
||||
raise RuntimeError(msg)
|
||||
|
||||
id_ = response.json().get('RecordId', None)
|
||||
if id_ is not None:
|
||||
ws_endpoint_name = self._conf[entity_name]['endpoint']
|
||||
self._transaction_created_ids.insert(0, (ws_endpoint_name, id_))
|
||||
|
||||
def start_transaction(self):
|
||||
self._initialize_transaction_storage()
|
||||
self._in_transaction = True
|
||||
|
||||
def finish_transaction(self):
|
||||
self._in_transaction = False
|
||||
self._transaction_created_ids = None
|
||||
|
||||
def get_endpoint(self, entity_name):
|
||||
return self._conf[entity_name]['endpoint']
|
||||
|
||||
def get_serializers_to(self, entity_name):
|
||||
return self._conf[entity_name]['serializers']['to']
|
||||
|
||||
def get_serializers_from(self, entity_name):
|
||||
return self._conf[entity_name]['serializers']['from']
|
||||
|
||||
def retrieve_by_name(self, entity_name, name):
|
||||
endpoint = self.get_endpoint(entity_name)
|
||||
serializer_from = self.get_serializers_from(entity_name)
|
||||
response = self.client.find_by_name(endpoint, name=name)
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
elif response.status_code != 200:
|
||||
raise ValueError(f"{response.status_code}: {response.text}")
|
||||
|
||||
ws_entity = response.json()
|
||||
|
||||
return None if ws_entity is None else serializer_from(ws_entity,
|
||||
client=self)
|
||||
|
||||
def retrieve_by_id(self, entity_name, _id):
|
||||
endpoint = self.get_endpoint(entity_name)
|
||||
serializer_from = self.get_serializers_from(entity_name)
|
||||
response = self.client.retrieve(endpoint, record_id=_id)
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
elif response.status_code != 200:
|
||||
raise ValueError(f"{response.status_code}: {response.text}")
|
||||
|
||||
ws_entity = response.json()
|
||||
|
||||
return serializer_from(ws_entity, client=self)
|
||||
|
||||
def create(self, entity_name, entity):
|
||||
endpoint = self.get_endpoint(entity_name)
|
||||
serializer_to = self.get_serializers_to(entity_name)
|
||||
serializer_from = self.get_serializers_from(entity_name)
|
||||
data = serializer_to(entity, client=self)
|
||||
response = self.client.create(endpoint, data=data)
|
||||
if response.status_code == 200:
|
||||
if self._in_transaction:
|
||||
self._add_created_to_transaction_storage(response, entity_name)
|
||||
return serializer_from(response.json(), client=self)
|
||||
else:
|
||||
msg = f"return_code: {response.status_code}. msg: {response.json()['errors']['Value']}"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
def delete_by_id(self, entity_name, record_id):
|
||||
endpoint = self.get_endpoint(entity_name)
|
||||
response = self.client.delete(endpoint, record_id=record_id)
|
||||
if response.status_code != 200:
|
||||
error = response.json()
|
||||
# msg = f'{error["Title"]: {error["Details"]}}'
|
||||
raise RuntimeError(error)
|
||||
|
||||
def delete_by_name(self, entity_name, record_name):
|
||||
endpoint = self.get_endpoint(entity_name)
|
||||
response = self.client.find_by_name(endpoint, record_name)
|
||||
if response.status_code != 200:
|
||||
error = response.json()
|
||||
# msg = f'{error["Title"]: {error["Details"]}}'
|
||||
raise RuntimeError(error)
|
||||
try:
|
||||
record_id = response.json()['RecordId']
|
||||
except TypeError:
|
||||
raise ValueError(f'The given record_name {record_name} does not exists')
|
||||
self.delete_by_id(entity_name, record_id=record_id)
|
||||
|
||||
def search(self, entity_name, query):
|
||||
endpoint = self.get_endpoint(entity_name)
|
||||
serializer_from = self.get_serializers_from(entity_name)
|
||||
response = self.client.search(endpoint, search_query=query)
|
||||
if response.status_code != 200:
|
||||
error = response.json()
|
||||
# msg = f'{error["Title"]: {error["Details"]}}'
|
||||
raise RuntimeError(error)
|
||||
search_result = response.json()
|
||||
# pprint(search_result)
|
||||
result = {'total': search_result['TotalCount'],
|
||||
'records': [serializer_from(record, client=self)
|
||||
for record in search_result['Records']]}
|
||||
return result
|
||||
|
||||
def update(self, entity_name, entity):
|
||||
record_id = entity.record_id
|
||||
if record_id is None:
|
||||
msg = 'In order to update the record, you need the recordId in the entity'
|
||||
raise ValueError(msg)
|
||||
endpoint = self.get_endpoint(entity_name)
|
||||
serializer_to = self.get_serializers_to(entity_name)
|
||||
serializer_from = self.get_serializers_from(entity_name)
|
||||
data = serializer_to(entity, client=self, update=True)
|
||||
# print('update')
|
||||
# pprint(entity.dict())
|
||||
# print(data)
|
||||
# pprint(data, width=200)
|
||||
response = self.client.update(endpoint, record_id=record_id, data=data)
|
||||
if response.status_code == 200:
|
||||
# print('receive')
|
||||
# pprint(response.json())
|
||||
entity = serializer_from(response.json(), client=self)
|
||||
# pprint(entity.dict())
|
||||
return entity
|
||||
|
||||
else:
|
||||
msg = f"return_code: {response.status_code}. msg: {response.text}"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
def rollback(self):
|
||||
self._in_transaction = False
|
||||
self.client.rollback(self._transaction_created_ids)
|
||||
self._transaction_created_ids = None
|
||||
@@ -0,0 +1,7 @@
|
||||
SEQUENCE_WS = 'sequence'
|
||||
STRAIN_WS = 'strain'
|
||||
GROWTH_MEDIUM_WS = 'growth_medium'
|
||||
TAXONOMY_WS = 'taxonomy'
|
||||
COUNTRY_WS = 'country'
|
||||
ONTOBIOTOPE_WS = 'ontobiotope'
|
||||
BIBLIOGRAPHY_WS = 'bibliography'
|
||||
@@ -0,0 +1,214 @@
|
||||
import time
|
||||
import re
|
||||
import sys
|
||||
|
||||
import requests
|
||||
from requests_oauthlib import OAuth2Session
|
||||
from oauthlib.oauth2 import LegacyApplicationClient
|
||||
from oauthlib.oauth2.rfc6749.errors import InvalidGrantError
|
||||
|
||||
from mirri.entities.strain import ValidationError
|
||||
|
||||
|
||||
class BiolomicsClient:
|
||||
schemas = None
|
||||
allowed_fields = None
|
||||
|
||||
def __init__(self, server_url, api_version, client_id, client_secret,
|
||||
username, password, website_id=1, verbose=False):
|
||||
self._client_id = client_id
|
||||
self._client_secret = client_secret
|
||||
self._username = username
|
||||
self._password = password
|
||||
self._client = None
|
||||
self.server_url = server_url
|
||||
self._api_version = api_version
|
||||
self._auth_url = self.server_url + "/connect/token"
|
||||
self.access_token = None
|
||||
self.website_id = website_id
|
||||
self._verbose = verbose
|
||||
self._schema = self.get_schemas()
|
||||
|
||||
def get_access_token(self):
|
||||
if self._client is None:
|
||||
self._client = LegacyApplicationClient(client_id=self._client_id)
|
||||
authenticated = False
|
||||
else:
|
||||
expires_at = self._client.token["expires_at"]
|
||||
authenticated = expires_at > time.time()
|
||||
if not authenticated:
|
||||
oauth = OAuth2Session(client=self._client)
|
||||
try:
|
||||
token = oauth.fetch_token(
|
||||
token_url=self._auth_url,
|
||||
username=self._username,
|
||||
password=self._password,
|
||||
client_id=self._client_id,
|
||||
client_secret=self._client_secret,
|
||||
)
|
||||
except InvalidGrantError:
|
||||
oauth.close()
|
||||
raise
|
||||
self.access_token = token["access_token"]
|
||||
oauth.close()
|
||||
return self.access_token
|
||||
|
||||
def _build_headers(self):
|
||||
self.get_access_token()
|
||||
return {
|
||||
"accept": "application/json",
|
||||
"websiteId": str(self.website_id),
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
}
|
||||
|
||||
def get_detail_url(self, end_point, record_id, api_version=None):
|
||||
# api_version = self._api_version if api_version is None else api_version
|
||||
if api_version:
|
||||
return "/".join([self.server_url, api_version, 'data',
|
||||
end_point, str(record_id)])
|
||||
else:
|
||||
return "/".join([self.server_url, 'data', end_point, str(record_id)])
|
||||
|
||||
def get_list_url(self, end_point):
|
||||
return "/".join([self.server_url, 'data', end_point])
|
||||
# return "/".join([self.server_url, self._api_version, 'data', end_point])
|
||||
|
||||
def get_search_url(self, end_point):
|
||||
return "/".join([self.server_url, self._api_version, 'search', end_point])
|
||||
|
||||
def get_find_by_name_url(self, end_point):
|
||||
return "/".join([self.get_search_url(end_point), 'findByName'])
|
||||
|
||||
def search(self, end_point, search_query):
|
||||
self._check_end_point_exists(end_point)
|
||||
header = self._build_headers()
|
||||
url = self.get_search_url(end_point)
|
||||
time0 = time.time()
|
||||
response = requests.post(url, json=search_query, headers=header)
|
||||
time1 = time.time()
|
||||
if self._verbose:
|
||||
sys.stdout.write(f'Search to {end_point} request time for {url}: {time1 - time0}\n')
|
||||
return response
|
||||
|
||||
def retrieve(self, end_point, record_id):
|
||||
self._check_end_point_exists(end_point)
|
||||
header = self._build_headers()
|
||||
url = self.get_detail_url(end_point, record_id, api_version=self._api_version)
|
||||
time0 = time.time()
|
||||
response = requests.get(url, headers=header)
|
||||
time1 = time.time()
|
||||
if self._verbose:
|
||||
sys.stdout.write(f'Get to {end_point} request time for {url}: {time1-time0}\n')
|
||||
return response
|
||||
|
||||
def create(self, end_point, data):
|
||||
self._check_end_point_exists(end_point)
|
||||
self._check_data_consistency(data, self.allowed_fields[end_point])
|
||||
header = self._build_headers()
|
||||
url = self.get_list_url(end_point)
|
||||
return requests.post(url, json=data, headers=header)
|
||||
|
||||
def update(self, end_point, record_id, data):
|
||||
self._check_end_point_exists(end_point)
|
||||
self._check_data_consistency(data, self.allowed_fields[end_point],
|
||||
update=True)
|
||||
header = self._build_headers()
|
||||
url = self.get_detail_url(end_point, record_id=record_id)
|
||||
return requests.put(url, json=data, headers=header)
|
||||
|
||||
def delete(self, end_point, record_id):
|
||||
self._check_end_point_exists(end_point)
|
||||
header = self._build_headers()
|
||||
url = self.get_detail_url(end_point, record_id)
|
||||
return requests.delete(url, headers=header)
|
||||
|
||||
def find_by_name(self, end_point, name):
|
||||
self._check_end_point_exists(end_point)
|
||||
header = self._build_headers()
|
||||
url = self.get_find_by_name_url(end_point)
|
||||
response = requests.get(url, headers=header, params={'name': name})
|
||||
return response
|
||||
|
||||
def get_schemas(self):
|
||||
if self.schemas is None:
|
||||
headers = self._build_headers()
|
||||
url = self.server_url + '/schemas'
|
||||
response = requests.get(url, headers=headers)
|
||||
if response.status_code == 200:
|
||||
self.schemas = response.json()
|
||||
else:
|
||||
raise ValueError(f"{response.status_code}: {response.text}")
|
||||
if self.allowed_fields is None:
|
||||
self.allowed_fields = self._process_schema(self.schemas)
|
||||
return self.schemas
|
||||
|
||||
@staticmethod
|
||||
def _process_schema(schemas):
|
||||
schema = schemas[0]
|
||||
allowed_fields = {}
|
||||
for endpoint_schema in schema['TableViews']:
|
||||
endpoint_name = endpoint_schema['TableViewName']
|
||||
endpoint_values = endpoint_schema['ResultFields']
|
||||
fields = {field['title']: field for field in endpoint_values}
|
||||
allowed_fields[endpoint_name] = fields
|
||||
return allowed_fields
|
||||
|
||||
def _check_end_point_exists(self, endpoint):
|
||||
if endpoint not in self.allowed_fields.keys():
|
||||
raise ValueError(f'{endpoint} not a recognised endpoint')
|
||||
|
||||
def _check_data_consistency(self, data, allowed_fields, update=False):
|
||||
update_mandatory = set(['RecordDetails', 'RecordName', 'RecordId'])
|
||||
if update and not update_mandatory.issubset(data.keys()):
|
||||
msg = 'Updating data keys must be RecordDetails, RecordName and RecordId'
|
||||
raise ValidationError(msg)
|
||||
|
||||
if not update and set(data.keys()).difference(['RecordDetails', 'RecordName', 'Acronym']):
|
||||
msg = 'data keys must be RecordDetails and RecordName or Acronym'
|
||||
raise ValidationError(msg)
|
||||
for field_name, field_value in data['RecordDetails'].items():
|
||||
if field_name not in allowed_fields:
|
||||
raise ValidationError(f'{field_name} not in allowed fields')
|
||||
|
||||
field_schema = allowed_fields[field_name]
|
||||
self._check_field_schema(field_name, field_schema, field_value)
|
||||
|
||||
@staticmethod
|
||||
def _check_field_schema(field_name, field_schema, field_value):
|
||||
if field_schema['FieldType'] != field_value['FieldType']:
|
||||
msg = f"Bad FieldType ({field_value['FieldType']}) for {field_name}. "
|
||||
msg += f"It should be {field_schema['FieldType']}"
|
||||
raise ValidationError(msg)
|
||||
|
||||
states = field_schema['states'] if 'states' in field_schema else None
|
||||
if states:
|
||||
states = [re.sub(r" *\(.*\)", "", s) for s in states]
|
||||
|
||||
subfields = field_schema['subfields'] if 'subfields' in field_schema else None
|
||||
if subfields is not None and states is not None:
|
||||
subfield_names = [subfield['SubFieldName']
|
||||
for subfield in subfields if subfield['IsUsed']]
|
||||
|
||||
for val in field_value['Value']:
|
||||
if val['Name'] not in subfield_names:
|
||||
msg = f"{field_name}: {val['Name']} not in {subfield_names}"
|
||||
raise ValidationError(msg)
|
||||
|
||||
if val['Value'] not in states:
|
||||
|
||||
msg = f"{field_value['Value']} not a valid value for "
|
||||
msg += f"{field_name}, Allowed values: {'. '.join(states)}"
|
||||
raise ValidationError(msg)
|
||||
|
||||
elif states is not None:
|
||||
if field_value['Value'] not in states:
|
||||
msg = f"{field_value['Value']} not a valid value for "
|
||||
msg += f"{field_name}, Allowed values: {'. '.join(states)}"
|
||||
raise ValidationError(msg)
|
||||
|
||||
def rollback(self, created_ids):
|
||||
for endpoint, id_ in created_ids:
|
||||
try:
|
||||
self.delete(end_point=endpoint, record_id=id_)
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user