From 0bb8f9f3d715aae786b449e3bd52b810282213de Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Thu, 31 Oct 2019 07:55:14 -0700 Subject: [PATCH 01/17] removing case conflict --- .../Tutorials/interlex_remotes_tutorial.ipynb | 742 ------------------ 1 file changed, 742 deletions(-) delete mode 100644 ilxutils/Tutorials/interlex_remotes_tutorial.ipynb diff --git a/ilxutils/Tutorials/interlex_remotes_tutorial.ipynb b/ilxutils/Tutorials/interlex_remotes_tutorial.ipynb deleted file mode 100644 index f596fda2..00000000 --- a/ilxutils/Tutorials/interlex_remotes_tutorial.ipynb +++ /dev/null @@ -1,742 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# INSTALL\n", - "- WARNING ::: ONLY DO ONCE \n", - " - update devconfig in ~/.config/pyontutils/devonfig.yaml\n", - " - scigraph_api: http://scigraph.scicrunch.io:9000/scigraph\n", - " - Install both pyontutils and ilxutils with pyontutils\n", - " - cd ~/git/pyontutils\n", - " - pip3 install --user --editable .\n", - " - cd ~/git/pyontutils/ilxutils/\n", - " - pip3 install --user --editable .\n", - " - Clone ontquery and install\n", - " - cd ~/git\n", - " - git clone https://github.com/tgbugs/ontquery.git\n", - " - cd ~/git/ontquery\n", - " - pip3 install --user --editable ." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Maintainance\n", - "- update repos\n", - " - cd ~/git/pyontutils\n", - " - git pull \n", - " - cd ~/git/ontquery\n", - " - git pull" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Google Sheets Import\n", - "### Need pyontutils secrets.yaml setup first!" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
termcurie
1abdominal cavityUBERON:0003684
2abdominal wallUBERON:0003697
3adipose tissueUBERON:0001013
4adult organismUBERON:0007023
5alimentary part of gastrointestinal systemUBERON:0005409
\n", - "
" - ], - "text/plain": [ - "0 term curie\n", - "1 abdominal cavity UBERON:0003684\n", - "2 abdominal wall UBERON:0003697\n", - "3 adipose tissue UBERON:0001013\n", - "4 adult organism UBERON:0007023\n", - "5 alimentary part of gastrointestinal system UBERON:0005409" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from pyontutils.sheets import Sheet\n", - "import pandas as pd\n", - "KEY_NAME = 'sparc-terms'\n", - "SHEET_NAME = 'Minimal information model(MIS)'\n", - "\n", - "class Brainstem(Sheet):\n", - " name = KEY_NAME # key name you gave the google sheet id value in secrets.yaml\n", - " sheet_name = SHEET_NAME # the actual sheet name on the google sheet\n", - " fetch_grid = True # meta data in self.grid that has detials like bolding\n", - "\n", - "brainstem = Brainstem()\n", - "df = pd.DataFrame(brainstem.raw_values)\n", - "df.columns = df.iloc[0]\n", - "df.drop(df.index[0], inplace=True)\n", - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['abdominal cavity',\n", - " 'abdominal wall',\n", - " 'adipose tissue',\n", - " 'adult organism',\n", - " 'alimentary part of gastrointestinal system',\n", - " 'arterial blood',\n", - " 'biceps femoris',\n", - " 'blood',\n", - " 'bolus of food',\n", - " 'brainstem']" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(df.term)[:10]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# CSV or TSV EXAMPLE" - ] - }, - { - "cell_type": "raw", - "metadata": {}, - "source": [ - "import pandas as pd\n", - "\n", - "csv_df = pd.DataFrame('/path/to/csv')\n", - "tsv_df = pd.DataFrame('/path/to/tsv', delimiter='\\t')\n", - "\n", - "csv_df.head() # returns top 5 rows\n", - "csv_df.column_name # specific column name will return a Series which will act like a list" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# QUERY DATABASES " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[OntTerm('HBA:3999', label='brain (hba)'),\n", - " OntTerm('FMA:50801', label='Brain'),\n", - " OntTerm('UBERON:0000955', label='brain'),\n", - " OntTerm('UBERON:6110636', label='adult cerebral ganglion'),\n", - " OntTerm('ILX:0101431', label='Brain'),\n", - " OntTerm('ILX:0101433', label='Brain Infarction'),\n", - " OntTerm('ILX:0506386', label='Brain Aneurysm'),\n", - " OntTerm('ILX:0433050', label='Brain Chemistry'),\n", - " OntTerm('ILX:0641746', label='alpha BRAIN'),\n", - " OntTerm('ILX:0726394', label='brain meninx'),\n", - " OntTerm('ILX:0729002', label='brain commissure'),\n", - " OntTerm('ILX:0101434', label='Brain Ischemia'),\n", - " OntTerm('ILX:0461406', label='Brain Death'),\n", - " OntTerm('ILX:0733041', label='brain endothelium')]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Give \"query\" a usable parameter to query the databases \n", - "from pyontutils.core import query # OntTerm\n", - "query(term='brain')" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[OntTerm('ILX:0103358', label='DN1 neuron'),\n", - " OntTerm('ILX:0109525', label='Pupal DN1 period neuron')]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# similar entities will show\n", - "# default limit is 10\n", - "query(term='DN1 neuron', limit=2) " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[OntTerm('UBERON:0000955', label='brain')]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Faster and more accurate with curie/iri\n", - "query(curie='UBERON:0000955')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'prefix': 'UBERON',\n", - " 'suffix': '0000955',\n", - " 'orig_kwargs': {'iri': 'http://purl.obolibrary.org/obo/UBERON_0000955',\n", - " 'curie_or_iri': None,\n", - " 'label': None,\n", - " 'term': None,\n", - " 'search': None,\n", - " 'validated': None,\n", - " 'query': None},\n", - " 'kwargs': {'iri': 'http://purl.obolibrary.org/obo/UBERON_0000955',\n", - " 'curie_or_iri': None,\n", - " 'label': None,\n", - " 'term': None,\n", - " 'search': None,\n", - " 'validated': None,\n", - " 'query': None},\n", - " 'label': 'brain',\n", - " 'labels': ['brain'],\n", - " 'definition': 'The brain is the center of the nervous system in all vertebrate, and most invertebrate, animals. Some primitive animals such as jellyfish and starfish have a decentralized nervous system without a brain, while sponges lack any nervous system at all. In vertebrates, the brain is located in the head, protected by the skull and close to the primary sensory apparatus of vision, hearing, balance, taste, and smell[WP].',\n", - " 'synonyms': ['the brain',\n", - " 'synganglion',\n", - " 'suprasegmental structures',\n", - " 'suprasegmental levels of nervous system',\n", - " 'encephalon'],\n", - " 'deprecated': False,\n", - " 'predicates': {},\n", - " '_type': OntId('owl:Class'),\n", - " '_types': (OntId('owl:Class'),),\n", - " '_graph': None,\n", - " '_source': ,\n", - " 'validated': True,\n", - " '_query_result': QueryResult({'iri': 'http://purl.obolibrary.org/obo/UBERON_0000955', 'curie': 'UBERON:0000955', 'label': 'brain', 'labels': ['brain'], 'definition': 'The brain is the center of the nervous system in all vertebrate, and most invertebrate, animals. Some primitive animals such as jellyfish and starfish have a decentralized nervous system without a brain, while sponges lack any nervous system at all. In vertebrates, the brain is located in the head, protected by the skull and close to the primary sensory apparatus of vision, hearing, balance, taste, and smell[WP].', 'synonyms': ['the brain', 'synganglion', 'suprasegmental structures', 'suprasegmental levels of nervous system', 'encephalon'], 'deprecated': False, 'predicates': {}, 'type': OntId('owl:Class'), 'types': (OntId('owl:Class'),), '_graph': None, 'source': })}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "entity = query(curie='UBERON:0000955')[0]\n", - "# Full result attribute\n", - "vars(entity)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# DEBUGGING HINT\n", - "- 1 \"?\" at the end of a function or class will return its params, docstring, and pathing. \n", - "- 2 \"??\" returns the ENTIRE class/functions " - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\u001b[0;31mSignature:\u001b[0m \u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mCall signature:\u001b[0m\n", - "\u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mterm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mprefix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mcategory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mabbrev\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0msearch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0msuffix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mcurie\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0miri\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mexclude_prefix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mdirection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'OUTGOING'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0minclude_deprecated\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mType:\u001b[0m OntQueryCli\n", - "\u001b[0;31mString form:\u001b[0m \n", - "\u001b[0;31mFile:\u001b[0m ~/Dropbox/git/ontquery/ontquery/query.py\n", - "\u001b[0;31mDocstring:\u001b[0m \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "query?" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# BONUS!" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Concurrently search! (Run multiple query functions at the same time)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Futures compiled\n" - ] - }, - { - "data": { - "text/plain": [ - "[({'curie': 'UBERON:0000955'}, [OntTerm('UBERON:0000955', label='brain')]),\n", - " ({'curie': 'UBERON:6110636'},\n", - " [OntTerm('UBERON:6110636', label='adult cerebral ganglion')])]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from pyontutils.utils import Async, deferred\n", - "from pyontutils.core import OntTerm, ixr, query\n", - "from typing import List, Tuple\n", - "\n", - "# query.setup()\n", - "\n", - "def queries(kwargs_list:List[dict]) -> List[Tuple[str, dict]]:\n", - " '''Asynchronously query databases to dramatically increase runtime un users end \n", - " \n", - " Examples:\n", - " >>> queries([{'term':'Brain'},])\n", - " [({'term': 'Brain'},\n", - " [OntTerm('HBA:3999', label='brain (hba)'),\n", - " OntTerm('FMA:50801', label='Brain'),\n", - " OntTerm('UBERON:0000955', label='brain'),\n", - " OntTerm('UBERON:6110636', label='adult cerebral ganglion')])]\n", - " >>> queries([{'curie':'UBERON:0000955'},])\n", - " [({'curie': 'UBERON:0000955'}, [OntTerm('UBERON:0000955', label='brain')])]\n", - " \n", - " Definitions:\n", - " kwargs == common name given to dictionary input for function\n", - " tuple == a list that you cannot update. \n", - " lambda == short-hand for single line function creation (func = lambda : ) \n", - " \n", - " Args:\n", - " kwargs_list (list): A list of dictionaries that are paramaters for the query function\n", - " \n", - " Returns:\n", - " List[tuple]: A list of tuples all being of (, ). \n", - " '''\n", - " # create a query function wrapper to return tuple\n", - " # kwargs -> (kwargs, query_result)\n", - " # We do this in case 2+ queries return the same results & the output WILL NOT have the same input order\n", - " gin = lambda kwargs: (kwargs, query(**kwargs))\n", - " # run each query instance at the same time\n", - " results = Async(use_nest_asyncio=True)(deferred(gin)(kwargs) for kwargs in kwargs_list)\n", - " return results \n", - "\n", - "queries([{'curie':'UBERON:0000955'}, {'curie':'UBERON:6110636'}])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pyontutils.utils import Async, deferred\n", - "from pyontutils.core import OntTerm, ixr, query\n", - "from typing import List, Tuple\n", - "def queries(url_list:List[dict]) -> List[Tuple[str, dict]]:\n", - " def gin(url):\n", - " return requests.get(url).text\n", - " # run each query instance at the same time\n", - " results = Async(limit=5)(deferred(gin)(url) for url in url_list)\n", - " return results \n", - "list_tuples(url, html)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Modifing TEST InterLex" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# TEST InterLex endpoints\n", - "from ilxutils.remotes import interlex_remote_test as ixrt" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# GET ENTITY" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': '661544',\n", - " 'orig_uid': '34142',\n", - " 'uid': '34142',\n", - " 'orig_cid': '0',\n", - " 'cid': '0',\n", - " 'ilx': 'ilx_0738390',\n", - " 'label': 'Offical label',\n", - " 'type': 'term',\n", - " 'definition': 'official definition',\n", - " 'comment': 'helpful misc',\n", - " 'version': '3',\n", - " 'status': '0',\n", - " 'display_superclass': '1',\n", - " 'orig_time': '1564695195',\n", - " 'time': '1570826848',\n", - " 'synonyms': [{'id': '1776645',\n", - " 'tid': '661544',\n", - " 'literal': 'Encephalon',\n", - " 'type': '',\n", - " 'time': '1570826848',\n", - " 'version': '3'},\n", - " {'id': '1776646',\n", - " 'tid': '661544',\n", - " 'literal': 'Cerebro',\n", - " 'type': '',\n", - " 'time': '1570826848',\n", - " 'version': '3'}],\n", - " 'superclasses': [],\n", - " 'existing_ids': [{'id': '3885545',\n", - " 'tid': '661544',\n", - " 'curie': 'ILX:0738390',\n", - " 'iri': 'http://uri.interlex.org/base/ilx_0738390',\n", - " 'curie_catalog_id': '3885424',\n", - " 'version': '3',\n", - " 'time': '1570826848',\n", - " 'preferred': '1'}],\n", - " 'relationships': [],\n", - " 'mappings': [],\n", - " 'annotations': [],\n", - " 'ontologies': []}" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ixrt.ilx_cli.get_entity('tmp_0738390')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ADD" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\u001b[0;31mSignature:\u001b[0m\n", - "\u001b[0mixrt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_entity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0msubThingOf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mdefinition\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0msynonyms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mcomment\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mDocstring:\u001b[0m \n", - "\u001b[0;31mFile:\u001b[0m ~/Dropbox/git/ontquery/ontquery/plugins/services.py\n", - "\u001b[0;31mType:\u001b[0m method\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "ixrt.add_entity?" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "QueryResult({'iri': 'http://uri.interlex.org/base/ilx_0738390', 'curie': 'ILX:0738390', 'label': 'official test label', 'labels': (), 'definition': 'definition', 'synonyms': ('Encephalon', 'Cerebro'), 'deprecated': None, 'predicates': {'comment': 'helpful misc'}, 'type': None, 'types': (), '_graph': None, 'source': })\n" - ] - } - ], - "source": [ - "entity = dict(\n", - " label = 'offical label', # Can only one unique label per person\n", - " type = 'term', # OPTIONS: term, annotation, relationship, cde, fde, pde\n", - " definition = 'official definition',\n", - " comment = 'helpful misc',\n", - " # Optional\n", - " subThingOf = '', # WARNING ::: must have at last '', can be blank but please fill this in if you can. \n", - " synonyms = ['Encephalon', 'Cerebro'],\n", - " predicates = {} # annotations and/or relationships to add\n", - " # TODO: existing_ids will be an option later\n", - ")\n", - "result = ixrt.add_entity(**entity)\n", - "print(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# UPDATE" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\u001b[0;31mSignature:\u001b[0m\n", - "\u001b[0mixrt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_entity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0milx_id\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0msubThingOf\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mdefinition\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0msynonyms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mcomment\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mpredicates_to_add\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m \u001b[0mpredicates_to_delete\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", - "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mDocstring:\u001b[0m \n", - "\u001b[0;31mFile:\u001b[0m ~/Dropbox/git/ontquery/ontquery/plugins/services.py\n", - "\u001b[0;31mType:\u001b[0m method\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "ixrt.update_entity?" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m[2019-10-11 13:47:28,619]\u001b[0m - \u001b[32m INFO\u001b[0m - ontquery - \u001b[34minterlex_client.py:796 \u001b[0m - {'ilx_id': 'ILX:0738390', 'label': 'Offical label', 'type': 'term', 'definition': 'official definition', 'comment': 'helpful misc', 'superclass': '', 'synonyms': ['Encephalon', 'Cerebro']}\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "QueryResult({'iri': 'http://uri.interlex.org/base/ilx_0738390', 'curie': 'ILX:0738390', 'label': 'Offical label', 'labels': (), 'definition': 'official definition', 'synonyms': ('Encephalon', 'Cerebro'), 'deprecated': None, 'predicates': {'comment': 'helpful misc'}, 'type': None, 'types': (), '_graph': None, 'source': })\n" - ] - } - ], - "source": [ - "from ilxutils.remotes import interlex_remote_test as ixrt\n", - "entity = dict(\n", - " ilx_id = 'ILX:0738390',\n", - " label = 'Offical label', # Can only one unique label per person\n", - " type = 'term', # OPTIONS: term, annotation, relationship, cde, fde, pde\n", - " definition = 'official definition',\n", - " comment = 'helpful misc',\n", - " # Optional\n", - " subThingOf = '', # WARNING ::: must have at last '', can be blank but please fill this in if you can. \n", - " synonyms = ['Encephalon', 'Cerebro'],\n", - " predicates_to_add = {}, # annotations and/or relationships to add\n", - " predicates_to_delete = {}, # annotations and/or relationships to del\n", - " # TODO: existing_ids will be an option later\n", - ")\n", - "result = ixrt.update_entity(**entity)\n", - "print(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# PRODUCTION \n", - "# BE CAREFUL PLEASE :)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# PRODUCTION \n", - "from ilxutils.remotes import interlex_remote_production as ixr\n", - "# BE CAREFUL :)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} From 125b51c585b831ec0f69d4eabdd303b96e8cfc0a Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Tue, 5 Nov 2019 10:03:45 -0800 Subject: [PATCH 02/17] new elastic search wrapper --- ilxutils/ilxutils/elasticsearch_wrapper.py | 107 +++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 ilxutils/ilxutils/elasticsearch_wrapper.py diff --git a/ilxutils/ilxutils/elasticsearch_wrapper.py b/ilxutils/ilxutils/elasticsearch_wrapper.py new file mode 100644 index 00000000..ee9e8d76 --- /dev/null +++ b/ilxutils/ilxutils/elasticsearch_wrapper.py @@ -0,0 +1,107 @@ +from functools import wraps +import json +import os +import subprocess +import docopt +from elasticsearch import Elasticsearch +BASHRC = lambda s: os.environ.get(s) + + +class ElasticSearchTools: + """ Shortcuts for common elasticsearch querys. """ + + def __init__(self, + host: str, index: str, type: str, + user: str, password: str, + size: int = 10, start: int = 0, + scheme: str = 'https',) -> None: + """ + :param str url: ElasticSearch url endpoint. + :param str index: + """ + self.url = f'{scheme}://{host}/{index}' + self.host, self.index, self.type = host, index, type + self.es = Elasticsearch(self.url, http_auth=(user, password)) + + def search(self, body: dict, **kwargs) -> dict: + """ Elasticsearch '/_search' feature. + + We use a framented index called a type. The type is the last index + while the real index becomes part of the host url. + + :param dict body: query dict. + :return: nested elasticsearch dict where hits are in ['hits']['hits'] + + >>>__search(body={ 'query': { 'match_all': {} } }) + """ + return self.es.search(index=self.type, body=body, **kwargs) + + def all_matches(self, sorting: str, size, start) -> dict: + """First or last set of entities. + + :param str sorting: asc for head or desc for tail. + :param int size: number of entities you want from head or tails. + :param int start: position of index you want to start from. + :return: elasticsearch _search dict + """ + if sorting.lower().strip() not in ['asc', 'desc']: + raise ValueError('sorting can only be asc or desc.') + body = { + 'query': { 'match_all': {} }, + 'sort': [ { '_id': sorting } ], + 'size': size, + 'from': start, + } + return self.search(body) + + def head(self, size=10, start=0): + """ See __end doc. """ + return self.all_matches(sorting='asc', size=size, start=start) + + def tail(self, size=10, start=0): + """ See __end doc. """ + return self.all_matches(sorting='desc', size=size, start=start) + + +class InterLexES(ElasticSearchTools): + + def __init__(self, beta=True): + super().__init__( + host = BASHRC('INTERLEX_ELASTIC_URL'), + index = 'interlex', + type = 'term', + user = BASHRC('INTERLEX_ELASTIC_USER'), + password = BASHRC('INTERLEX_ELASTIC_PASSWORD'), + ) + self.beta = beta + + def filter_tmp(self): + prefix = 'tmp_' if self.beta else 'ilx_' + return { 'prefix': { 'ilx' : { 'value': prefix } } } + + def all_matches(self, sorting: str, size, start) -> dict: + """First or last set of entities. + + :param str sorting: asc for head or desc for tail. + :param int size: number of entities you want from head or tails. + :param int start: position of index you want to start from. + :return: elasticsearch _search dict + """ + if sorting.lower().strip() not in ['asc', 'desc']: + raise ValueError('sorting can only be asc or desc.') + body = { + 'query': self.filter_tmp(), + 'sort': [ { '_id': sorting } ], + 'size': size, + 'from': start, + } + return self.search(body) + + +def main(): + ilxes = InterLexES(beta=False) + print(ilxes.tail(1)) + + +if __name__ == '__main__': + main() From c7c71dfdd789a552a87b47f6d789987b8b227874 Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Fri, 13 Dec 2019 12:40:32 -0800 Subject: [PATCH 03/17] shortcut to sql and update print on ontopandas --- ilxutils/ilxutils/ontopandas.py | 6 ++++++ ilxutils/ilxutils/sql.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 ilxutils/ilxutils/sql.py diff --git a/ilxutils/ilxutils/ontopandas.py b/ilxutils/ilxutils/ontopandas.py index faf2581b..f777863b 100644 --- a/ilxutils/ilxutils/ontopandas.py +++ b/ilxutils/ilxutils/ontopandas.py @@ -43,10 +43,12 @@ class OntoPandas: def __init__(self, obj: Union[rdflib.graph.Graph, str], query:str=defaultquery, + curie:bool=True, qnamed:bool=False, str_vals:bool=False,) -> None: self.query = query self.qnamed = qnamed + self.curie = curie self.str_vals = str_vals self.g = obj # could be path self.path = obj # could be graph @@ -270,6 +272,10 @@ def get_sparql_dataframe( self ): df = df.where((pd.notnull(df)), None) # default Null is fricken Float NaN df = df.reset_index().rename(columns={'index':'iri'}) + + if self.curie: + df['curie'] = df.apply(lambda row: self.qname(row.iri), axis = 1) + return df diff --git a/ilxutils/ilxutils/sql.py b/ilxutils/ilxutils/sql.py new file mode 100644 index 00000000..c4176a71 --- /dev/null +++ b/ilxutils/ilxutils/sql.py @@ -0,0 +1,16 @@ +from interlex_sql import IlxSql +import os + +def production_sql(from_backup=True): + return IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_PRODUCTION'), from_backup=from_backup) + +def beta_sql(from_backup=True): + # TEST{#} should be a thing since this still relies on main sql test + return IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_BETA'), from_backup=from_backup) + +# entities = [] +# for ilx, group in ex.groupby('ilx'): +# if not any(list(group['preferred'] == '1')): +# entities.append(ilx) + +# from ontquery.plugins.services.interlex_client import InterLexClient From 4559c51ec0ef9e3b75b1cfcc7a31b67296cac9da Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Tue, 7 Jan 2020 11:45:33 -0800 Subject: [PATCH 04/17] this was lost --- pyontutils/ontutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyontutils/ontutils.py b/pyontutils/ontutils.py index 0be7935b..f4fc60d1 100755 --- a/pyontutils/ontutils.py +++ b/pyontutils/ontutils.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3.7 #!/usr/bin/env pypy3 -from pyontutils.config import auth +from pyontutils.config import auth, devconfig __doc__ = f"""Common commands for ontology processes. Also old ontology refactors to run in the root ttl folder. From a27a253e9f56b863814f570a0fd7ed6289672e04 Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Tue, 7 Jan 2020 11:48:00 -0800 Subject: [PATCH 05/17] update ilx references --- ilxutils/ilx-playground.ipynb | 16 ++-- .../tutorials/interlex_remotes_tutorial.ipynb | 82 ++++--------------- 2 files changed, 21 insertions(+), 77 deletions(-) diff --git a/ilxutils/ilx-playground.ipynb b/ilxutils/ilx-playground.ipynb index 7f8afdfb..7072d902 100644 --- a/ilxutils/ilx-playground.ipynb +++ b/ilxutils/ilx-playground.ipynb @@ -1,12 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 33, @@ -681,7 +674,12 @@ { "cell_type": "code", "execution_count": 10, - "metadata": {}, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + } + }, "outputs": [ { "name": "stderr", @@ -6477,7 +6475,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.5" } }, "nbformat": 4, diff --git a/ilxutils/tutorials/interlex_remotes_tutorial.ipynb b/ilxutils/tutorials/interlex_remotes_tutorial.ipynb index f596fda2..85b28da4 100644 --- a/ilxutils/tutorials/interlex_remotes_tutorial.ipynb +++ b/ilxutils/tutorials/interlex_remotes_tutorial.ipynb @@ -42,76 +42,22 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 1, "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
termcurie
1abdominal cavityUBERON:0003684
2abdominal wallUBERON:0003697
3adipose tissueUBERON:0001013
4adult organismUBERON:0007023
5alimentary part of gastrointestinal systemUBERON:0005409
\n", - "
" - ], - "text/plain": [ - "0 term curie\n", - "1 abdominal cavity UBERON:0003684\n", - "2 abdominal wall UBERON:0003697\n", - "3 adipose tissue UBERON:0001013\n", - "4 adult organism UBERON:0007023\n", - "5 alimentary part of gastrointestinal system UBERON:0005409" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" + "ename": "AttributeError", + "evalue": "'NoneType' object has no attribute 'exists'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mfetch_grid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;31m# meta data in self.grid that has detials like bolding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mbrainstem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBrainstem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbrainstem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw_values\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Dropbox/git/pyontutils/pyontutils/sheets.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, sheet_name, fetch_grid, readonly)\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadonly\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreadonly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 211\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Dropbox/git/pyontutils/pyontutils/sheets.py\u001b[0m in \u001b[0;36m_setup\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadonly\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSheet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__spreadsheet_service_ro'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m \u001b[0mservice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_oauth_service\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreadonly\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadonly\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# I think it is correct to keep this ephimoral\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 217\u001b[0m \u001b[0mSheet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__spreadsheet_service_ro\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mservice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspreadsheets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Dropbox/git/pyontutils/pyontutils/sheets.py\u001b[0m in \u001b[0;36mget_oauth_service\u001b[0;34m(api, version, readonly, SCOPES)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[0mstore_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mauth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_auth_var\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mstore_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 62\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'exists'" + ] } ], "source": [ @@ -734,7 +680,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.4" } }, "nbformat": 4, From 3c72023e7d73a98f2508e58a317cc69bf5e69ff8 Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Mon, 24 Feb 2020 16:55:40 -0800 Subject: [PATCH 06/17] updates --- ilxutils/ilxutils/backup_ilx.py | 10 ++-- ilxutils/ilxutils/interlex_sql.py | 11 ++-- ilxutils/ilxutils/nltklib.py | 3 +- ilxutils/ilxutils/remotes.py | 29 +++++----- ilxutils/ilxutils/scicrunch_session.py | 74 ++++++++++++++++++++++++++ ilxutils/ilxutils/sql.py | 2 +- 6 files changed, 106 insertions(+), 23 deletions(-) create mode 100644 ilxutils/ilxutils/scicrunch_session.py diff --git a/ilxutils/ilxutils/backup_ilx.py b/ilxutils/ilxutils/backup_ilx.py index a4b102f7..2a94a962 100644 --- a/ilxutils/ilxutils/backup_ilx.py +++ b/ilxutils/ilxutils/backup_ilx.py @@ -1,9 +1,13 @@ from pathlib import Path as p -from ilxutils.interlex_sql import IlxSql -from ilxutils.tools import create_pickle +from interlex_sql import IlxSql +# from tools import create_pickle +import pickle import os - +def create_pickle(data, outfilename): + with open(outfilename, 'wb') as outfile: + pickle.dump(data, outfile) + sql = IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_PRODUCTION')) diff --git a/ilxutils/ilxutils/interlex_sql.py b/ilxutils/ilxutils/interlex_sql.py index 256a0f83..ce45497b 100755 --- a/ilxutils/ilxutils/interlex_sql.py +++ b/ilxutils/ilxutils/interlex_sql.py @@ -73,7 +73,7 @@ def get_terms(self): return self.terms engine = create_engine(self.db_url) data = """ - SELECT t.id as tid, t.ilx, t.label, t.definition, t.type, t.comment, t.version, t.uid, t.time + SELECT t.id as tid, t.ilx, t.label, t.definition, t.type, t.comment, t.version, t.uid, t.cid, t.time, t.status FROM terms t GROUP BY t.ilx """ @@ -118,7 +118,7 @@ def get_existing_ids(self): return self.existing_ids engine = create_engine(self.db_url) data = """ - SELECT tei.tid, tei.curie, tei.iri, tei.preferred, t.ilx, t.label, t.definition + SELECT tei.tid, tei.curie, tei.iri, tei.preferred, t.ilx, t.label, t.definition, t.status FROM ( SELECT * FROM terms @@ -140,24 +140,27 @@ def get_relationships(self): engine = create_engine(self.db_url) data = """ SELECT - t1.id as term1_tid, t1.ilx AS term1_ilx, t1.type as term1_type, - t2.id as term2_tid, t2.ilx AS term2_ilx, t2.type as term2_type, + t1.id as term1_tid, t1.ilx AS term1_ilx, t1.type as term1_type, t1.label as term1_label, + t2.id as term2_tid, t2.ilx AS term2_ilx, t2.type as term2_type, t2.label as term2_label, t3.id as relationship_tid, t3.ilx AS relationship_ilx, t3.label as relationship_label FROM term_relationships AS tr JOIN ( SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) t1 ON t1.id = tr.term1_id JOIN ( SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) AS t2 ON t2.id = tr.term2_id JOIN ( SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) AS t3 ON t3.id = tr.relationship_tid """ self.relationships = pd.read_sql(data, engine) diff --git a/ilxutils/ilxutils/nltklib.py b/ilxutils/ilxutils/nltklib.py index da5c9918..27acae74 100644 --- a/ilxutils/ilxutils/nltklib.py +++ b/ilxutils/ilxutils/nltklib.py @@ -137,7 +137,8 @@ def sentence_similarity(sentence1, sentence2, ignore_integers=False): tokens2 = word_tokenize(sentence2) tokens1 = clean_tokens(tokens1, ignore_integers) tokens2 = clean_tokens(tokens2, ignore_integers) - + print(tokens1) + print(tokens2) # tag sentence1 = pos_tag(tokens1) sentence2 = pos_tag(tokens2) diff --git a/ilxutils/ilxutils/remotes.py b/ilxutils/ilxutils/remotes.py index 4137adb1..711766c2 100644 --- a/ilxutils/ilxutils/remotes.py +++ b/ilxutils/ilxutils/remotes.py @@ -2,19 +2,20 @@ from pyontutils.core import OntTerm import os -TEST = 'https://test3.scicrunch.org/api/1/' -PRODUCTION = 'https://scicrunch.org/api/1/' +def remote(server=''): -InterLexRemote = oq.plugin.get('InterLex') -interlex_remote_production = InterLexRemote( - # When ready, should be changed to 'https://scicrunch.org/api/1/' for production - apiEndpoint = PRODUCTION -) -interlex_remote_production.setup(instrumented=OntTerm) + # Request interlex remote (scigraph is also an option for plugins) + InterLexRemote = oq.plugin.get('InterLex') -# InterLexRemote = oq.plugin.get('InterLex') -# interlex_remote_test = InterLexRemote( -# # When ready, should be changed to 'https://scicrunch.org/api/1/' for production -# apiEndpoint = TEST -# ) -# interlex_remote_test.setup(instrumented=OntTerm) + if server: + server = server if server.endswith('.') else server + '.' + endpoint = f'https://{server}scicrunch.org/api/1/' + + # + interlex_remote = InterLexRemote() + + # setup inheritance classes + interlex_remote.setup(instrumented=OntTerm) + interlex_remote.apiEndpoint = endpoint + + return interlex_remote diff --git a/ilxutils/ilxutils/scicrunch_session.py b/ilxutils/ilxutils/scicrunch_session.py new file mode 100644 index 00000000..b1b0bfa2 --- /dev/null +++ b/ilxutils/ilxutils/scicrunch_session.py @@ -0,0 +1,74 @@ +import json +import requests +from typing import Union, Dict, List +from urllib.parse import urljoin + + +class ScicrunchSession: + """ Boiler plate for SciCrunch server responses. """ + + def __init__(self, + key: str, + host: str = 'scicrunch.org', + auth: tuple = (None, None)) -> None: + """ Initialize Session with SciCrunch Server. + + :param str key: API key for SciCrunch [should work for test hosts]. + :param str host: Base url for hosting server [can take localhost:8080]. + :param str user: username for test server. + :param str password: password for test server. + """ + self.key = key + self.host = host + + # https is only for security level environments + if self.host.startswith('localhost'): + self.api = "http://" + self.host + '/api/1/' + else: + self.api = "https://" + self.host + '/api/1/' + + self.session = requests.Session() + self.session.auth = auth + self.session.headers.update({'Content-type': 'application/json'}) + + def __session_shortcut(self, endpoint: str, data: dict, session_type: str = 'GET') -> dict: + """ Short for both GET and POST. + + Will only crash if success is False or if there a 400+ error. + """ + def _prepare_data(data: dict) -> dict: + """ Check if request data inputed has key and proper format. """ + if data is None: + data = {'key': self.key} + elif isinstance(data, dict): + data.update({'key': self.key}) + else: + raise ValueError('request session data must be of type dictionary') + return json.dumps(data) + + url = urljoin(self.api, endpoint) + data = _prepare_data(data) + try: + # TODO: Could use a Request here to shorten code. + if session_type == 'GET': + response = self.session.get(url, data=data) + else: + response = self.session.post(url, data=data) + # crashes if success on the server side is False + if not response.json()['success']: + raise ValueError(response.text + f' -> STATUS CODE: {response.status_code}') + response.raise_for_status() + # crashes if the server couldn't use it or it never made it. + except requests.exceptions.HTTPError as error: + raise error + + # {'data':{}, 'success':bool} + return response.json()['data'] + + def get(self, endpoint: str, data: dict = None) -> dict: + """ Quick GET for SciCrunch. """ + return self.__session_shortcut(endpoint, data, 'GET') + + def post(self, endpoint: str , data: dict = None) -> dict: + """ Quick POST for SciCrunch. """ + return self.__session_shortcut(endpoint, data, 'POST') diff --git a/ilxutils/ilxutils/sql.py b/ilxutils/ilxutils/sql.py index c4176a71..b3470e74 100644 --- a/ilxutils/ilxutils/sql.py +++ b/ilxutils/ilxutils/sql.py @@ -1,4 +1,4 @@ -from interlex_sql import IlxSql +from .interlex_sql import IlxSql import os def production_sql(from_backup=True): From 53d71c4ff2323fe06c72aa0a781408f4dc5b3d8a Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Fri, 10 Apr 2020 13:28:58 -0700 Subject: [PATCH 07/17] updated tutorials --- .../tutorials/interlex_remotes_tutorial.ipynb | 182 +++++++----------- 1 file changed, 70 insertions(+), 112 deletions(-) diff --git a/ilxutils/tutorials/interlex_remotes_tutorial.ipynb b/ilxutils/tutorials/interlex_remotes_tutorial.ipynb index 85b28da4..db9004a3 100644 --- a/ilxutils/tutorials/interlex_remotes_tutorial.ipynb +++ b/ilxutils/tutorials/interlex_remotes_tutorial.ipynb @@ -42,24 +42,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "'NoneType' object has no attribute 'exists'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mfetch_grid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;31m# meta data in self.grid that has detials like bolding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mbrainstem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBrainstem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbrainstem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw_values\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Dropbox/git/pyontutils/pyontutils/sheets.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, sheet_name, fetch_grid, readonly)\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadonly\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreadonly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 211\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Dropbox/git/pyontutils/pyontutils/sheets.py\u001b[0m in \u001b[0;36m_setup\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadonly\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSheet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__spreadsheet_service_ro'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m \u001b[0mservice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_oauth_service\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreadonly\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadonly\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# I think it is correct to keep this ephimoral\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 217\u001b[0m \u001b[0mSheet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__spreadsheet_service_ro\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mservice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspreadsheets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Dropbox/git/pyontutils/pyontutils/sheets.py\u001b[0m in \u001b[0;36mget_oauth_service\u001b[0;34m(api, version, readonly, SCOPES)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[0mstore_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mauth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_auth_var\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mstore_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 62\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'exists'" - ] - } - ], + "outputs": [], "source": [ "from pyontutils.sheets import Sheet\n", "import pandas as pd\n", @@ -136,29 +121,17 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[OntTerm('HBA:3999', label='brain (hba)'),\n", - " OntTerm('FMA:50801', label='Brain'),\n", - " OntTerm('UBERON:0000955', label='brain'),\n", - " OntTerm('UBERON:6110636', label='adult cerebral ganglion'),\n", - " OntTerm('ILX:0101431', label='Brain'),\n", - " OntTerm('ILX:0101433', label='Brain Infarction'),\n", - " OntTerm('ILX:0506386', label='Brain Aneurysm'),\n", - " OntTerm('ILX:0433050', label='Brain Chemistry'),\n", - " OntTerm('ILX:0641746', label='alpha BRAIN'),\n", - " OntTerm('ILX:0726394', label='brain meninx'),\n", - " OntTerm('ILX:0729002', label='brain commissure'),\n", - " OntTerm('ILX:0101434', label='Brain Ischemia'),\n", - " OntTerm('ILX:0461406', label='Brain Death'),\n", - " OntTerm('ILX:0733041', label='brain endothelium')]" + "[OntTerm('UBERON:0000955', label='brain'),\n", + " OntTerm('UBERON:6110636', label='adult cerebral ganglion')]" ] }, - "execution_count": 3, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -171,17 +144,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[OntTerm('ILX:0103358', label='DN1 neuron'),\n", - " OntTerm('ILX:0109525', label='Pupal DN1 period neuron')]" + "[]" ] }, - "execution_count": 4, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -189,12 +161,12 @@ "source": [ "# similar entities will show\n", "# default limit is 10\n", - "query(term='DN1 neuron', limit=2) " + "query(term='brain', limit=10, prefix=('ILX')) " ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -203,7 +175,7 @@ "[OntTerm('UBERON:0000955', label='brain')]" ] }, - "execution_count": 5, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -215,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -223,20 +195,6 @@ "text/plain": [ "{'prefix': 'UBERON',\n", " 'suffix': '0000955',\n", - " 'orig_kwargs': {'iri': 'http://purl.obolibrary.org/obo/UBERON_0000955',\n", - " 'curie_or_iri': None,\n", - " 'label': None,\n", - " 'term': None,\n", - " 'search': None,\n", - " 'validated': None,\n", - " 'query': None},\n", - " 'kwargs': {'iri': 'http://purl.obolibrary.org/obo/UBERON_0000955',\n", - " 'curie_or_iri': None,\n", - " 'label': None,\n", - " 'term': None,\n", - " 'search': None,\n", - " 'validated': None,\n", - " 'query': None},\n", " 'label': 'brain',\n", " 'labels': ['brain'],\n", " 'definition': 'The brain is the center of the nervous system in all vertebrate, and most invertebrate, animals. Some primitive animals such as jellyfish and starfish have a decentralized nervous system without a brain, while sponges lack any nervous system at all. In vertebrates, the brain is located in the head, protected by the skull and close to the primary sensory apparatus of vision, hearing, balance, taste, and smell[WP].',\n", @@ -250,12 +208,12 @@ " '_type': OntId('owl:Class'),\n", " '_types': (OntId('owl:Class'),),\n", " '_graph': None,\n", - " '_source': ,\n", + " '_source': ,\n", " 'validated': True,\n", - " '_query_result': QueryResult({'iri': 'http://purl.obolibrary.org/obo/UBERON_0000955', 'curie': 'UBERON:0000955', 'label': 'brain', 'labels': ['brain'], 'definition': 'The brain is the center of the nervous system in all vertebrate, and most invertebrate, animals. Some primitive animals such as jellyfish and starfish have a decentralized nervous system without a brain, while sponges lack any nervous system at all. In vertebrates, the brain is located in the head, protected by the skull and close to the primary sensory apparatus of vision, hearing, balance, taste, and smell[WP].', 'synonyms': ['the brain', 'synganglion', 'suprasegmental structures', 'suprasegmental levels of nervous system', 'encephalon'], 'deprecated': False, 'predicates': {}, 'type': OntId('owl:Class'), 'types': (OntId('owl:Class'),), '_graph': None, 'source': })}" + " '_query_result': QueryResult({'iri': 'http://purl.obolibrary.org/obo/UBERON_0000955', 'curie': 'UBERON:0000955', 'label': 'brain', 'labels': ['brain'], 'definition': 'The brain is the center of the nervous system in all vertebrate, and most invertebrate, animals. Some primitive animals such as jellyfish and starfish have a decentralized nervous system without a brain, while sponges lack any nervous system at all. In vertebrates, the brain is located in the head, protected by the skull and close to the primary sensory apparatus of vision, hearing, balance, taste, and smell[WP].', 'synonyms': ['the brain', 'synganglion', 'suprasegmental structures', 'suprasegmental levels of nervous system', 'encephalon'], 'deprecated': False, 'predicates': {}, 'type': OntId('owl:Class'), 'types': (OntId('owl:Class'),), '_graph': None, 'source': })}" ] }, - "execution_count": 6, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -277,7 +235,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -301,9 +259,12 @@ "\u001b[0;34m\u001b[0m \u001b[0mdirection\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'OUTGOING'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0minclude_deprecated\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0minclude_supers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0minclude_all_services\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mraw\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mType:\u001b[0m OntQueryCli\n", - "\u001b[0;31mString form:\u001b[0m \n", + "\u001b[0;31mString form:\u001b[0m \n", "\u001b[0;31mFile:\u001b[0m ~/Dropbox/git/ontquery/ontquery/query.py\n", "\u001b[0;31mDocstring:\u001b[0m \n" ] @@ -332,7 +293,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -350,7 +311,7 @@ " [OntTerm('UBERON:6110636', label='adult cerebral ganglion')])]" ] }, - "execution_count": 8, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -391,30 +352,12 @@ " # We do this in case 2+ queries return the same results & the output WILL NOT have the same input order\n", " gin = lambda kwargs: (kwargs, query(**kwargs))\n", " # run each query instance at the same time\n", - " results = Async(use_nest_asyncio=True)(deferred(gin)(kwargs) for kwargs in kwargs_list)\n", + " results = Async()(deferred(gin)(kwargs) for kwargs in kwargs_list)\n", " return results \n", "\n", "queries([{'curie':'UBERON:0000955'}, {'curie':'UBERON:6110636'}])" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pyontutils.utils import Async, deferred\n", - "from pyontutils.core import OntTerm, ixr, query\n", - "from typing import List, Tuple\n", - "def queries(url_list:List[dict]) -> List[Tuple[str, dict]]:\n", - " def gin(url):\n", - " return requests.get(url).text\n", - " # run each query instance at the same time\n", - " results = Async(limit=5)(deferred(gin)(url) for url in url_list)\n", - " return results \n", - "list_tuples(url, html)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -424,12 +367,13 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "# TEST InterLex endpoints\n", - "from ilxutils.remotes import interlex_remote_test as ixrt" + "from ilxutils.remotes import remote\n", + "ixrt = remote(server='test3')" ] }, { @@ -441,7 +385,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -453,35 +397,35 @@ " 'orig_cid': '0',\n", " 'cid': '0',\n", " 'ilx': 'ilx_0738390',\n", - " 'label': 'Offical label',\n", + " 'label': 'official test label',\n", " 'type': 'term',\n", - " 'definition': 'official definition',\n", + " 'definition': 'definition',\n", " 'comment': 'helpful misc',\n", - " 'version': '3',\n", + " 'version': '2',\n", " 'status': '0',\n", " 'display_superclass': '1',\n", " 'orig_time': '1564695195',\n", - " 'time': '1570826848',\n", - " 'synonyms': [{'id': '1776645',\n", + " 'time': '1564695333',\n", + " 'synonyms': [{'id': '1776589',\n", " 'tid': '661544',\n", " 'literal': 'Encephalon',\n", " 'type': '',\n", - " 'time': '1570826848',\n", - " 'version': '3'},\n", - " {'id': '1776646',\n", + " 'time': '1564695333',\n", + " 'version': '2'},\n", + " {'id': '1776590',\n", " 'tid': '661544',\n", " 'literal': 'Cerebro',\n", " 'type': '',\n", - " 'time': '1570826848',\n", - " 'version': '3'}],\n", + " 'time': '1564695333',\n", + " 'version': '2'}],\n", " 'superclasses': [],\n", - " 'existing_ids': [{'id': '3885545',\n", + " 'existing_ids': [{'id': '3885425',\n", " 'tid': '661544',\n", " 'curie': 'ILX:0738390',\n", " 'iri': 'http://uri.interlex.org/base/ilx_0738390',\n", " 'curie_catalog_id': '3885424',\n", - " 'version': '3',\n", - " 'time': '1570826848',\n", + " 'version': '2',\n", + " 'time': '1564695334',\n", " 'preferred': '1'}],\n", " 'relationships': [],\n", " 'mappings': [],\n", @@ -489,7 +433,7 @@ " 'ontologies': []}" ] }, - "execution_count": 2, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -507,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -522,9 +466,11 @@ "\u001b[0;34m\u001b[0m \u001b[0msynonyms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mcomment\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mexisting_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mcid\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mDocstring:\u001b[0m \n", - "\u001b[0;31mFile:\u001b[0m ~/Dropbox/git/ontquery/ontquery/plugins/services.py\n", + "\u001b[0;31mFile:\u001b[0m ~/Dropbox/git/ontquery/ontquery/plugins/services/interlex.py\n", "\u001b[0;31mType:\u001b[0m method\n" ] }, @@ -538,14 +484,14 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "QueryResult({'iri': 'http://uri.interlex.org/base/ilx_0738390', 'curie': 'ILX:0738390', 'label': 'official test label', 'labels': (), 'definition': 'definition', 'synonyms': ('Encephalon', 'Cerebro'), 'deprecated': None, 'predicates': {'comment': 'helpful misc'}, 'type': None, 'types': (), '_graph': None, 'source': })\n" + "QueryResult({'iri': 'http://uri.interlex.org/base/ilx_0738390', 'curie': 'ILX:0738390', 'label': 'official test label', 'labels': (), 'definition': 'definition', 'synonyms': ('Encephalon', 'Cerebro'), 'deprecated': None, 'predicates': {'comment': 'helpful misc'}, 'type': None, 'types': (), '_graph': None, 'source': })\n" ] } ], @@ -574,7 +520,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -590,10 +536,20 @@ "\u001b[0;34m\u001b[0m \u001b[0msynonyms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mcomment\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mpredicates_to_add\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0madd_existing_ids\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mdelete_existing_ids\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m \u001b[0mpredicates_to_delete\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mcid\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mDocstring:\u001b[0m \n", - "\u001b[0;31mFile:\u001b[0m ~/Dropbox/git/ontquery/ontquery/plugins/services.py\n", + "\u001b[0;31mDocstring:\u001b[0m\n", + "Update existing entity.\n", + "\n", + ":param List[dict] add_existing_ids: iris and curies to be added to entity.\n", + ":param List[dict] delete_existing_ids: iris and curies to be deleted from entity.\n", + "\n", + ">>>update_entity(add_existing_ids=[{'ilx_id':'ilx_1234567', 'iri':'http://abc.org/abc_123', 'curie':'ABC:123'}])\n", + ">>>update_entity(delete_existing_ids=[{'ilx_id':'ilx_1234567', 'iri':'http://abc.org/abc_123', 'curie':'ABC:123'}])\n", + "\u001b[0;31mFile:\u001b[0m ~/Dropbox/git/ontquery/ontquery/plugins/services/interlex.py\n", "\u001b[0;31mType:\u001b[0m method\n" ] }, @@ -607,26 +563,27 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m[2019-10-11 13:47:28,619]\u001b[0m - \u001b[32m INFO\u001b[0m - ontquery - \u001b[34minterlex_client.py:796 \u001b[0m - {'ilx_id': 'ILX:0738390', 'label': 'Offical label', 'type': 'term', 'definition': 'official definition', 'comment': 'helpful misc', 'superclass': '', 'synonyms': ['Encephalon', 'Cerebro']}\u001b[0m\n" + "\u001b[32m[2020-04-10 13:25:53,802]\u001b[0m - \u001b[32m INFO\u001b[0m - ontquery - \u001b[34minterlex_client.py:962 \u001b[0m - {'ilx_id': 'ILX:0738390', 'label': 'Offical label', 'type': 'term', 'definition': 'official definition', 'comment': 'helpful misc', 'superclass': '', 'synonyms': ['Encephalon', 'Cerebro'], 'add_existing_ids': None, 'delete_existing_ids': None, 'status': '0', 'cid': None}\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "QueryResult({'iri': 'http://uri.interlex.org/base/ilx_0738390', 'curie': 'ILX:0738390', 'label': 'Offical label', 'labels': (), 'definition': 'official definition', 'synonyms': ('Encephalon', 'Cerebro'), 'deprecated': None, 'predicates': {'comment': 'helpful misc'}, 'type': None, 'types': (), '_graph': None, 'source': })\n" + "QueryResult({'iri': 'http://uri.interlex.org/base/ilx_0738390', 'curie': 'ILX:0738390', 'label': 'Offical label', 'labels': (), 'definition': 'official definition', 'synonyms': ('Encephalon', 'Cerebro'), 'deprecated': None, 'predicates': {'comment': 'helpful misc'}, 'type': None, 'types': (), '_graph': None, 'source': })\n" ] } ], "source": [ - "from ilxutils.remotes import interlex_remote_test as ixrt\n", + "from ilxutils.remotes import remote\n", + "ixrt = remote(server='test3')\n", "entity = dict(\n", " ilx_id = 'ILX:0738390',\n", " label = 'Offical label', # Can only one unique label per person\n", @@ -654,12 +611,13 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "# PRODUCTION \n", - "from ilxutils.remotes import interlex_remote_production as ixr\n", + "from ilxutils.remotes import remote \n", + "ixr = remote()\n", "# BE CAREFUL :)" ] } @@ -680,7 +638,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.7.6" } }, "nbformat": 4, From 3e5e5c99f7a89e4afd2b2dd8bf74a198ee7adc1b Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Fri, 10 Apr 2020 14:33:25 -0700 Subject: [PATCH 08/17] fixed bugs --- ilxutils/ilxutils/interlex_sql.py | 223 ++++++++++-------- ilxutils/ilxutils/remotes.py | 2 +- ilxutils/ilxutils/scicrunch_session.py | 113 ++++++++- .../tutorials/interlex_remotes_tutorial.ipynb | 59 +++++ 4 files changed, 284 insertions(+), 113 deletions(-) diff --git a/ilxutils/ilxutils/interlex_sql.py b/ilxutils/ilxutils/interlex_sql.py index ce45497b..733c410f 100755 --- a/ilxutils/ilxutils/interlex_sql.py +++ b/ilxutils/ilxutils/interlex_sql.py @@ -1,33 +1,84 @@ +from collections import defaultdict +import os from pathlib import Path +from typing import Union, Dict, Tuple, List + import pandas as pd from sqlalchemy import create_engine, inspect, Table, Column -from collections import defaultdict + from ilxutils.tools import light_degrade, open_pickle, create_pickle -import os -#ELASTIC = 'https://5f86098ac2b28a982cebf64e82db4ea2.us-west-2.aws.found.io:9243/interlex/term/' -TERMS_COMPLETE_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_terms_complete_backup.pickle' -TERMS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_terms_backup.pickle' -ANNOS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_annotations_backup.pickle' -RELAS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_relationships_backup.pickle' -SUPER_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_superclasses_backup.pickle' -SYNOS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_synonyms_backup.pickle' -EXIDS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_ex_backup.pickle' + + +TERMSC_BACKUP = 'ilx_db_terms_complete_backup.pickle' +TERMS_BACKUP = 'ilx_db_terms_backup.pickle' +ANNOS_BACKUP = 'ilx_db_annotations_backup.pickle' +RELAS_BACKUP = 'ilx_db_relationships_backup.pickle' +SUPER_BACKUP = 'ilx_db_superclasses_backup.pickle' +SYNOS_BACKUP = 'ilx_db_synonyms_backup.pickle' +EXIDS_BACKUP = 'ilx_db_existing_ids_backup.pickle' + + +def pathing(path, check_path=False): + """ Extract absolute path from shortened or relative paths. + + :param str path: path to file or folder. + + :examples: + >>>pathing('~/shortened.filepath') + >>>pathing('../relative.filepath') + >>>pathing('relative.filepath') + >>>pathing('/home/absoulte/filepath') + """ + path = Path(path) + if str(path).startswith('~'): + path = path.expanduser() + else: + path = path.resolve() + + if check_path: + if not path.is_file() and path.is_dir(): + raise ValueError(f'{path} does not exit') + + return path class IlxSql(): - def __init__(self, db_url, pre_load=False, from_backup=False): - self.db_url = db_url - self.engine = create_engine(self.db_url) - self.local_degrade = lambda string: string.lower().strip() # current degrade of choice for sql + def __init__(self, + db_url: str, + from_backup: bool = False, + pre_load: bool = False, + backups_folder: str = '~/.interlex_backups'): + + self.engine = create_engine(db_url) self.from_backup = from_backup - self.terms_complete = self.get_terms_complete() if pre_load else pd.DataFrame - self.terms = self.get_terms() if pre_load else pd.DataFrame - self.superclasses = self.get_superclasses if pre_load else pd.DataFrame - self.annotations = self.get_annotations() if pre_load else pd.DataFrame - self.existing_ids = self.get_existing_ids() if pre_load else pd.DataFrame - self.relationships = self.get_relationships() if pre_load else pd.DataFrame - self.synonyms = self.get_synonyms() if pre_load else pd.DataFrame + + self.save_folder = pathing(backups_folder) + try: + self.save_folder.mkdir() + except: + pass + + self.terms = pd.DataFrame + self.superclasses = pd.DataFrame + self.annotations = pd.DataFrame + self.existing_ids = pd.DataFrame + self.relationships = pd.DataFrame + self.synonyms = pd.DataFrame + + # Auto load tables from backup + if pre_load: + self.pre_loader() + + + def pre_loader(self): + # self.terms_complete = self.get_terms_complete() if pre_load else pd.DataFrame + self.terms = self.get_terms() + # self.superclasses = self.get_superclasses() + self.existing_ids = self.get_existing_ids() + self.synonyms = self.get_synonyms() + # self.relationships = self.get_relationships() + # self.annotations = self.get_annotations() def fetch_terms_complete(self): if self.terms_complete.empty: @@ -66,29 +117,24 @@ def fetch_superclasses(self): def get_terms(self): ''' GROUP BY is a shortcut to only getting the first in every list of group ''' - if not self.terms.empty: - return self.terms if self.from_backup: - self.terms = open_pickle(TERMS_BACKUP_PATH) + self.terms = open_pickle(self.save_folder / TERMS_BACKUP) return self.terms - engine = create_engine(self.db_url) - data = """ + sql_query = """ SELECT t.id as tid, t.ilx, t.label, t.definition, t.type, t.comment, t.version, t.uid, t.cid, t.time, t.status FROM terms t GROUP BY t.ilx + HAVING t.status = '0' """ - self.terms = pd.read_sql(data, engine) - create_pickle(self.terms, TERMS_BACKUP_PATH) + self.terms = pd.read_sql(sql_query, self.engine) + create_pickle(self.terms, self.save_folder / TERMS_BACKUP) return self.terms def get_annotations(self): - if not self.annotations: - return self.fetch_annotations() if self.from_backup: - self.annotations = open_pickle(ANNOS_BACKUP_PATH) + self.annotations = open_pickle(self.save_folder / ANNOS_BACKUP) return self.annotations - engine = create_engine(self.db_url) - data = """ + sql_query = """ SELECT ta.tid, ta.annotation_tid as annotation_type_tid, t1.ilx as term_ilx, t2.ilx as annotation_type_ilx, @@ -99,46 +145,43 @@ def get_annotations(self): SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) AS t1 ON ta.tid=t1.id JOIN ( SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) AS t2 ON ta.annotation_tid=t2.id """ - self.annotations = pd.read_sql(data, engine) - create_pickle(self.annotations, ANNOS_BACKUP_PATH) + self.annotations = pd.read_sql(sql_query, self.engine) + create_pickle(self.annotations, self.save_folder / ANNOS_BACKUP) return self.annotations def get_existing_ids(self): - if not self.existing_ids.empty: - return self.existing_ids if self.from_backup: - self.existing_ids = open_pickle(EXIDS_BACKUP_PATH) + self.existing_ids = open_pickle(self.save_folder / EXIDS_BACKUP) return self.existing_ids - engine = create_engine(self.db_url) - data = """ + sql_query = """ SELECT tei.tid, tei.curie, tei.iri, tei.preferred, t.ilx, t.label, t.definition, t.status FROM ( SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) as t JOIN term_existing_ids AS tei ON t.id = tei.tid """ - self.existing_ids = pd.read_sql(data, engine) - create_pickle(self.existing_ids, EXIDS_BACKUP_PATH) + self.existing_ids = pd.read_sql(sql_query, self.engine) + create_pickle(self.existing_ids, self.save_folder / EXIDS_BACKUP) return self.existing_ids def get_relationships(self): - if not self.relationships.empty: - return self.relationships if self.from_backup: - self.relationships = open_pickle(RELAS_BACKUP_PATH) + self.relationships = open_pickle(self.save_folder / RELAS_BACKUP) return self.relationships - engine = create_engine(self.db_url) - data = """ + sql_query = """ SELECT t1.id as term1_tid, t1.ilx AS term1_ilx, t1.type as term1_type, t1.label as term1_label, t2.id as term2_tid, t2.ilx AS term2_ilx, t2.type as term2_type, t2.label as term2_label, @@ -163,18 +206,15 @@ def get_relationships(self): HAVING terms.status = '0' ) AS t3 ON t3.id = tr.relationship_tid """ - self.relationships = pd.read_sql(data, engine) - create_pickle(self.relationships, RELAS_BACKUP_PATH) + self.relationships = pd.read_sql(sql_query, self.engine) + create_pickle(self.relationships, self.save_folder / RELAS_BACKUP) return self.relationships def get_superclasses(self): - if not self.superclasses.empty: - return self.superclasses if self.from_backup: - self.superclasses = open_pickle(SUPER_BACKUP_PATH) + self.superclasses = open_pickle(self.save_folder / SUPER_BACKUP) return self.superclasses - engine = create_engine(self.db_url) - data = """ + sql_query = """ SELECT ts.tid, ts.superclass_tid, t1.label as term_label, t1.ilx as term_ilx, @@ -184,46 +224,44 @@ def get_superclasses(self): SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) as t1 ON t1.id = ts.tid JOIN ( SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) AS t2 ON t2.id = ts.superclass_tid """ - self.superclasses = pd.read_sql(data, engine) - create_pickle(self.superclasses, SUPER_BACKUP_PATH) + self.superclasses = pd.read_sql(sql_query, self.engine) + create_pickle(self.superclasses, self.save_folder / SUPER_BACKUP) return self.superclasses def get_synonyms(self): - if not self.synonyms.empty: - return self.synonyms if self.from_backup: - self.synonyms = open_pickle(SYNOS_BACKUP_PATH) + self.synonyms = open_pickle(self.save_folder / SYNOS_BACKUP) return self.synonyms - engine = create_engine(self.db_url) - data = """ + sql_query = """ SELECT ts.tid as tid, t.ilx, ts.literal, ts.type FROM term_synonyms AS ts JOIN ( SELECT * FROM terms GROUP BY terms.ilx + HAVING terms.status = '0' ) AS t WHERE ts.tid=t.id """ - self.synonyms = pd.read_sql(data, engine) - create_pickle(self.synonyms, SYNOS_BACKUP_PATH) + self.synonyms = pd.read_sql(sql_query, self.engine) + create_pickle(self.synonyms, self.save_folder / SYNOS_BACKUP) return self.synonyms def get_terms_complete(self) -> pd.DataFrame: ''' Gets complete entity data like term/view ''' - if not self.terms_complete.empty: - return self.terms_complete if self.from_backup: - self.terms_complete = open_pickle(TERMS_COMPLETE_BACKUP_PATH) + self.terms_complete = open_pickle(self.save_folder / TERMSC_BACKUP) return self.terms_complete ilx2synonyms = self.get_ilx2synonyms() ilx2existing_ids = self.get_ilx2existing_ids() @@ -239,37 +277,22 @@ def get_terms_complete(self) -> pd.DataFrame: row['superclass'] = ilx2superclass.get(row['ilx']) ilx_complete.append(row) terms_complete = pd.DataFrame(ilx_complete) - create_pickle(terms_complete, TERMS_COMPLETE_BACKUP_PATH) + create_pickle(terms_complete, self.save_folder / TERMSC_BACKUP) return terms_complete - def get_label2id(self): - self.terms = self.fetch_terms() - visited = {} - label_to_id = defaultdict(lambda: defaultdict(list)) - for row in self.terms.itertuples(): - label = self.local_degrade(row.label) - if not visited.get((label, row.type, row.ilx)): - if row.type == 'term': - label_to_id[label]['term'].append(int(row.id)) - visited[(label, row.type, row.ilx)] = True - elif row.type == 'cde': - label_to_id[label]['cde'].append(int(row.id)) - visited[(label, row.type, row.ilx)] = True - elif row.type == 'fde': - label_to_id[label]['fde'].append(int(row.id)) - visited[(label, row.type, row.ilx)] = True - return label_to_id - - def get_label2ilxs(self): - self.terms = self.fetch_terms() - visited = {} - label_to_ilx = defaultdict(list) - for row in self.terms.itertuples(): - label = self.local_degrade(row.label) - if not visited.get((label, row.type, row.ilx)): - label_to_ilx[label].append(str(row.ilx)) - visited[(label, row.type, row.ilx)] = True - return label_to_ilx + def get_label2id(self, clean: object = None) -> Dict[str, list]: + if not clean: + clean = lambda string: string.lower().strip() + label2id = defaultdict(list) + [label2id[clean(row.label)].append(row.id) for row in self.fetch_terms().itertuples()] + return label2id + + def get_label2ilx(self, clean: object = None) -> Dict[str, list]: + if not clean: + clean = lambda string: string.lower().strip() + label2ilx = defaultdict(list) + [label2ilx[clean(row.label)].append(row.ilx) for row in self.fetch_terms().itertuples()] + return label2ilx def get_label2rows(self): self.terms_complete = self.fetch_terms_complete() @@ -446,19 +469,19 @@ def get_fragment2rows(self): return fragement2rows def show_tables(self): - data = "SHOW tables;" - return pd.read_sql(data, self.engine) + sql_query = "SHOW tables;" + return pd.read_sql(sql_query, self.engine) def get_table(self, tablename, limit=5): - data = """ + sql_query = """ SELECT * FROM {tablename} LIMIT {limit} """.format(tablename=tablename, limit=limit) - return pd.read_sql(data, self.engine) + return pd.read_sql(sql_query, self.engine) def get_custom(self, data): - return pd.read_sql(data, self.engine) + return pd.read_sql(sql_query, self.engine) def main(): diff --git a/ilxutils/ilxutils/remotes.py b/ilxutils/ilxutils/remotes.py index 711766c2..76d99fd3 100644 --- a/ilxutils/ilxutils/remotes.py +++ b/ilxutils/ilxutils/remotes.py @@ -8,7 +8,7 @@ def remote(server=''): InterLexRemote = oq.plugin.get('InterLex') if server: - server = server if server.endswith('.') else server + '.' + server = server if server.endswith('.') else server + '.' endpoint = f'https://{server}scicrunch.org/api/1/' # diff --git a/ilxutils/ilxutils/scicrunch_session.py b/ilxutils/ilxutils/scicrunch_session.py index b1b0bfa2..459d8d13 100644 --- a/ilxutils/ilxutils/scicrunch_session.py +++ b/ilxutils/ilxutils/scicrunch_session.py @@ -1,16 +1,26 @@ import json -import requests -from typing import Union, Dict, List +from typing import Union, Dict, List, Tuple from urllib.parse import urljoin +import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +import nest_asyncio +nest_asyncio.apply() + +from pyontutils.utils import Async, deferred -class ScicrunchSession: + +class SciCrunchSession: """ Boiler plate for SciCrunch server responses. """ def __init__(self, key: str, - host: str = 'scicrunch.org', - auth: tuple = (None, None)) -> None: + host: str = 'scicrunch.org', # MAIN TEST -> test3.scicrunch.org + auth: tuple = ('scicrunch', 'perl22(query)'), + retries: int = 3, + backoff_factor: float = 1.0, + status_forcelist: tuple = (400, 500, 502, 504),) -> None: """ Initialize Session with SciCrunch Server. :param str key: API key for SciCrunch [should work for test hosts]. @@ -30,6 +40,16 @@ def __init__(self, self.session = requests.Session() self.session.auth = auth self.session.headers.update({'Content-type': 'application/json'}) + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, # 400 for no ILX ID generated. + ) + # adapter = HTTPAdapter(max_retries=retry) + # self.session.mount('http://', adapter) + # self.session.mount('https://', adapter) def __session_shortcut(self, endpoint: str, data: dict, session_type: str = 'GET') -> dict: """ Short for both GET and POST. @@ -46,7 +66,13 @@ def _prepare_data(data: dict) -> dict: raise ValueError('request session data must be of type dictionary') return json.dumps(data) + # urljoin bug; .com/ap1/1/ + /test/ != .com/ap1/1/test/ but .com/test/ + # HOWEVER .com/ap1/1/ + test/ == .com/ap1/1/test/ + endpoint = endpoint[1:] if endpoint.startswith('/') else endpoint url = urljoin(self.api, endpoint) + if data: + for key, value in data.items(): + url = url.format(**{key:value}) data = _prepare_data(data) try: # TODO: Could use a Request here to shorten code. @@ -55,20 +81,83 @@ def _prepare_data(data: dict) -> dict: else: response = self.session.post(url, data=data) # crashes if success on the server side is False - if not response.json()['success']: - raise ValueError(response.text + f' -> STATUS CODE: {response.status_code}') + if response.json()['success'] == False: + # Need to retry if server fails to create the ILX ID. + if response.json().get('errormsg') == 'could not generate ILX identifier': + return response.json() + raise ValueError(response.text + f' -> STATUS CODE: {response.status_code} @ URL: {response.url}') response.raise_for_status() # crashes if the server couldn't use it or it never made it. - except requests.exceptions.HTTPError as error: - raise error + except: + raise requests.exceptions.HTTPError(f'{response.text} {response.status_code}') - # {'data':{}, 'success':bool} + # response.json() == {'data':{}, 'success':bool} return response.json()['data'] - def get(self, endpoint: str, data: dict = None) -> dict: + def _get(self, endpoint: str, data: dict = None) -> dict: """ Quick GET for SciCrunch. """ return self.__session_shortcut(endpoint, data, 'GET') - def post(self, endpoint: str , data: dict = None) -> dict: + def _post(self, endpoint: str , data: dict = None) -> dict: """ Quick POST for SciCrunch. """ return self.__session_shortcut(endpoint, data, 'POST') + + def get(self, func, data_list=None) -> List[Tuple[str, dict]]: + if not data_list: + return self._get(endpoint) + # worker + gin = lambda data: self._get(endpoint, data) + # Builds futures dynamically + return Async()(deferred(gin)(data) for data in data_list) + + def post(self, func: object, data_list: list) -> List[Tuple[str, dict]]: + # worker; return server_response first then initial data input + gin = lambda data: (data, func(data)) + + # Builds futures dynamically + responses = Async()(deferred(gin)(data) for data in data_list) + + # BUG: ilx_ids are created on the PHP side and are slow. Duplicates + # are known to be created "func hit at same time" so we need to a new + # session and try again. + number_of_batch_retries = 0 + while number_of_batch_retries < 10: + data_queue = [] + for response in responses: + data, server_response = response + print(server_response) + if server_response.get('errormsg') == 'could not generate ILX identifier': + data_queue.append(data) + if data_queue == []: + break + responses = Async()(deferred(gin)(data) for data in data_queue) + number_of_batch_retries += 1 + return + + def get(self, urls, limit=5): + + async def get_single(url, session, auth): + async with session.get(url) as response: + try: + output = await response.json() + except: + output = await response.text() + ValueError(f'{output} with status code [{response.status}]') + return output + + async def get_all(urls, connector, loop): + tasks = [] + async with ClientSession(connector=connector, loop=loop, + auth=self.auth, raise_for_status=True) as session: + for i, url in enumerate(urls): + task = asyncio.ensure_future(get_single(url, session, self.auth)) + tasks.append(task) + return (await asyncio.gather(*tasks)) + + # rate limiter; should be between 20 and 80; 100 maxed out server + connector = TCPConnector(limit=limit) + loop = asyncio.get_event_loop() # event loop initialize + # tasks to do; data is in json format [{},] + future = asyncio.ensure_future(get_all(urls, connector, loop)) + outputs = loop.run_until_complete(future) # loop until done + return {k: v for keyval in outputs for k, v in keyval.items()} diff --git a/ilxutils/tutorials/interlex_remotes_tutorial.ipynb b/ilxutils/tutorials/interlex_remotes_tutorial.ipynb index db9004a3..5a27e35a 100644 --- a/ilxutils/tutorials/interlex_remotes_tutorial.ipynb +++ b/ilxutils/tutorials/interlex_remotes_tutorial.ipynb @@ -620,6 +620,65 @@ "ixr = remote()\n", "# BE CAREFUL :)" ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': '661544',\n", + " 'orig_uid': '34142',\n", + " 'uid': '34142',\n", + " 'orig_cid': '0',\n", + " 'cid': '0',\n", + " 'ilx': 'ilx_0738390',\n", + " 'label': 'Offical label',\n", + " 'type': 'term',\n", + " 'definition': 'official definition',\n", + " 'comment': 'helpful misc',\n", + " 'version': '3',\n", + " 'status': '0',\n", + " 'display_superclass': '1',\n", + " 'orig_time': '1564695195',\n", + " 'time': '1586550353',\n", + " 'synonyms': [{'id': '1845765',\n", + " 'tid': '661544',\n", + " 'literal': 'Encephalon',\n", + " 'type': '',\n", + " 'time': '1586550353',\n", + " 'version': '3'},\n", + " {'id': '1845766',\n", + " 'tid': '661544',\n", + " 'literal': 'Cerebro',\n", + " 'type': '',\n", + " 'time': '1586550353',\n", + " 'version': '3'}],\n", + " 'superclasses': [],\n", + " 'existing_ids': [{'id': '4972084',\n", + " 'tid': '661544',\n", + " 'curie': 'ILX:0738390',\n", + " 'iri': 'http://uri.interlex.org/base/ilx_0738390',\n", + " 'curie_catalog_id': '3885424',\n", + " 'version': '3',\n", + " 'time': '1586550353',\n", + " 'preferred': '1'}],\n", + " 'relationships': [],\n", + " 'mappings': [],\n", + " 'annotations': [],\n", + " 'ontologies': []}" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ixrt.ilx_cli.get_entity('ilx_0738390')" + ] } ], "metadata": { From e75362f0639e7eb67981e3ff24517c3dada781e6 Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Wed, 20 May 2020 17:16:28 -0700 Subject: [PATCH 09/17] bug fixes --- ilxutils/ilxutils/interlex_sql.py | 2 +- ilxutils/ilxutils/scicrunch_session.py | 176 ++++++++++++++----------- 2 files changed, 101 insertions(+), 77 deletions(-) diff --git a/ilxutils/ilxutils/interlex_sql.py b/ilxutils/ilxutils/interlex_sql.py index 733c410f..1cf7e509 100755 --- a/ilxutils/ilxutils/interlex_sql.py +++ b/ilxutils/ilxutils/interlex_sql.py @@ -480,7 +480,7 @@ def get_table(self, tablename, limit=5): """.format(tablename=tablename, limit=limit) return pd.read_sql(sql_query, self.engine) - def get_custom(self, data): + def get_custom(self, sql_query): return pd.read_sql(sql_query, self.engine) diff --git a/ilxutils/ilxutils/scicrunch_session.py b/ilxutils/ilxutils/scicrunch_session.py index 459d8d13..a2030575 100644 --- a/ilxutils/ilxutils/scicrunch_session.py +++ b/ilxutils/ilxutils/scicrunch_session.py @@ -5,8 +5,6 @@ import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry -import nest_asyncio -nest_asyncio.apply() from pyontutils.utils import Async, deferred @@ -14,28 +12,50 @@ class SciCrunchSession: """ Boiler plate for SciCrunch server responses. """ + class Error(Exception): + """Script could not complete.""" + + class NoApiKeyError(Error): + """ No api key has been set """ + + class IncorrectAPIKeyError(Error): + """Incorrect API key for scicrunch website used.""" + def __init__(self, key: str, - host: str = 'scicrunch.org', # MAIN TEST -> test3.scicrunch.org - auth: tuple = ('scicrunch', 'perl22(query)'), - retries: int = 3, - backoff_factor: float = 1.0, - status_forcelist: tuple = (400, 500, 502, 504),) -> None: + host: str = 'test3.scicrunch.org', # MAIN TEST -> test3.scicrunch.org + auth: tuple = ('', ''), # user, password for authentication + retries: int = 3, # retries if code in status_forcelist + backoff_factor: float = 1.0, # delay factor for reties + status_forcelist: tuple = (500, 502, 504), # flagged codes for retry + ) -> None: """ Initialize Session with SciCrunch Server. :param str key: API key for SciCrunch [should work for test hosts]. :param str host: Base url for hosting server [can take localhost:8080]. - :param str user: username for test server. - :param str password: password for test server. """ self.key = key - self.host = host + self.host = '' + self.api = '' + + # Pull host for potential url + if host.startswith('http'): + host = urlparse(host).netloc - # https is only for security level environments - if self.host.startswith('localhost'): - self.api = "http://" + self.host + '/api/1/' + # Use host to create api url + if host.startswith('localhost'): + self.host = "http://" + host + self.api = self.host + '/api/1/' else: - self.api = "https://" + self.host + '/api/1/' + self.host = "https://" + host + self.api = self.host + '/api/1/' + + # Api key check + if self.key is None: # injected by orthauth + # Error here because viewing without a key handled in InterLexRemote not here + raise self.NoApiKeyError('You have not set an API key for the SciCrunch API!') + if not requests.get(self.api+'user/info', params={'key':self.key}).status_code in [200, 201]: + raise self.IncorrectAPIKeyError(f'api_key given is incorrect.') self.session = requests.Session() self.session.auth = auth @@ -47,9 +67,9 @@ def __init__(self, backoff_factor=backoff_factor, status_forcelist=status_forcelist, # 400 for no ILX ID generated. ) - # adapter = HTTPAdapter(max_retries=retry) - # self.session.mount('http://', adapter) - # self.session.mount('https://', adapter) + adapter = HTTPAdapter(max_retries=retry) + self.session.mount('http://', adapter) + self.session.mount('https://', adapter) def __session_shortcut(self, endpoint: str, data: dict, session_type: str = 'GET') -> dict: """ Short for both GET and POST. @@ -74,17 +94,15 @@ def _prepare_data(data: dict) -> dict: for key, value in data.items(): url = url.format(**{key:value}) data = _prepare_data(data) + # TODO: Could use a Request here to shorten code. + if session_type == 'GET': + response = self.session.get(url, data=data) + else: + response = self.session.post(url, data=data) try: - # TODO: Could use a Request here to shorten code. - if session_type == 'GET': - response = self.session.get(url, data=data) - else: - response = self.session.post(url, data=data) # crashes if success on the server side is False if response.json()['success'] == False: # Need to retry if server fails to create the ILX ID. - if response.json().get('errormsg') == 'could not generate ILX identifier': - return response.json() raise ValueError(response.text + f' -> STATUS CODE: {response.status_code} @ URL: {response.url}') response.raise_for_status() # crashes if the server couldn't use it or it never made it. @@ -102,62 +120,68 @@ def _post(self, endpoint: str , data: dict = None) -> dict: """ Quick POST for SciCrunch. """ return self.__session_shortcut(endpoint, data, 'POST') - def get(self, func, data_list=None) -> List[Tuple[str, dict]]: - if not data_list: - return self._get(endpoint) + def get(self, endpoint, data_list, tag=None) -> List[Tuple[str, dict]]: # worker - gin = lambda data: self._get(endpoint, data) + gin = lambda endpoint, data: (tag, self._get(endpoint, data)) # Builds futures dynamically - return Async()(deferred(gin)(data) for data in data_list) + return Async()(deferred(gin)(endpoint, data) for endpoint, data in zip(endpoint, data_list)) - def post(self, func: object, data_list: list) -> List[Tuple[str, dict]]: + def post(self, endpoint: object, data_list: list) -> List[Tuple[str, dict]]: # worker; return server_response first then initial data input - gin = lambda data: (data, func(data)) + gin = lambda data: (data, self._post(endpoint, data)) # Builds futures dynamically responses = Async()(deferred(gin)(data) for data in data_list) - - # BUG: ilx_ids are created on the PHP side and are slow. Duplicates - # are known to be created "func hit at same time" so we need to a new - # session and try again. - number_of_batch_retries = 0 - while number_of_batch_retries < 10: - data_queue = [] - for response in responses: - data, server_response = response - print(server_response) - if server_response.get('errormsg') == 'could not generate ILX identifier': - data_queue.append(data) - if data_queue == []: - break - responses = Async()(deferred(gin)(data) for data in data_queue) - number_of_batch_retries += 1 - return - - def get(self, urls, limit=5): - - async def get_single(url, session, auth): - async with session.get(url) as response: - try: - output = await response.json() - except: - output = await response.text() - ValueError(f'{output} with status code [{response.status}]') - return output - - async def get_all(urls, connector, loop): - tasks = [] - async with ClientSession(connector=connector, loop=loop, - auth=self.auth, raise_for_status=True) as session: - for i, url in enumerate(urls): - task = asyncio.ensure_future(get_single(url, session, self.auth)) - tasks.append(task) - return (await asyncio.gather(*tasks)) - - # rate limiter; should be between 20 and 80; 100 maxed out server - connector = TCPConnector(limit=limit) - loop = asyncio.get_event_loop() # event loop initialize - # tasks to do; data is in json format [{},] - future = asyncio.ensure_future(get_all(urls, connector, loop)) - outputs = loop.run_until_complete(future) # loop until done - return {k: v for keyval in outputs for k, v in keyval.items()} + return responses + + # def post(self, func: object, data_list: list) -> List[Tuple[str, dict]]: + # # worker; return server_response first then initial data input + # gin = lambda data: (data, func(data)) + # + # # Builds futures dynamically + # responses = Async()(deferred(gin)(data) for data in data_list) + # + # # BUG: ilx_ids are created on the PHP side and are slow. Duplicates + # # are known to be created "func hit at same time" so we need to a new + # # session and try again. + # number_of_batch_retries = 0 + # while number_of_batch_retries < 10: + # data_queue = [] + # for response in responses: + # data, server_response = response + # print(server_response) + # if server_response.get('errormsg') == 'could not generate ILX identifier': + # data_queue.append(data) + # if data_queue == []: + # break + # responses = Async()(deferred(gin)(data) for data in data_queue) + # number_of_batch_retries += 1 + # return + + # def get(self, urls, limit=5): + # + # async def get_single(url, session, auth): + # async with session.get(url) as response: + # try: + # output = await response.json() + # except: + # output = await response.text() + # ValueError(f'{output} with status code [{response.status}]') + # return output + # + # async def get_all(urls, connector, loop): + # tasks = [] + # async with ClientSession(connector=connector, loop=loop, + # auth=self.auth, raise_for_status=True) as session: + # for i, url in enumerate(urls): + # task = asyncio.ensure_future(get_single(url, session, self.auth)) + # tasks.append(task) + # return (await asyncio.gather(*tasks)) + # + # # rate limiter; should be between 20 and 80; 100 maxed out server + # connector = TCPConnector(limit=limit) + # loop = asyncio.get_event_loop() # event loop initialize + # # tasks to do; data is in json format [{},] + # future = asyncio.ensure_future(get_all(urls, connector, loop)) + # outputs = loop.run_until_complete(future) # loop until done + # return {k: v for keyval in outputs for k, v in keyval.items()} From c0c16924e96d8989bf0d8f156b69ddcb7eb6c49e Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Thu, 1 Oct 2020 13:24:59 -0700 Subject: [PATCH 10/17] gradual updates --- ilxutils/ilxutils/backup_ilx.py | 4 +- ilxutils/ilxutils/elasticsearch_wrapper.py | 14 +- ilxutils/ilxutils/interlex_sanity_checks.py | 4 + ilxutils/ilxutils/interlex_sql.py | 51 +++- ilxutils/ilxutils/remotes.py | 2 +- ilxutils/ilxutils/scicrunch_session.py | 2 +- ilxutils/ilxutils/sparql.ipynb | 296 ++++++++++++++++++++ 7 files changed, 354 insertions(+), 19 deletions(-) create mode 100644 ilxutils/ilxutils/interlex_sanity_checks.py create mode 100644 ilxutils/ilxutils/sparql.ipynb diff --git a/ilxutils/ilxutils/backup_ilx.py b/ilxutils/ilxutils/backup_ilx.py index 2a94a962..20006cf1 100644 --- a/ilxutils/ilxutils/backup_ilx.py +++ b/ilxutils/ilxutils/backup_ilx.py @@ -4,10 +4,12 @@ import pickle import os + def create_pickle(data, outfilename): with open(outfilename, 'wb') as outfile: pickle.dump(data, outfile) - + + sql = IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_PRODUCTION')) diff --git a/ilxutils/ilxutils/elasticsearch_wrapper.py b/ilxutils/ilxutils/elasticsearch_wrapper.py index ee9e8d76..e9767e0e 100644 --- a/ilxutils/ilxutils/elasticsearch_wrapper.py +++ b/ilxutils/ilxutils/elasticsearch_wrapper.py @@ -36,6 +36,17 @@ def search(self, body: dict, **kwargs) -> dict: """ return self.es.search(index=self.type, body=body, **kwargs) + def scroll(self, body: dict, size: int, **kwargs) -> dict: + body['size'] = 10000 + body['from'] = 0 + hits = [] + print(body) + for step in range(0, size, 10000): + hits += self.es.search(index=self.type, body=body, **kwargs)['hits']['hits'] + body['from'] = step + print(body) + return hits + def all_matches(self, sorting: str, size, start) -> dict: """First or last set of entities. @@ -67,7 +78,8 @@ class InterLexES(ElasticSearchTools): def __init__(self, beta=True): super().__init__( - host = BASHRC('INTERLEX_ELASTIC_URL'), + host = BASHRC('SCICRUNCH_ELASTIC_URL'), + # index = 'interlex_2019oct28', index = 'interlex', type = 'term', user = BASHRC('INTERLEX_ELASTIC_USER'), diff --git a/ilxutils/ilxutils/interlex_sanity_checks.py b/ilxutils/ilxutils/interlex_sanity_checks.py new file mode 100644 index 00000000..d953953b --- /dev/null +++ b/ilxutils/ilxutils/interlex_sanity_checks.py @@ -0,0 +1,4 @@ +from .sql import production_sql + +ilx_sql = production_sql(from_backup=True) +ex = ilx_sql.get_existing_ids() diff --git a/ilxutils/ilxutils/interlex_sql.py b/ilxutils/ilxutils/interlex_sql.py index 1cf7e509..d1badb67 100755 --- a/ilxutils/ilxutils/interlex_sql.py +++ b/ilxutils/ilxutils/interlex_sql.py @@ -42,7 +42,7 @@ def pathing(path, check_path=False): return path -class IlxSql(): +class IlxSql: def __init__(self, db_url: str, @@ -266,6 +266,7 @@ def get_terms_complete(self) -> pd.DataFrame: ilx2synonyms = self.get_ilx2synonyms() ilx2existing_ids = self.get_ilx2existing_ids() ilx2annotations = self.get_ilx2annotations() + ilx2relationships = self.get_ilx2relationships() ilx2superclass = self.get_ilx2superclass() ilx_complete = [] header = ['Index'] + list(self.fetch_terms().columns) @@ -292,20 +293,34 @@ def get_label2ilx(self, clean: object = None) -> Dict[str, list]: clean = lambda string: string.lower().strip() label2ilx = defaultdict(list) [label2ilx[clean(row.label)].append(row.ilx) for row in self.fetch_terms().itertuples()] + [label2ilx[clean(row.literal)].append(row.ilx) for row in self.fetch_synonyms().itertuples()] return label2ilx - def get_label2rows(self): - self.terms_complete = self.fetch_terms_complete() - visited = {} - label2rows = defaultdict(list) - header = ['Index'] + list(self.terms_complete.columns) - for row in self.terms_complete.itertuples(): - row = {header[i]:val for i, val in enumerate(row)} - label = self.local_degrade(row['label']) - if not visited.get((label, row['type'], row['ilx'])): - label2rows[label].append(row) - visited[(label, row['type'], row['ilx'])] = True - return label2rows + # def get_label2rows(self): + # self.terms = self.fetch_terms() + # visited = {} + # label2rows = defaultdict(list) + # header = ['Index'] + list(self.terms.columns) + # for row in self.terms.itertuples(): + # row = {header[i]: val for i, val in enumerate(row)} + # label = row['label'].lower().strip() + # if not visited.get((label, row['type'], row['ilx'])): + # label2rows[label].append(row) + # visited[(label, row['type'], row['ilx'])] = True + # return label2rows + + def get_label2rows(self, clean: object = None) -> Dict[str, list]: + if not clean: + clean = lambda string: string.lower().strip() + label2ilx = defaultdict(list) + [label2ilx[clean(row.label)].append(row) for row in self.fetch_terms().itertuples()] + [label2ilx[clean(row.literal)].append(row) for row in self.fetch_synonyms().itertuples()] + return label2ilx + + def get_ilx2synonyms(self) -> defaultdict(list): + ilx2synonyms = defaultdict(list) + [ilx2synonyms[row.ilx].append(row.literal) for row in self.fetch_synonyms().itertuples()] + return ilx2synonyms def get_definition2rows(self): self.terms = self.fetch_terms() @@ -313,8 +328,8 @@ def get_definition2rows(self): definition2rows = defaultdict(list) header = ['Index'] + list(self.terms.columns) for row in self.terms.itertuples(): - row = {header[i]:val for i, val in enumerate(row)} - definition = self.local_degrade(row['definition']) + row = {header[i]: val for i, val in enumerate(row)} + definition = row['definition'].lower().strip() if not definition or definition == ' ': continue if not visited.get((definition, row['type'], row['ilx'])): @@ -372,6 +387,12 @@ def get_tid2annotations(self, clean:bool=True): tid2annotations[row['tid']].append(row) return tid2annotations + def get_ilx2relationships(self): + ilx2relationships = defaultdict(list) + header = ['Index'] + list(self.fetch_relationships().columns) + for row in self.fetch_relationships().itertuples(): + row = {header[i]:val for i, val in enumerate(row)} + def get_ilx2annotations(self, clean:bool=True): ''' clean: for list of literals only ''' ilx2annotations = defaultdict(list) diff --git a/ilxutils/ilxutils/remotes.py b/ilxutils/ilxutils/remotes.py index 76d99fd3..979ae58f 100644 --- a/ilxutils/ilxutils/remotes.py +++ b/ilxutils/ilxutils/remotes.py @@ -15,7 +15,7 @@ def remote(server=''): interlex_remote = InterLexRemote() # setup inheritance classes - interlex_remote.setup(instrumented=OntTerm) interlex_remote.apiEndpoint = endpoint + interlex_remote.setup(instrumented=OntTerm) return interlex_remote diff --git a/ilxutils/ilxutils/scicrunch_session.py b/ilxutils/ilxutils/scicrunch_session.py index a2030575..1c68a422 100644 --- a/ilxutils/ilxutils/scicrunch_session.py +++ b/ilxutils/ilxutils/scicrunch_session.py @@ -37,7 +37,7 @@ def __init__(self, self.key = key self.host = '' self.api = '' - + # Pull host for potential url if host.startswith('http'): host = urlparse(host).netloc diff --git a/ilxutils/ilxutils/sparql.ipynb b/ilxutils/ilxutils/sparql.ipynb new file mode 100644 index 00000000..fb11bab8 --- /dev/null +++ b/ilxutils/ilxutils/sparql.ipynb @@ -0,0 +1,296 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from pyontutils.core import OntResIri\n", + "g = OntResIri('https://cassava.ucsd.edu/sparc/exports/curation-export.ttl').graph" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "q2=\"\"\"\n", + "SELECT ?subj ?pred ?obj\n", + "WHERE {\n", + " TEMP:hasDerivedInformationAsParticipant ?obj .\n", + "?subj TEMP:hasDerivedInformationAsParticipant ?obj .\n", + "}\n", + "\"\"\"\n", + "\n", + "templates = SparqlQueryTemplates(g.namespace_manager)\n", + "q = templates.dataset_group(\n", + " subject='https://api.blackfynn.io/datasets/N:dataset:bc4071fd-aba1-4fe5-a59e-3da5affbc5fb/subjects/10653',\n", + ")\n", + "# print(q)\n", + "ts = []\n", + "sp = g.query(q2)" + ] + }, + { + "cell_type": "code", + "execution_count": 211, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 211, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from typing import Union, Dict, List, Tuple\n", + "import unittest\n", + "\n", + "import rdflib\n", + "from rdflib.plugins import sparql\n", + "import pytest\n", + "\n", + "from pyontutils.core import OntResIri\n", + "\n", + "Semantic = Union[rdflib.URIRef, rdflib.Literal, rdflib.BNode]\n", + "\n", + "\n", + "class TestCurationExportTtl:\n", + "\n", + " def __init__(self):\n", + " self.ori = OntResIri('https://cassava.ucsd.edu/sparc/exports/curation-export.ttl')\n", + " self.graph = self.ori.graph\n", + " self.spaql_templates = SparqlQueryTemplates(self.graph)\n", + "\n", + " def test_dataset_group(self):\n", + " \"\"\" sparql queries here \"\"\"\n", + " subj = rdflib.URIRef('https://api.blackfynn.io/datasets/N:dataset:c2a014b8-2c15-4269-b10a-3345420e3d56/subjects/53')\n", + " query = self.spaql_templates.dataset_group()\n", + " assert len(list(self.graph.query(query, initBindings={'target': subj}))) > 0\n", + "\n", + " def test_related_datasets(self):\n", + " subj = rdflib.util.from_n3('dataset:bec4d335-9377-4863-9017-ecd01170f354', nsm=self.graph)\n", + " query = self.spaql_templates.related_datasets()\n", + " assert len(list(self.graph.query(query, initBindings={'target': subj}))) > 0\n", + "\n", + "\n", + "class SparqlQueryTemplates:\n", + " \"\"\" Creates SPARQL query templates. \"\"\"\n", + "\n", + " def __init__(self, nsm=None):\n", + " self.nsm = nsm if nsm else rdflib.Graph().namespace_manager\n", + " self.prefixes = {p:ns for p, ns in self.nsm.namespaces() if p}\n", + "\n", + " def sparql_iri(self, iri: Union[rdflib.URIRef, str]) -> str:\n", + " \"\"\" Converts IRIs and curies to a usable format for SPARQL queries. \"\"\"\n", + " if iri.startswith('http') or isinstance(iri, rdflib.URIRef):\n", + " return '<'+str(iri)+'>'\n", + " return iri\n", + "\n", + " def dataset_group(self) -> str:\n", + " \"\"\" Get all subject groups and dataset associated with subject input.\n", + "\n", + " :returns: list of tuples containing: subject, subjects group, and subjects dataset.\n", + " \"\"\"\n", + " query = \"\"\"\n", + " SELECT ?subj ?group ?dataset\n", + " WHERE {\n", + " ?target TEMP:hasAssignedGroup ?group .\n", + " ?subj TEMP:hasAssignedGroup ?group .\n", + " ?subj TEMP:hasDerivedInformationAsParticipant ?dataset .\n", + " }\n", + " \"\"\"\n", + " return sparql.prepareQuery(query, initNs=self.prefixes)\n", + "\n", + " def related_datasets(self) -> str:\n", + " \"\"\" Get all related datasets of subject.\n", + "\n", + " :returns: list of tuples containing: subject & subjects shared dataset.\n", + " \"\"\"\n", + " query = \"\"\"\n", + " SELECT ?subj ?dataset\n", + " WHERE {\n", + " ?target TEMP:collectionTitle ?dataset .\n", + " ?subj TEMP:collectionTitle ?dataset .\n", + " }\n", + " \"\"\"\n", + " return sparql.prepareQuery(query, initNs=self.prefixes)\n", + " \n", + "TestCurationExportTtl().test_dataset_group()" + ] + }, + { + "cell_type": "code", + "execution_count": 205, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 205, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "t.test_dataset_group()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "rdflib.term.URIRef('http://www.w3.org/2000/01/rdf-schema#label')" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from rdflib import RDFS\n", + "from_n3('')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "module 'rdflib.plugins' has no attribute 'sparql'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrdflib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplugins\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparql\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepareQuery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m: module 'rdflib.plugins' has no attribute 'sparql'" + ] + } + ], + "source": [ + "rdflib.plugins.sparql.prepareQuery(query, initN)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from rdflib.plugins import sparql" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rdflib==5.0.0.dev0\n" + ] + } + ], + "source": [ + "!pip3 freeze | grep rdflib" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (, line 1)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m from rdflib.plugins\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], + "source": [ + "from rdflib.plugins.sp" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "module 'rdflib' has no attribute 'plugins'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrdflib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mrdflib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplugins\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparql\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepareQuery\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m: module 'rdflib' has no attribute 'plugins'" + ] + } + ], + "source": [ + "import rdflib\n", + "rdflib.plugins.sparql.prepareQuery" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from pyontutils.core import OntResIri\n", + "from rdflib.plugins.sparql import prepareQuery" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 8b874247beafdb1cb4629b52ccb7c32cdb9a2244 Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Wed, 21 Oct 2020 08:51:28 -0700 Subject: [PATCH 11/17] moved tutorial to docs --- ilxutils/{tutorials => docs}/interlex_remotes_tutorial.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename ilxutils/{tutorials => docs}/interlex_remotes_tutorial.ipynb (99%) diff --git a/ilxutils/tutorials/interlex_remotes_tutorial.ipynb b/ilxutils/docs/interlex_remotes_tutorial.ipynb similarity index 99% rename from ilxutils/tutorials/interlex_remotes_tutorial.ipynb rename to ilxutils/docs/interlex_remotes_tutorial.ipynb index 5a27e35a..82d7c9a0 100644 --- a/ilxutils/tutorials/interlex_remotes_tutorial.ipynb +++ b/ilxutils/docs/interlex_remotes_tutorial.ipynb @@ -697,7 +697,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.5" } }, "nbformat": 4, From a6c9c77471479a40447548a7da5d09f879467b83 Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Thu, 10 Dec 2020 05:34:55 -0800 Subject: [PATCH 12/17] added proper test port --- ilxutils/ilxutils/sql.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ilxutils/ilxutils/sql.py b/ilxutils/ilxutils/sql.py index b3470e74..a895a4db 100644 --- a/ilxutils/ilxutils/sql.py +++ b/ilxutils/ilxutils/sql.py @@ -4,13 +4,9 @@ def production_sql(from_backup=True): return IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_PRODUCTION'), from_backup=from_backup) -def beta_sql(from_backup=True): - # TEST{#} should be a thing since this still relies on main sql test - return IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_BETA'), from_backup=from_backup) - -# entities = [] -# for ilx, group in ex.groupby('ilx'): -# if not any(list(group['preferred'] == '1')): -# entities.append(ilx) - -# from ontquery.plugins.services.interlex_client import InterLexClient +def test_sql(db_url=None, schema=None, from_backup=True): + db_url = os.environ.get('SCICRUNCH_DB_URL_TEST') if not db_url else db_url + # Default test schema is nif_test3 + if schema: + db_url = '/'.join(db_url.split('/')[:-1] + [schema]) + return IlxSql(db_url=db_url, from_backup=from_backup) From 21de3e68d126395fdead5afe2add63788564fd7f Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Sun, 3 Jan 2021 11:28:29 -0800 Subject: [PATCH 13/17] pax-spine reference files --- nifstd/resources/pax-spine-2009.txt | 230 +++++++++++++++++++++++++ nifstd/resources/pax-spine-2013.txt | 256 ++++++++++++++++++++++++++++ 2 files changed, 486 insertions(+) create mode 100644 nifstd/resources/pax-spine-2009.txt create mode 100644 nifstd/resources/pax-spine-2013.txt diff --git a/nifstd/resources/pax-spine-2009.txt b/nifstd/resources/pax-spine-2009.txt new file mode 100644 index 00000000..7b06f4ca --- /dev/null +++ b/nifstd/resources/pax-spine-2009.txt @@ -0,0 +1,230 @@ +List of Structures +; # +; A +accessory nerve nucleus 11N +adductor motoneurons of lamina 9 Ad9 +anterior spinal artery asp +axial muscle motoneurons of lamina 9 Ax9 +; B +biceps motoneurons of lamina 9 Bi9 +; C +cauda equina CE +central canal CC +central cervical nucleus CeCv +coccygeal vertebra CoVert +cremaster motoneurons of lamina 9 Cr9 +crural extensor motoneurons of lamina 9 CEx9 +crural flexor motoneurons of lamina 9 CFl9 +cuneate fasciculus cu +; D +deltoid motoneurons of lamina 9 De9 +dorsal corticospinal tract dcs +dorsal funiculus df +dorsal nucleus (Clarke) D +dorsal root dr +dorsal root ganglion DRG +dorsolateral fasciculus (Lissauer) dl +; E +external anal sphincter motoneurons of lamina 9 ExA9 +external urethral sphincter motoneurons of lamina 9 ExU9 +; F +forearm extensor motoneurons of lamina 9 FEx9 +forearm flexor motoneurons of lamina 9 FFl9 +; G +Gl9 gracile fasciculus gr +gluteal motoneurons of lamina 9 Gl9 +; H +hamstring motoneurons of lamina 9 Hm9 +; I +iliopsoas motoneurons of lamina 9 IPs9 +infrahyoid muscle motoneurons of lamina 9 IH9 +intercalated nucleus ICl +intercalated nucleus, paraependymal part IClPe +intercostal muscle motoneurons of lamina 9 ICo9 +intermediolateral column IML +intermediomedial column IMM +internal basilar nucleus IB +; J +; K +; L +lamina 1 of the spinal gray 1Sp +lamina 10 of the spinal gray 10Sp +lamina 2 of the spinal gray 2Sp +lamina 2 of the spinal gray, inner part 2SpI +lamina 2 of the spinal gray, outer part 2SpO +lamina 3 of the spinal gray 3Sp +lamina 4 of the spinal gray 4Sp +lamina 5 of the spinal gray 5Sp +lamina 5 of the spinal gray, lateral part 5SpL +lamina 6 of the spinal gray 6Sp +lamina 6 of the spinal gray, lateral part 6SpL +lamina 6 of the spinal gray, medial part 6SpM +lamina 7 of the spinal gray 7Sp +lamina 8 of the spinal gray 8Sp +lamina 9 of the spinal gray 9Sp +lamina 5 of the spinal gray, medial part 5SpM +lateral cervical nucleus LatC +lateral funiculus lf +lateral spinal nucleus LSp +latissimus dorsi motoneurons of lamina 9 LD9 +levator scapulae motoneurons of lamina 9 LS9 +lumbar dorsal commissural nucleus LDCom +lumbar precerebellar nucleus LPrCb +; M +manus motoneurons of lamina 9 Man9 +; N +; O +; P +pectoral muscle motoneurons of lamina 9 Pec9 +pes motoneurons of lamina 9 Pes9 +phrenic motoneurons of lamina 9 Ph9 +postsynaptic dorsal column pathway psdc +psoas motoneurons of lamina 9 Ps9 +; Q +quadratus lumborum motoneurons of lamina 9 QL9 +quadriceps motoneurons of lamina 9 Q9 +; R +rhomboid muscle motoneurons of lamina 9 Rh9 +rubrospinal tract rs +; S +sacral dorsal commissural nucleus SDCom +sacral parasympathetic nucleus SPSy +sacral precerebellar nucleus SPrCb +scalene muscle motoneurons of lamina 9 Sc9 +serratus anterior motoneurons in lamina 9 Sr9 +spinal accessory nerve 11n +spinous process of vertebra SpPr +sternomastoid motoneurons of lamina 9 SM9 +supraspinatus and infraspinatus motoneurons of lamina 9 SI9 +; T +tail muscle motoneurons of lamina 9 Tail9 +thoracoabdominal wall muscle motoneurons of lamina 9 ThAb9 +trapezius and sternomastoid motoneurons of lamina 9 TzSM9 +trapezius motoneurons of lamina 9 Tz9 +triceps motoneurons of lamina 9 Tr9 +; U +; V +ventral funiculus vf +ventral median fissure vmf +ventral root vr +ventral white commissure vwc +vertebral arch VAr +vertebral body VB +; W +; X +; Y +; Z + +List of Abbreviations +; # +10Sp lamina 10 of the spinal gray +11N accessory nerve nucleus +11n spinal accessory nerve +1Sp lamina 1 of the spinal gray +2Sp lamina 2 of the spinal gray +2SpI lamina 2 of the spinal gray, inner part +2SpO lamina 2 of the spinal gray, outer part +3Sp lamina 3 of the spinal gray +4Sp lamina 4 of the spinal gray +5Sp lamina 5 of the spinal gray +5SpL lamina 5 of the spinal gray, lateral part +5SpM lamina 5 of the spinal gray, medial part +6Sp lamina 6 of the spinal gray +6SpL lamina 6 of the spinal gray, lateral part +6SpM lamina 6 of the spinal gray, medial part +7Sp lamina 7 of the spinal gray +8Sp lamina 8 of the spinal gray +9Sp lamina 9 of the spinal gray +; A +Ad9 adductor motoneurons of lamina 9 +Ax9 axial muscle motoneurons of lamina 9 +asp anterior spinal artery +; B +Bi9 biceps motoneurons of lamina 9 +; C +CE cauda equina +CEx9 crural extensor motoneurons of lamina 9 +CFl9 crural flexor motoneurons of lamina 9 +CeCv central cervical nucleus +CoVert coccygeal vertebra +Cr9 cremaster motoneurons of lamina 9 +cu cuneate fasciculus +; D +D dorsal nucleus +DRG dorsal root ganglion +De9 deltoid motoneurons of lamina 9 +dcs dorsal corticospinal tract +df dorsal funiculus +dl dorsolateral fasciculus +dr dorsal root +; E +ExA9 external anal sphincter motoneurons of lamina 9 +ExU9 external urethral sphincter motoneurons of lamina 9 +; F +FEx9 forearm extensor motoneurons of lamina 9 +FFl9 forearm flexor motoneurons of lamina 9 +; G +Gl9 gluteal motoneurons of lamina 9 +gr gracile fasciculus +; H +Hm9 hamstring motoneurons of lamina 9 +; I +IB internal basilar nucleus +ICl intercalated nucleus +IClPe intercalated nucleus, paraependymal part +ICo9 intercostal muscle motoneurons of lamina 9 +IH9 infrahyoid muscle motoneurons of lamina 9 +IML intermediolateral column +IMM intermediomedial column +; J +; K +; L +LD9 latissimus dorsi motoneurons of lamina 9 +LDCom lumbar dorsal commissural nucleus +LPrCb lumbar precerebellar nucleus +LS9 levator scapulae motoneurons of lamina 9 +LSp lateral spinal nucleus +LatC lateral cervical nucleus +lf lateral funiculus +; M +Man9 manus motoneurons of lamina 9 +; N +; O +; P +Pec9 pectoral muscle motoneurons of lamina 9 +Pes9 pes motoneurons of lamina 9 +Ph9 phrenic motoneurons of lamina 9 +Ps9 psoas motoneurons of lamina 9 +psdc postsynaptic dorsal column pathway +; Q +Q9 quadriceps motoneurons of lamina 9 +QL9 quadratus lumborum motoneurons of lamina 9 +; R +Rh9 rhomboid muscle motoneurons of lamina 9 +rs rubrospinal tract +; S +SDCom sacral dorsal commissural nucleus +SI9 supraspinatus and infraspinatus motoneurons of lamina 9 +SM9 sternomastoid motoneurons of lamina 9 +SPSy sacral parasympathetic nucleus +SPrCb sacral precerebellar nucleus +Sc9 scalene muscle motoneurons of lamina 9 +SpPr spinous process of vertebra +Sr9 serratus anterior motoneurons in lamina 9 +; T +Tail9 tail muscle motoneurons of lamina 9 +ThAb9 thoracoabdominal wall muscle motoneurons of lamina 9 +Tr9 triceps motoneurons of lamina 9 +TzSM9 trapezius and sternomastoid motoneurons of lamina 9 +; U +; V +VAr vertebral arch +VB vertebral body +vf ventral funiculus +vmf ventral median fissure +vr ventral root +vwc ventral white commissure +; W +; X +; Y +; Z diff --git a/nifstd/resources/pax-spine-2013.txt b/nifstd/resources/pax-spine-2013.txt new file mode 100644 index 00000000..6143055d --- /dev/null +++ b/nifstd/resources/pax-spine-2013.txt @@ -0,0 +1,256 @@ +List of Structures +; # +; A +adductor motoneurons of lamina 9 Ad9 +anterior corticospinal tract acs +anterior spinal artery asp +area 10 of the spinal gray 10Sp +axial muscle motoneurons of lamina 9 Ax9 +; B +biceps motoneurons of lamina 9 Bi9 +; C +cauda equina CE +caudal reticulospinal tract crsp +central canal CC +central cervical nucleus CeCv +coccygeal vertebra CoVert +cremaster motoneurons of lamina 9 Cr9 +crural extensor motoneurons of lamina 9 CEx9 +crural flexor motoneurons of lamina 9 CF19 +cuneate fasciculus cu +; D +deltoid motoneurons of lamina 9 De9 +dorsal corticospinal tract dcs +dorsal funiculus df +dorsal nucleus (clarke) D +dorsal root dr +dorsal root dr +dorsal root ganglion DRG +dorsal spinocerebellar tract dsc +dorsolateral fasciculus (lissauer) dl +dorsolateral spinothalamic tract dlst +; E +external anal sphincter motoneurons of lamina 9 ExA9 +external urethral sphincter motoneurons of lamina 9 ExU9 +; F +forearm extensor motoneurons of lamina 9 FEx9 +forearm flexor motoneurons of lamina 9 FFl9 +; G +gluteal motoneurons of lamina 9 Gl9 +gracile fasciculus gr +; H +hamstring motoneurons of lamina 9 Hm9 +; I +iliopsoas motoneurons of lamina 9 IPs9 +infrahyoid muscle motoneurons of lamina 9 IH9 +intercalated nucleus ICl +intercalated nucleus, paraependymal part ICIPe +intercostal muscle motoneurons of lamina 9 ICo9 +intermediolateral column IML +intermediolateral column, funicular part IMFL +intermediomedial column IMM +internal basilar nucleus IB +; J +; K +; L +lamina 1 of the spinal gray 1Sp +lamina 2 of the spinal gray 2Sp +lamina 2 of the spinal gray, inner part 2SpI +lamina 2 of the spinal gray, outer part 2SpO +lamina 3 of the spinal gray 3Sp +lamina 4 of the spinal gray 4Sp +lamina 5 of the spinal gray 5Sp +lamina 5 of the spinal gray, lateral part 5SpL +lamina 5 of the spinal gray, medial part 5SpM +lamina 6 of the spinal gray 6Sp +lamina 6 of the spinal gray, lateral part 6SpL +lamina 6 of the spinal gray, medial part 6SpM +lamina 7 of the spinal gray 7Sp +lamina 8 of the spinal gray 8Sp +lamina 9 of the spinal gray 9Sp +lateral cervical nucleus LatC +lateral corticospinal tract lcs +lateral funiculus lf +lateral spinal nucleus LSp +lateral spinothalamic tract lst +lateral vestibulospinal tract lvs +latissimus dorsi motoneurons of lamina 9 LD9 +levator scapulae motoneurons of lamina 9 LS9 +lumbar dorsal commissural nucleus LDCom +lumbar precerebellar nucleus LPrCb +; M +manus motoneurons of lamina 9 Man9 +medial vestibulospinal tract mvs +; N +; O +; P +pectoral muscle motoneurons of lamina 9 Pec9 +pes motoneurons of lamina 9 Pes9 +phrenic motoneurons of lamina 9 Ph9 +postsynaptic dorsal column pathway psdc +psoas muscle motoneurons of lamina 9 Ps9 +; Q +quadratus lumborum motoneurons of lamina 9 QL9 +quadriceps motoneurons of lamina 9 Q9 +; R +rhomboid muscle motoneurons of lamina 9 Rh9 +rostral reticulospinal tract rrsp +rubrospinal tract rs +; S +sacral dorsal commissural nucleus SDCom +sacral parasympathetic nucleus SPSy +sacral precerebellar nucleus SPrCb +scalene muscle motoneurons of lamina 9 Sc9 +serratus anterior motoneurons in lamina 9 Sr9 +spinal accessory nerve 11n +spinous process of vertebra SpPr +sternomastoid motoneurons of lamina 9 SM9 +supraspinatus and infraspinatus motoneurons of lamina 9 SI9 +; T +tail muscle motoneurons of lamina 9 Tail9 +thoracoabdominal wall muscle motoneurons of lamina 9 ThAb9 +trapezius and sternomastoid motoneurons of lamina 9 TzSM9 +trapezius motoneurons of lamina 9 Tz9 +triceps motoneurons of lamina 9 Tr9 +; U +; V +ventral funiculus vf +ventral median fissure vmf +ventral root vr +ventral spinocerebellar tract vsc +ventral spinothalamic tract vst +ventral white commissure vwc +vertebral arch VAr +vertebral body VB +; W +; X +; Y +; Z + +List of Abbreviations +; # +; A +adductor motoneurons of lamina 9 Ad9 +anterior corticospinal tract acs +anterior spinal artery asp +area 10 of the spinal gray 10Sp +axial muscle motoneurons of lamina 9 Ax9 +; B +biceps motoneurons of lamina 9 Bi9 +; C +cauda equina CE +caudal reticulospinal tract crsp +central canal CC +central cervical nucleus CeCv +coccygeal vertebra CoVert +cremaster motoneurons of lamina 9 Cr9 +crural extensor motoneurons of lamina 9 CEx9 +crural flexor motoneurons of lamina 9 CF19 +cuneate fasciculus cu +; D +deltoid motoneurons of lamina 9 De9 +dorsal corticospinal tract dcs +dorsal funiculus df +dorsal nucleus (clarke) D +dorsal root dr +dorsal root ganglion DRG +dorsal spinocerebellar tract dsc +dorsolateral fasciculus (lissauer) dl +dorsolateral spinothalamic tract dlst +; E +external anal sphincter motoneurons of lamina 9 ExA9 +external urethral sphincter motoneurons of lamina 9 ExU9 +; F +forearm extensor motoneurons of lamina 9 FEx9 +forearm flexor motoneurons of lamina 9 FFl9 +; G +gluteal motoneurons of lamina 9 Gl9 +gracile fasciculus gr +; H +hamstring motoneurons of lamina 9 Hm9 +; I +iliopsoas motoneurons of lamina 9 IPs9 +infrahyoid muscle motoneurons of lamina 9 IH9 +intercalated nucleus ICl +intercalated nucleus, paraependymal part ICIPe +intercostal muscle motoneurons of lamina 9 ICo9 +intermediolateral column IML +intermediolateral column, funicular part IMFL +intermediomedial column IMM +internal basilar nucleus IB +; J +; K +; L +lamina 1 of the spinal gray 1Sp +lamina 2 of the spinal gray 2Sp +lamina 2 of the spinal gray, inner part 2SpI +lamina 2 of the spinal gray, outer part 2SpO +lamina 3 of the spinal gray 3Sp +lamina 4 of the spinal gray 4Sp +lamina 5 of the spinal gray 5Sp +lamina 5 of the spinal gray, lateral part 5SpL +lamina 5 of the spinal gray, medial part 5SpM +lamina 6 of the spinal gray 6Sp +lamina 6 of the spinal gray, lateral part 6SpL +lamina 6 of the spinal gray, medial part 6SpM +lamina 7 of the spinal gray 7Sp +lamina 8 of the spinal gray 8Sp +lamina 9 of the spinal gray 9Sp +lateral cervical nucleus LatC +lateral corticospinal tract lcs +lateral funiculus lf +lateral spinal nucleus LSp +lateral spinothalamic tract lst +lateral vestibulospinal tract lvs +latissimus dorsi motoneurons of lamina 9 LD9 +levator scapulae motoneurons of lamina 9 LS9 +lumbar dorsal commissural nucleus LDCom +lumbar precerebellar nucleus LPrCb +; M +manus motoneurons of lamina 9 Man9 +medial vestibulospinal tract mvs +; N +; O +; P +pectoral muscle motoneurons of lamina 9 Pec9 +pes motoneurons of lamina 9 Pes9 +phrenic motoneurons of lamina 9 Ph9 +postsynaptic dorsal column pathway psdc +psoas muscle motoneurons of lamina 9 Ps9 +; Q +quadratus lumborum motoneurons of lamina 9 QL9 +quadriceps motoneurons of lamina 9 Q9 +; R +rhomboid muscle motoneurons of lamina 9 Rh9 +rostral reticulospinal tract rrsp +rubrospinal tract rs +; S +sacral dorsal commissural nucleus SDCom +sacral parasympathetic nucleus SPSy +sacral precerebellar nucleus SPrCb +scalene muscle motoneurons of lamina 9 Sc9 +serratus anterior motoneurons in lamina 9 Sr9 +spinal accessory nerve 11n +spinous process of vertebra SpPr +sternomastoid motoneurons of lamina 9 SM9 +supraspinatus and infraspinatus motoneurons of lamina 9 SI9 +; T +tail muscle motoneurons of lamina 9 Tail9 +thoracoabdominal wall muscle motoneurons of lamina 9 ThAb9 +trapezius and sternomastoid motoneurons of lamina 9 TzSM9 +trapezius motoneurons of lamina 9 Tz9 +triceps motoneurons of lamina 9 Tr9 +; U +; V +ventral funiculus vf +ventral median fissure vmf +ventral root vr +ventral spinocerebellar tract vsc +ventral spinothalamic tract vst +ventral white commissure vwc +vertebral arch VAr +vertebral body VB +; W +; X +; Y +; Z From abf65460d3df0335da0c7f9c938cb01173a16430 Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Tue, 25 May 2021 19:11:12 -0700 Subject: [PATCH 14/17] ilxutils quality of life updates --- ilxutils/ilxutils/elasticsearch_wrapper.py | 2 +- ilxutils/ilxutils/interlex_sql.py | 300 ++++++++----------- ilxutils/ilxutils/tools.py | 321 +++++++++++++++++++++ ilxutils/setup.py | 3 + 4 files changed, 452 insertions(+), 174 deletions(-) create mode 100644 ilxutils/ilxutils/tools.py diff --git a/ilxutils/ilxutils/elasticsearch_wrapper.py b/ilxutils/ilxutils/elasticsearch_wrapper.py index e9767e0e..d203f50e 100644 --- a/ilxutils/ilxutils/elasticsearch_wrapper.py +++ b/ilxutils/ilxutils/elasticsearch_wrapper.py @@ -79,7 +79,7 @@ class InterLexES(ElasticSearchTools): def __init__(self, beta=True): super().__init__( host = BASHRC('SCICRUNCH_ELASTIC_URL'), - # index = 'interlex_2019oct28', + # index = 'Interlex_old', index = 'interlex', type = 'term', user = BASHRC('INTERLEX_ELASTIC_USER'), diff --git a/ilxutils/ilxutils/interlex_sql.py b/ilxutils/ilxutils/interlex_sql.py index d1badb67..a9283d00 100755 --- a/ilxutils/ilxutils/interlex_sql.py +++ b/ilxutils/ilxutils/interlex_sql.py @@ -1,84 +1,33 @@ -from collections import defaultdict -import os from pathlib import Path -from typing import Union, Dict, Tuple, List - import pandas as pd from sqlalchemy import create_engine, inspect, Table, Column - -from ilxutils.tools import light_degrade, open_pickle, create_pickle - - -TERMSC_BACKUP = 'ilx_db_terms_complete_backup.pickle' -TERMS_BACKUP = 'ilx_db_terms_backup.pickle' -ANNOS_BACKUP = 'ilx_db_annotations_backup.pickle' -RELAS_BACKUP = 'ilx_db_relationships_backup.pickle' -SUPER_BACKUP = 'ilx_db_superclasses_backup.pickle' -SYNOS_BACKUP = 'ilx_db_synonyms_backup.pickle' -EXIDS_BACKUP = 'ilx_db_existing_ids_backup.pickle' - - -def pathing(path, check_path=False): - """ Extract absolute path from shortened or relative paths. - - :param str path: path to file or folder. - - :examples: - >>>pathing('~/shortened.filepath') - >>>pathing('../relative.filepath') - >>>pathing('relative.filepath') - >>>pathing('/home/absoulte/filepath') - """ - path = Path(path) - if str(path).startswith('~'): - path = path.expanduser() - else: - path = path.resolve() - - if check_path: - if not path.is_file() and path.is_dir(): - raise ValueError(f'{path} does not exit') - - return path - - -class IlxSql: - - def __init__(self, - db_url: str, - from_backup: bool = False, - pre_load: bool = False, - backups_folder: str = '~/.interlex_backups'): - - self.engine = create_engine(db_url) +from collections import defaultdict +from .tools import light_degrade, open_pickle, create_pickle +import os +#ELASTIC = 'https://5f86098ac2b28a982cebf64e82db4ea2.us-west-2.aws.found.io:9243/interlex/term/' +TERMS_COMPLETE_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_terms_complete_backup.pickle' +TERMS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_terms_backup.pickle' +ANNOS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_annotations_backup.pickle' +RELAS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_relationships_backup.pickle' +SUPER_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_superclasses_backup.pickle' +SYNOS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_synonyms_backup.pickle' +EXIDS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_ex_backup.pickle' + + +class IlxSql(): + + def __init__(self, db_url, pre_load=False, from_backup=False): + self.db_url = db_url + self.engine = create_engine(self.db_url) + self.local_degrade = lambda string: string.lower().strip() # current degrade of choice for sql self.from_backup = from_backup - - self.save_folder = pathing(backups_folder) - try: - self.save_folder.mkdir() - except: - pass - - self.terms = pd.DataFrame - self.superclasses = pd.DataFrame - self.annotations = pd.DataFrame - self.existing_ids = pd.DataFrame - self.relationships = pd.DataFrame - self.synonyms = pd.DataFrame - - # Auto load tables from backup - if pre_load: - self.pre_loader() - - - def pre_loader(self): - # self.terms_complete = self.get_terms_complete() if pre_load else pd.DataFrame - self.terms = self.get_terms() - # self.superclasses = self.get_superclasses() - self.existing_ids = self.get_existing_ids() - self.synonyms = self.get_synonyms() - # self.relationships = self.get_relationships() - # self.annotations = self.get_annotations() + self.terms_complete = self.get_terms_complete() if pre_load else pd.DataFrame + self.terms = self.get_terms() if pre_load else pd.DataFrame + self.superclasses = self.get_superclasses if pre_load else pd.DataFrame + self.annotations = self.get_annotations() if pre_load else pd.DataFrame + self.existing_ids = self.get_existing_ids() if pre_load else pd.DataFrame + self.relationships = self.get_relationships() if pre_load else pd.DataFrame + self.synonyms = self.get_synonyms() if pre_load else pd.DataFrame def fetch_terms_complete(self): if self.terms_complete.empty: @@ -117,104 +66,113 @@ def fetch_superclasses(self): def get_terms(self): ''' GROUP BY is a shortcut to only getting the first in every list of group ''' + if not self.terms.empty: + return self.terms if self.from_backup: - self.terms = open_pickle(self.save_folder / TERMS_BACKUP) + self.terms = open_pickle(TERMS_BACKUP_PATH) return self.terms - sql_query = """ - SELECT t.id as tid, t.ilx, t.label, t.definition, t.type, t.comment, t.version, t.uid, t.cid, t.time, t.status + engine = create_engine(self.db_url) + data = """ + SELECT t.id as tid, t.ilx, t.label, t.definition, t.type, t.comment, t.version, t.uid, t.time, t.status FROM terms t GROUP BY t.ilx - HAVING t.status = '0' """ - self.terms = pd.read_sql(sql_query, self.engine) - create_pickle(self.terms, self.save_folder / TERMS_BACKUP) + self.terms = pd.read_sql(data, engine) + create_pickle(self.terms, TERMS_BACKUP_PATH) return self.terms def get_annotations(self): + if not self.annotations: + return self.fetch_annotations() if self.from_backup: - self.annotations = open_pickle(self.save_folder / ANNOS_BACKUP) + self.annotations = open_pickle(ANNOS_BACKUP_PATH) return self.annotations - sql_query = """ + engine = create_engine(self.db_url) + data = """ SELECT ta.tid, ta.annotation_tid as annotation_type_tid, t1.ilx as term_ilx, t2.ilx as annotation_type_ilx, t2.label as annotation_type_label, - ta.value + ta.value, ta.withdrawn FROM term_annotations AS ta JOIN ( SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) AS t1 ON ta.tid=t1.id JOIN ( SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) AS t2 ON ta.annotation_tid=t2.id """ - self.annotations = pd.read_sql(sql_query, self.engine) - create_pickle(self.annotations, self.save_folder / ANNOS_BACKUP) + self.annotations = pd.read_sql(data, engine) + create_pickle(self.annotations, ANNOS_BACKUP_PATH) return self.annotations def get_existing_ids(self): + if not self.existing_ids.empty: + return self.existing_ids if self.from_backup: - self.existing_ids = open_pickle(self.save_folder / EXIDS_BACKUP) + self.existing_ids = open_pickle(EXIDS_BACKUP_PATH) return self.existing_ids - sql_query = """ + engine = create_engine(self.db_url) + data = """ SELECT tei.tid, tei.curie, tei.iri, tei.preferred, t.ilx, t.label, t.definition, t.status FROM ( SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) as t JOIN term_existing_ids AS tei ON t.id = tei.tid """ - self.existing_ids = pd.read_sql(sql_query, self.engine) - create_pickle(self.existing_ids, self.save_folder / EXIDS_BACKUP) + self.existing_ids = pd.read_sql(data, engine) + create_pickle(self.existing_ids, EXIDS_BACKUP_PATH) return self.existing_ids def get_relationships(self): + if not self.relationships.empty: + return self.relationships if self.from_backup: - self.relationships = open_pickle(self.save_folder / RELAS_BACKUP) + self.relationships = open_pickle(RELAS_BACKUP_PATH) return self.relationships - sql_query = """ + engine = create_engine(self.db_url) + data = """ SELECT - t1.id as term1_tid, t1.ilx AS term1_ilx, t1.type as term1_type, t1.label as term1_label, - t2.id as term2_tid, t2.ilx AS term2_ilx, t2.type as term2_type, t2.label as term2_label, - t3.id as relationship_tid, t3.ilx AS relationship_ilx, t3.label as relationship_label + t1.id as term1_tid, t1.ilx AS term1_ilx, t1.type as term1_type, + t2.id as term2_tid, t2.ilx AS term2_ilx, t2.type as term2_type, + t3.id as relationship_tid, t3.ilx AS relationship_ilx, t3.label as relationship_label, + tr.withdrawn FROM term_relationships AS tr JOIN ( SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) t1 ON t1.id = tr.term1_id JOIN ( SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) AS t2 ON t2.id = tr.term2_id JOIN ( SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) AS t3 ON t3.id = tr.relationship_tid """ - self.relationships = pd.read_sql(sql_query, self.engine) - create_pickle(self.relationships, self.save_folder / RELAS_BACKUP) + self.relationships = pd.read_sql(data, engine) + create_pickle(self.relationships, RELAS_BACKUP_PATH) return self.relationships def get_superclasses(self): + if not self.superclasses.empty: + return self.superclasses if self.from_backup: - self.superclasses = open_pickle(self.save_folder / SUPER_BACKUP) + self.superclasses = open_pickle(SUPER_BACKUP_PATH) return self.superclasses - sql_query = """ + engine = create_engine(self.db_url) + data = """ SELECT ts.tid, ts.superclass_tid, t1.label as term_label, t1.ilx as term_ilx, @@ -224,49 +182,50 @@ def get_superclasses(self): SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) as t1 ON t1.id = ts.tid JOIN ( SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) AS t2 ON t2.id = ts.superclass_tid """ - self.superclasses = pd.read_sql(sql_query, self.engine) - create_pickle(self.superclasses, self.save_folder / SUPER_BACKUP) + self.superclasses = pd.read_sql(data, engine) + create_pickle(self.superclasses, SUPER_BACKUP_PATH) return self.superclasses def get_synonyms(self): + if not self.synonyms.empty: + return self.synonyms if self.from_backup: - self.synonyms = open_pickle(self.save_folder / SYNOS_BACKUP) + self.synonyms = open_pickle(SYNOS_BACKUP_PATH) return self.synonyms - sql_query = """ + engine = create_engine(self.db_url) + data = """ SELECT ts.tid as tid, t.ilx, ts.literal, ts.type FROM term_synonyms AS ts JOIN ( SELECT * FROM terms GROUP BY terms.ilx - HAVING terms.status = '0' ) AS t WHERE ts.tid=t.id """ - self.synonyms = pd.read_sql(sql_query, self.engine) - create_pickle(self.synonyms, self.save_folder / SYNOS_BACKUP) + self.synonyms = pd.read_sql(data, engine) + create_pickle(self.synonyms, SYNOS_BACKUP_PATH) return self.synonyms def get_terms_complete(self) -> pd.DataFrame: ''' Gets complete entity data like term/view ''' + if not self.terms_complete.empty: + return self.terms_complete if self.from_backup: - self.terms_complete = open_pickle(self.save_folder / TERMSC_BACKUP) + self.terms_complete = open_pickle(TERMS_COMPLETE_BACKUP_PATH) return self.terms_complete ilx2synonyms = self.get_ilx2synonyms() ilx2existing_ids = self.get_ilx2existing_ids() ilx2annotations = self.get_ilx2annotations() - ilx2relationships = self.get_ilx2relationships() ilx2superclass = self.get_ilx2superclass() ilx_complete = [] header = ['Index'] + list(self.fetch_terms().columns) @@ -278,49 +237,50 @@ def get_terms_complete(self) -> pd.DataFrame: row['superclass'] = ilx2superclass.get(row['ilx']) ilx_complete.append(row) terms_complete = pd.DataFrame(ilx_complete) - create_pickle(terms_complete, self.save_folder / TERMSC_BACKUP) + create_pickle(terms_complete, TERMS_COMPLETE_BACKUP_PATH) return terms_complete - def get_label2id(self, clean: object = None) -> Dict[str, list]: - if not clean: - clean = lambda string: string.lower().strip() - label2id = defaultdict(list) - [label2id[clean(row.label)].append(row.id) for row in self.fetch_terms().itertuples()] - return label2id - - def get_label2ilx(self, clean: object = None) -> Dict[str, list]: - if not clean: - clean = lambda string: string.lower().strip() - label2ilx = defaultdict(list) - [label2ilx[clean(row.label)].append(row.ilx) for row in self.fetch_terms().itertuples()] - [label2ilx[clean(row.literal)].append(row.ilx) for row in self.fetch_synonyms().itertuples()] - return label2ilx - - # def get_label2rows(self): - # self.terms = self.fetch_terms() - # visited = {} - # label2rows = defaultdict(list) - # header = ['Index'] + list(self.terms.columns) - # for row in self.terms.itertuples(): - # row = {header[i]: val for i, val in enumerate(row)} - # label = row['label'].lower().strip() - # if not visited.get((label, row['type'], row['ilx'])): - # label2rows[label].append(row) - # visited[(label, row['type'], row['ilx'])] = True - # return label2rows - - def get_label2rows(self, clean: object = None) -> Dict[str, list]: - if not clean: - clean = lambda string: string.lower().strip() - label2ilx = defaultdict(list) - [label2ilx[clean(row.label)].append(row) for row in self.fetch_terms().itertuples()] - [label2ilx[clean(row.literal)].append(row) for row in self.fetch_synonyms().itertuples()] - return label2ilx - - def get_ilx2synonyms(self) -> defaultdict(list): - ilx2synonyms = defaultdict(list) - [ilx2synonyms[row.ilx].append(row.literal) for row in self.fetch_synonyms().itertuples()] - return ilx2synonyms + def get_label2id(self): + self.terms = self.fetch_terms() + visited = {} + label_to_id = defaultdict(lambda: defaultdict(list)) + for row in self.terms.itertuples(): + label = self.local_degrade(row.label) + if not visited.get((label, row.type, row.ilx)): + if row.type == 'term': + label_to_id[label]['term'].append(int(row.id)) + visited[(label, row.type, row.ilx)] = True + elif row.type == 'cde': + label_to_id[label]['cde'].append(int(row.id)) + visited[(label, row.type, row.ilx)] = True + elif row.type == 'fde': + label_to_id[label]['fde'].append(int(row.id)) + visited[(label, row.type, row.ilx)] = True + return label_to_id + + def get_label2ilxs(self): + self.terms = self.fetch_terms() + visited = {} + label_to_ilx = defaultdict(list) + for row in self.terms.itertuples(): + label = self.local_degrade(row.label) + if not visited.get((label, row.type, row.ilx)): + label_to_ilx[label].append(str(row.ilx)) + visited[(label, row.type, row.ilx)] = True + return label_to_ilx + + def get_label2rows(self): + self.terms_complete = self.fetch_terms_complete() + visited = {} + label2rows = defaultdict(list) + header = ['Index'] + list(self.terms_complete.columns) + for row in self.terms_complete.itertuples(): + row = {header[i]:val for i, val in enumerate(row)} + label = self.local_degrade(row['label']) + if not visited.get((label, row['type'], row['ilx'])): + label2rows[label].append(row) + visited[(label, row['type'], row['ilx'])] = True + return label2rows def get_definition2rows(self): self.terms = self.fetch_terms() @@ -328,8 +288,8 @@ def get_definition2rows(self): definition2rows = defaultdict(list) header = ['Index'] + list(self.terms.columns) for row in self.terms.itertuples(): - row = {header[i]: val for i, val in enumerate(row)} - definition = row['definition'].lower().strip() + row = {header[i]:val for i, val in enumerate(row)} + definition = self.local_degrade(row['definition']) if not definition or definition == ' ': continue if not visited.get((definition, row['type'], row['ilx'])): @@ -387,12 +347,6 @@ def get_tid2annotations(self, clean:bool=True): tid2annotations[row['tid']].append(row) return tid2annotations - def get_ilx2relationships(self): - ilx2relationships = defaultdict(list) - header = ['Index'] + list(self.fetch_relationships().columns) - for row in self.fetch_relationships().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} - def get_ilx2annotations(self, clean:bool=True): ''' clean: for list of literals only ''' ilx2annotations = defaultdict(list) @@ -490,19 +444,19 @@ def get_fragment2rows(self): return fragement2rows def show_tables(self): - sql_query = "SHOW tables;" - return pd.read_sql(sql_query, self.engine) + data = "SHOW tables;" + return pd.read_sql(data, self.engine) def get_table(self, tablename, limit=5): - sql_query = """ + data = """ SELECT * FROM {tablename} LIMIT {limit} """.format(tablename=tablename, limit=limit) - return pd.read_sql(sql_query, self.engine) + return pd.read_sql(data, self.engine) - def get_custom(self, sql_query): - return pd.read_sql(sql_query, self.engine) + def get_custom(self, data): + return pd.read_sql(data, self.engine) def main(): diff --git a/ilxutils/ilxutils/tools.py b/ilxutils/ilxutils/tools.py new file mode 100644 index 00000000..d92290e5 --- /dev/null +++ b/ilxutils/ilxutils/tools.py @@ -0,0 +1,321 @@ +import json +import pickle +import re +import pandas as pd +from pathlib import Path as p +import pprint +from subprocess import call +from sys import exit +import csv +from typing import Union, Dict, List +import networkx as nx + +def sort_list_of_tuples_by_string(list_of_tuples:List[tuple], string_index:int) -> List[tuple]: + return sorted(list_of_tuples, key=lambda x: (str(x[string_index]).strip() in ['None', ''], x[string_index].lower())) + +def class_hierarchy(dag:List[tuple], descending=True) -> list: + ''' Topological Sorting + Args: + dag: directed acyclic graph that is a mappings of list of tuples with len of 2 + Returns: + Ordered list of single entities in topological ordering of choice + Examples: + >>> class_hierarchy([(1, 2), (2, 3)]) + [3, 2, 1] + ''' + dag = nx.DiGraph(dag) + dag = list(nx.topological_sort(dag)) + if descending: + dag = list(reversed(dag)) + return dag + +def clean(self, string, clean_scale:int=0): + if clean_scale == 0: + return string + elif clean_scale == 1: + return string.lower().strip() + elif clean_scale == 2: + return ' '.join(string_profiler(string)) + elif clean_scale == 3: + return ' '.join(string_profiler(string)).replace('obsolete', '').strip() + + +def string_profiler( + string: str, + start_delimiter: str='(', + end_delimiter: str=')', + remove: bool=True, + keep_delimiter: bool = True, + ) -> List[str]: + ''' + Seperates strings fragements into list based on the start and end delimiters + Args: + string: complete string you want to be broken up based on start and stop delimiters given + start_delimiter: delimiter element to start + end_delimiter: delimiter elemtent to end + remove: decide whether or not to keep strings inside the delimiters + Returns: + List[str]: list of strings that are split at start and end delimiters given and whether + or not you want to remove the string inside the delimiters + Tests: + long = '(life is is good) love world "(blah) blah" "here I am" once again "yes" blah ' + print(string_profiler(long)) + null = '' + print(string_profiler(null)) + short = '(life love) yes(and much more)' + print(string_profiler(short)) + short = 'yes "life love"' + print(string_profiler(short)) + ''' + outer_index = 0 # stepper for outer delimier string elements + inner_index = 0 # stepper for inner delimier string elements + curr_index = 0 # actual index of the current element in the string + string_list = [] # string broken up into individual elements whenever a start and end delimiter is hit + outer_string = '' # string tracked while outside the delimiters + inner_string = '' # string tracked while inside the delimiters + + for outer_index in range(len(string)): + # Actual pointer position (inner delimiter counter + outer delimiter counter) + curr_index = inner_index + outer_index + # Close once acutal index is at the end + # NOTE: outer_index will keep going till end regardless of hitting a delimiter and adding to inner stepper. + if curr_index == len(string): break + ### DELIMITER HIT ### + if string[curr_index] == start_delimiter: + # If we his a delimiter, collect the string previous to that as an element; flush + if outer_string: + # Option: .extend(outer_string.strip().split()) | If you want every word seperate. Maybe an option? + string_list.append(outer_string.strip()) + outer_string = '' + for j in range(curr_index+1, len(string)): + # Stepper that is pushed while in inner delimiter string. + inner_index += 1 + # Once we his the end delimiter, stop iterating through the inner delimiter string + if string[j] == end_delimiter: break + # String inside delimiters + inner_string += string[j] + # If you want the string inside the delimiters + if not remove: + if keep_delimiter: + inner_string = start_delimiter + inner_string + end_delimiter + string_list.append(inner_string) + # inner delimiter string restart + inner_string = '' + # String outside of the delimiters + else: outer_string += string[curr_index] + # End delimiter is either nested or not the real target; should ignore + if string[curr_index] == end_delimiter: + if string_list and outer_string: + string_list[-1] += outer_string + outer_string = '' + # In case of not hiting a delimiter at the end of the string, collect the remaining outer delimiter string + # Option: .extend(outer_string.strip().split()) | If you want every word seperate. Maybe an option? + if outer_string: string_list.append(outer_string.strip()) + return string_list + + +pp = pprint.PrettyPrinter(indent=4).pprint + + +class SetEncoder(json.JSONEncoder): + ''' Custom encoder to allow json to convert any sets in nested data to become lists ''' + def default(self, obj): + if isinstance(obj, set): + return sorted(list(obj)) + return json.JSONEncoder.default(self, obj) + +def is_file(path): + if p(path).is_file(): + return True + return False + + +def is_dict(path): + if p(path).is_dir(): + return True + return False + + +def tilda(obj): + if isinstance(obj, list): + return [str(p(o).expanduser()) if isinstance(o, str) else o for o in obj] + elif isinstance(obj, str): + return str(p(obj).expanduser()) + else: + return obj + + +def fix_path(path): + + def __fix_path(path): + if not isinstance(path, str): + return path + elif '~' == path[0]: + tilda_fixed_path = tilda(path) + if is_file(tilda_fixed_path): + return tilda_fixed_path + else: + exit(path, ': does not exit.') + elif is_file(p.home() / path): + return str(p().home() / path) + elif is_dict(p.home() / path): + return str(p().home() / path) + else: + return path + + if isinstance(path, str): + return __fix_path(path) + elif isinstance(path, list): + return [__fix_path(p) for p in path] + else: + return path + + +def compare_strings(s1, s2): + s1, s2 = degrade(s1), degrade(s2) + if s1 != s2: + return False + return True + + +def mydecoder(string): + try: + string.encode('ascii') + return string + except: + ustring = string.encode('utf-8') + string = re.sub(b"\xe2\x80\x90", b"-", ustring) + return string.decode('utf-8') + + +def __degrade(sub, var): + def helper(s): + s = str(s) + s = mydecoder(s) + s = re.sub(sub, "", s).lower().strip() + if not s: + return None + return s + + if isinstance(var, list): + return [helper(v) if v else v for v in var] + else: + if var: + return helper(var) + else: + return None + +def light_degrade(var): + sub = "\(|\)|'|"|\'|\"" + return __degrade(sub=sub, var=var) + + +def degrade(var): + sub = "\(|\)|'|"|\'|\"|-|,|_|:|\.| |;|#|>|<|`|~|@" + return __degrade(sub=sub, var=var) + + +def degrade_hash(mylist): + if not isinstance(var, list): + sys.exit('degrade_hash :: intended for lists only') + local_hash = {} + return {v: degraded(v) for v in mylist} + + +def namecheck(infilename): + if infilename == str(p.home() / 'Dropbox'): + sys.exit('DONT OVERWRITE THE DROPBOX') + + +def open_txt(infilename): + namecheck(infilename) + infilename = str(infilename) + if '.txt' not in infilename: + infilename += '.txt' + infilename = fix_path(infilename) + with open(infilename, 'r') as infile: + output = infile.read().strip() + infile.close() + return output + + +def create_txt(data, output): + namecheck(output) + output = str(output) + if '.txt' not in output: + output += '.txt' + output = fix_path(output) + with open(output, 'w') as outfile: + outfile.write(data) + outfile.close() + + +def create_json(data, output): + namecheck(output) + output = str(output) + if '.json' not in output: + output += '.json' + output = fix_path(output) + with open(output, 'w') as outfile: + json.dump(data, outfile, indent=4, cls=SetEncoder) + outfile.close() + +def open_json(infile): + namecheck(infile) + infile = str(infile) + if '.json' not in infile: + infile += '.json' + infile = fix_path(infile) + with open(infile, 'r') as _infile: + return json.load(_infile) + + +def create_pickle(data, output): + namecheck(output) + output = str(output) + if '.pickle' not in output: + output += '.pickle' + output = fix_path(output) + with open(output, 'wb') as outfile: + pickle.dump(data, outfile) + outfile.close() + + +def open_pickle(infile): + namecheck(infile) + infile = str(infile) + if '.pickle' not in infile: + infile += '.pickle' + infile = fix_path(infile) + with open(infile, 'rb') as _infile: + output = pickle.load(_infile) + _infile.close() + return output + + +def create_csv(rows, infile): + namecheck(infile) + infile = str(infile) + if '.csv' not in infile: + infile += '.csv' + infile = fix_path(infile) + with open(infile, 'wb') as csvfile: + filewriter = csv.writer(csvfile, + delimiter=',', + quotechar='|', + quoting=csv.QUOTE_MINIMAL,) + for row in rows: + filewriter.writerow(row) + csvfile.close() + + +def prettify_ontodiff_json(output): + namecheck(output) + output = str(output) # shell fixes output path itself + if '.json' not in output: + output += '.json' + shellcommand = 'ex -s +\'g/\[[\ \\n]\+"/j4\' -cwq ' + output + if call(shellcommand, shell=True) == 1: + print('Could not prettify the json file') + else: + print('Prettify Complete For:', output) diff --git a/ilxutils/setup.py b/ilxutils/setup.py index 623b1c01..665ddd28 100755 --- a/ilxutils/setup.py +++ b/ilxutils/setup.py @@ -23,8 +23,11 @@ 'progressbar2', 'aiohttp', 'asyncio', + 'mysql-connector-python', + 'elasticsearch', 'sqlalchemy', 'pathlib', + 'networkx', ], # TODO: add a get functionality thats more specific query # entry_points={ From 0eb349e103d7c93fda702e749f59897d88238160 Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Tue, 17 Aug 2021 21:16:58 -0700 Subject: [PATCH 15/17] sql update --- ilxutils/ilxutils/backup_ilx.py | 59 +++--- ilxutils/ilxutils/interlex_sql.py | 178 ++++++++++------- ilxutils/setup.py | 10 +- ilxutils/tools.py | 321 ------------------------------ 4 files changed, 139 insertions(+), 429 deletions(-) delete mode 100644 ilxutils/tools.py diff --git a/ilxutils/ilxutils/backup_ilx.py b/ilxutils/ilxutils/backup_ilx.py index 20006cf1..f0b2faf2 100644 --- a/ilxutils/ilxutils/backup_ilx.py +++ b/ilxutils/ilxutils/backup_ilx.py @@ -1,5 +1,5 @@ from pathlib import Path as p -from interlex_sql import IlxSql +from .interlex_sql import IlxSql # from tools import create_pickle import pickle import os @@ -10,39 +10,40 @@ def create_pickle(data, outfilename): pickle.dump(data, outfile) -sql = IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_PRODUCTION')) +def main(): + sql = IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_PRODUCTION')) + terms = sql.get_terms() + create_pickle(terms, p.home() / 'Dropbox/interlex_backups/ilx_db_terms_backup.pickle') + print('=== terms backup complete ===') + del terms -terms = sql.get_terms() -create_pickle(terms, p.home() / 'Dropbox/interlex_backups/ilx_db_terms_backup.pickle') -print('=== terms backup complete ===') -del terms + annos = sql.get_annotations() + create_pickle(annos, p.home() / 'Dropbox/interlex_backups/ilx_db_annos_backup.pickle') + print('=== annotations backup complete ===') + del annos + ex = sql.get_existing_ids() + create_pickle(ex, p.home() / 'Dropbox/interlex_backups/ilx_db_ex_backup.pickle') + print('=== existing ids backup complete ===') + del ex -annos = sql.get_annotations() -create_pickle(annos, p.home() / 'Dropbox/interlex_backups/ilx_db_annos_backup.pickle') -print('=== annotations backup complete ===') -del annos + synonyms = sql.get_synonyms() + create_pickle(synonyms, p.home() / 'Dropbox/interlex_backups/ilx_db_synonyms_backup.pickle') + print('=== synonyms backup complete ===') + del synonyms + superclasses = sql.get_superclasses() + create_pickle(superclasses, p.home() / 'Dropbox/interlex_backups/ilx_db_superclasses_backup.pickle') + print('=== superclasses backup complete ===') + del superclasses -ex = sql.get_existing_ids() -create_pickle(ex, p.home() / 'Dropbox/interlex_backups/ilx_db_ex_backup.pickle') -print('=== existing ids backup complete ===') -del ex + relationships = sql.get_relationships() + create_pickle(relationships, p.home() / 'Dropbox/interlex_backups/ilx_db_relationships_backup.pickle') + print('=== relationships backup complete ===') + del relationships + -synonyms = sql.get_synonyms() -create_pickle(synonyms, p.home() / 'Dropbox/interlex_backups/ilx_db_synonyms_backup.pickle') -print('=== synonyms backup complete ===') -del synonyms - - -superclasses = sql.get_superclasses() -create_pickle(superclasses, p.home() / 'Dropbox/interlex_backups/ilx_db_superclasses_backup.pickle') -print('=== superclasses backup complete ===') -del superclasses - -relationships = sql.get_relationships() -create_pickle(relationships, p.home() / 'Dropbox/interlex_backups/ilx_db_relationships_backup.pickle') -print('=== relationships backup complete ===') -del relationships +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/ilxutils/ilxutils/interlex_sql.py b/ilxutils/ilxutils/interlex_sql.py index a9283d00..544960e4 100755 --- a/ilxutils/ilxutils/interlex_sql.py +++ b/ilxutils/ilxutils/interlex_sql.py @@ -1,14 +1,17 @@ from pathlib import Path +import numpy as np import pandas as pd from sqlalchemy import create_engine, inspect, Table, Column from collections import defaultdict from .tools import light_degrade, open_pickle, create_pickle import os #ELASTIC = 'https://5f86098ac2b28a982cebf64e82db4ea2.us-west-2.aws.found.io:9243/interlex/term/' -TERMS_COMPLETE_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_terms_complete_backup.pickle' +TERMS_COMPLETE_BACKUP_PATH = Path.home( +)/'Dropbox/interlex_backups/ilx_db_terms_complete_backup.pickle' TERMS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_terms_backup.pickle' ANNOS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_annotations_backup.pickle' -RELAS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_relationships_backup.pickle' +RELAS_BACKUP_PATH = Path.home( +)/'Dropbox/interlex_backups/ilx_db_relationships_backup.pickle' SUPER_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_superclasses_backup.pickle' SYNOS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_synonyms_backup.pickle' EXIDS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_ex_backup.pickle' @@ -19,7 +22,8 @@ class IlxSql(): def __init__(self, db_url, pre_load=False, from_backup=False): self.db_url = db_url self.engine = create_engine(self.db_url) - self.local_degrade = lambda string: string.lower().strip() # current degrade of choice for sql + # current degrade of choice for sql + self.local_degrade = lambda string: string.lower().strip() self.from_backup = from_backup self.terms_complete = self.get_terms_complete() if pre_load else pd.DataFrame self.terms = self.get_terms() if pre_load else pd.DataFrame @@ -29,6 +33,25 @@ def __init__(self, db_url, pre_load=False, from_backup=False): self.relationships = self.get_relationships() if pre_load else pd.DataFrame self.synonyms = self.get_synonyms() if pre_load else pd.DataFrame + def fix_df_types(self, df: pd.DataFrame) -> pd.DataFrame: + """ + Convert Float Columns to Int with 0 for NaN while converting the rest of the NaNs to None. + + Parameters + ---------- + df : dataframe + + Returns + ------- + dataframe + 0 and None in place of nans + """ + float_col = df.select_dtypes(include=['float64']) + for col in float_col: + df[col] = df[col].replace({np.nan: 0}).astype(int) + df = df.replace({np.nan: None}) + return df + def fetch_terms_complete(self): if self.terms_complete.empty: return self.get_terms_complete() @@ -64,161 +87,168 @@ def fetch_superclasses(self): return self.get_superclasses() return self.superclasses - def get_terms(self): + def get_terms(self, status=0, refresh=False): ''' GROUP BY is a shortcut to only getting the first in every list of group ''' - if not self.terms.empty: + if not self.terms.empty and not refresh: return self.terms if self.from_backup: self.terms = open_pickle(TERMS_BACKUP_PATH) return self.terms engine = create_engine(self.db_url) - data = """ - SELECT t.id as tid, t.ilx, t.label, t.definition, t.type, t.comment, t.version, t.uid, t.time, t.status + data = f""" + SELECT t.* FROM terms t - GROUP BY t.ilx + WHERE t.status = '{status}' """ self.terms = pd.read_sql(data, engine) + self.terms = self.fix_df_types(self.terms) create_pickle(self.terms, TERMS_BACKUP_PATH) return self.terms - def get_annotations(self): - if not self.annotations: + def get_annotations(self, status=0, withdrawn=0, refresh=False): + if not self.annotations.empty and not refresh: return self.fetch_annotations() if self.from_backup: self.annotations = open_pickle(ANNOS_BACKUP_PATH) return self.annotations engine = create_engine(self.db_url) - data = """ + data = f""" SELECT - ta.tid, ta.annotation_tid as annotation_type_tid, - t1.ilx as term_ilx, t2.ilx as annotation_type_ilx, - t2.label as annotation_type_label, - ta.value, ta.withdrawn + t1.ilx as term_ilx, t1.label as term_label, t1.type as term_type, + t2.ilx as annotation_type_ilx, t2.label as annotation_type_label, + ta.* FROM term_annotations AS ta JOIN ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) AS t1 ON ta.tid=t1.id JOIN ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) AS t2 ON ta.annotation_tid=t2.id + WHERE ta.withdrawn = '{withdrawn}' """ self.annotations = pd.read_sql(data, engine) + self.annotations = self.fix_df_types(self.annotations) create_pickle(self.annotations, ANNOS_BACKUP_PATH) return self.annotations - def get_existing_ids(self): - if not self.existing_ids.empty: + def get_existing_ids(self, status=0, refresh=False): + if not self.existing_ids.empty and not refresh: return self.existing_ids if self.from_backup: self.existing_ids = open_pickle(EXIDS_BACKUP_PATH) return self.existing_ids engine = create_engine(self.db_url) - data = """ - SELECT tei.tid, tei.curie, tei.iri, tei.preferred, t.ilx, t.label, t.definition, t.status + data = f""" + SELECT tei.*, t.ilx, t.label, t.type FROM ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) as t JOIN term_existing_ids AS tei ON t.id = tei.tid """ self.existing_ids = pd.read_sql(data, engine) + self.existing_ids = self.fix_df_types(self.existing_ids) create_pickle(self.existing_ids, EXIDS_BACKUP_PATH) return self.existing_ids - def get_relationships(self): - if not self.relationships.empty: + def get_relationships(self, status=0, withdrawn=0, refresh=False): + if not self.relationships.empty and not refresh: return self.relationships if self.from_backup: self.relationships = open_pickle(RELAS_BACKUP_PATH) return self.relationships engine = create_engine(self.db_url) - data = """ + data = f""" SELECT - t1.id as term1_tid, t1.ilx AS term1_ilx, t1.type as term1_type, - t2.id as term2_tid, t2.ilx AS term2_ilx, t2.type as term2_type, - t3.id as relationship_tid, t3.ilx AS relationship_ilx, t3.label as relationship_label, - tr.withdrawn + t1.ilx AS term1_ilx, t1.type as term1_type, t1.label as term1_label, + t2.ilx AS term2_ilx, t2.type as term2_type, t2.label as term2_label, + t3.ilx AS relationship_ilx, t3.label as relationship_label, + tr.* FROM term_relationships AS tr JOIN ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) t1 ON t1.id = tr.term1_id JOIN ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) AS t2 ON t2.id = tr.term2_id JOIN ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) AS t3 ON t3.id = tr.relationship_tid + WHERE tr.withdrawn = '{withdrawn}' """ self.relationships = pd.read_sql(data, engine) + self.relationships = self.fix_df_types(self.relationships) create_pickle(self.relationships, RELAS_BACKUP_PATH) return self.relationships - def get_superclasses(self): - if not self.superclasses.empty: + def get_superclasses(self, status=0, withdrawn=0, refresh=False): + if not self.superclasses.empty and not refresh: return self.superclasses if self.from_backup: self.superclasses = open_pickle(SUPER_BACKUP_PATH) return self.superclasses engine = create_engine(self.db_url) - data = """ + data = f""" SELECT - ts.tid, ts.superclass_tid, - t1.label as term_label, t1.ilx as term_ilx, - t2.label as superclass_label, t2.ilx as superclass_ilx + t1.ilx as term_ilx, t1.label as term_label, + t2.ilx as superclass_ilx, t2.label as superclass_label, + ts.* FROM term_superclasses AS ts JOIN ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) as t1 ON t1.id = ts.tid JOIN ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) AS t2 ON t2.id = ts.superclass_tid """ self.superclasses = pd.read_sql(data, engine) + self.superclasses = self.fix_df_types(self.superclasses) create_pickle(self.superclasses, SUPER_BACKUP_PATH) return self.superclasses - def get_synonyms(self): - if not self.synonyms.empty: + def get_synonyms(self, status=0, refresh=False): + if not self.synonyms.empty and not refresh: return self.synonyms if self.from_backup: self.synonyms = open_pickle(SYNOS_BACKUP_PATH) return self.synonyms engine = create_engine(self.db_url) - data = """ - SELECT ts.tid as tid, t.ilx, ts.literal, ts.type + data = f""" + SELECT t.ilx, t.type as term_type, t.label, ts.* FROM term_synonyms AS ts JOIN ( SELECT * FROM terms - GROUP BY terms.ilx + WHERE status = '{status}' ) AS t WHERE ts.tid=t.id """ self.synonyms = pd.read_sql(data, engine) + self.synonyms = self.fix_df_types(self.synonyms) create_pickle(self.synonyms, SYNOS_BACKUP_PATH) return self.synonyms - def get_terms_complete(self) -> pd.DataFrame: + def get_terms_complete(self, status=0, withdrawn=0, refresh=False) -> pd.DataFrame: ''' Gets complete entity data like term/view ''' - if not self.terms_complete.empty: + if not self.terms_complete.empty and not refresh: return self.terms_complete if self.from_backup: self.terms_complete = open_pickle(TERMS_COMPLETE_BACKUP_PATH) @@ -230,9 +260,10 @@ def get_terms_complete(self) -> pd.DataFrame: ilx_complete = [] header = ['Index'] + list(self.fetch_terms().columns) for row in self.fetch_terms().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} row['synonyms'] = ilx2synonyms.get(row['ilx']) - row['existing_ids'] = ilx2existing_ids[row['ilx']] # if breaks we have worse problems + # if breaks we have worse problems + row['existing_ids'] = ilx2existing_ids[row['ilx']] row['annotations'] = ilx2annotations.get(row['ilx']) row['superclass'] = ilx2superclass.get(row['ilx']) ilx_complete.append(row) @@ -275,7 +306,7 @@ def get_label2rows(self): label2rows = defaultdict(list) header = ['Index'] + list(self.terms_complete.columns) for row in self.terms_complete.itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} label = self.local_degrade(row['label']) if not visited.get((label, row['type'], row['ilx'])): label2rows[label].append(row) @@ -288,7 +319,7 @@ def get_definition2rows(self): definition2rows = defaultdict(list) header = ['Index'] + list(self.terms.columns) for row in self.terms.itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} definition = self.local_degrade(row['definition']) if not definition or definition == ' ': continue @@ -301,7 +332,7 @@ def get_tid2row(self): tid2row = {} header = ['Index'] + list(self.fetch_terms().columns) for row in self.fetch_terms().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} tid2row[row['tid']] = row return tid2row @@ -309,16 +340,16 @@ def get_ilx2row(self): ilx2row = {} header = ['Index'] + list(self.fetch_terms().columns) for row in self.fetch_terms().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} ilx2row[row['ilx']] = row return ilx2row - def get_ilx2superclass(self, clean:bool=True): + def get_ilx2superclass(self, clean: bool = True): ''' clean: for list of literals only ''' ilx2superclass = defaultdict(list) header = ['Index'] + list(self.fetch_superclasses().columns) for row in self.fetch_superclasses().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} if clean: superclass = { 'tid': row['superclass_tid'], @@ -329,12 +360,12 @@ def get_ilx2superclass(self, clean:bool=True): ilx2superclass[row['term_ilx']].append(row) return ilx2superclass - def get_tid2annotations(self, clean:bool=True): + def get_tid2annotations(self, clean: bool = True): ''' clean: for list of literals only ''' tid2annotations = defaultdict(list) header = ['Index'] + list(self.fetch_annotations().columns) for row in self.fetch_annotations().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} if clean: annotation = { 'tid': row['tid'], @@ -347,12 +378,12 @@ def get_tid2annotations(self, clean:bool=True): tid2annotations[row['tid']].append(row) return tid2annotations - def get_ilx2annotations(self, clean:bool=True): + def get_ilx2annotations(self, clean: bool = True): ''' clean: for list of literals only ''' ilx2annotations = defaultdict(list) header = ['Index'] + list(self.fetch_annotations().columns) for row in self.fetch_annotations().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} if clean: annotation = { 'tid': row['tid'], @@ -365,28 +396,27 @@ def get_ilx2annotations(self, clean:bool=True): ilx2annotations[row['term_ilx']].append(row) return ilx2annotations - - def get_tid2synonyms(self, clean:bool=True): + def get_tid2synonyms(self, clean: bool = True): ''' clean: for list of literals only ''' tid2synonyms = {} header = ['Index'] + list(self.fetch_synonyms().columns) for row in self.fetch_synonyms().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} if clean: - synonym = {'literal':row['literal'], 'type':row['type']} + synonym = {'literal': row['literal'], 'type': row['type']} tid2synonyms[row['tid']].append(synonym) elif not clean: tid2synonyms[row['tid']].append(row) return tid2synonyms - def get_ilx2synonyms(self, clean:bool=True): + def get_ilx2synonyms(self, clean: bool = True): ''' clean: for list of literals only ''' ilx2synonyms = defaultdict(list) header = ['Index'] + list(self.fetch_synonyms().columns) for row in self.fetch_synonyms().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} if clean: - synonym = {'literal':row['literal'], 'type':row['type']} + synonym = {'literal': row['literal'], 'type': row['type']} ilx2synonyms[row['ilx']].append(synonym) elif not clean: ilx2synonyms[row['ilx']].append(row) @@ -396,7 +426,7 @@ def get_iri2row(self): iri2row = {} header = ['Index'] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} iri2row[row['iri']] = row return iri2row @@ -404,9 +434,9 @@ def get_tid2existing_ids(self, clean=True): tid2existing_ids = defaultdict(list) header = ['Index'] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} if clean: - existing_id = {'iri':row['iri'], 'curie':row['curie']} + existing_id = {'iri': row['iri'], 'curie': row['curie']} tid2existing_ids[row['tid']].append(existing_id) elif not clean: tid2existing_ids[row['tid']].append(row) @@ -416,9 +446,9 @@ def get_ilx2existing_ids(self, clean=True): ilx2existing_ids = defaultdict(list) header = ['Index'] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} if clean: - existing_id = {'iri':row['iri'], 'curie':row['curie']} + existing_id = {'iri': row['iri'], 'curie': row['curie']} ilx2existing_ids[row['ilx']].append(existing_id) elif not clean: ilx2existing_ids[row['ilx']].append(row) @@ -428,7 +458,7 @@ def get_curie2row(self): curie2row = {} header = ['Index'] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} + row = {header[i]: val for i, val in enumerate(row)} curie2row[row['curie']] = row return curie2row @@ -436,8 +466,8 @@ def get_fragment2rows(self): fragement2rows = defaultdict(list) header = ['Index'] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): - row = {header[i]:val for i, val in enumerate(row)} - if not row['curie']: # there are a few with no curies that will cause a false positive + row = {header[i]: val for i, val in enumerate(row)} + if not row['curie']: # there are a few with no curies that will cause a false positive continue fragment = row['curie'].split(':')[-1] fragement2rows[fragment].append(row) diff --git a/ilxutils/setup.py b/ilxutils/setup.py index 665ddd28..5335342a 100755 --- a/ilxutils/setup.py +++ b/ilxutils/setup.py @@ -30,9 +30,9 @@ 'networkx', ], # TODO: add a get functionality thats more specific query - # entry_points={ - # 'console_scripts': [ - # 'ilxutils = ilxutils.cli: main', - # ], - # }, + entry_points={ + 'console_scripts': [ + 'backup_ilx = ilxutils.backup_ilx:main', + ], + }, ) diff --git a/ilxutils/tools.py b/ilxutils/tools.py deleted file mode 100644 index d92290e5..00000000 --- a/ilxutils/tools.py +++ /dev/null @@ -1,321 +0,0 @@ -import json -import pickle -import re -import pandas as pd -from pathlib import Path as p -import pprint -from subprocess import call -from sys import exit -import csv -from typing import Union, Dict, List -import networkx as nx - -def sort_list_of_tuples_by_string(list_of_tuples:List[tuple], string_index:int) -> List[tuple]: - return sorted(list_of_tuples, key=lambda x: (str(x[string_index]).strip() in ['None', ''], x[string_index].lower())) - -def class_hierarchy(dag:List[tuple], descending=True) -> list: - ''' Topological Sorting - Args: - dag: directed acyclic graph that is a mappings of list of tuples with len of 2 - Returns: - Ordered list of single entities in topological ordering of choice - Examples: - >>> class_hierarchy([(1, 2), (2, 3)]) - [3, 2, 1] - ''' - dag = nx.DiGraph(dag) - dag = list(nx.topological_sort(dag)) - if descending: - dag = list(reversed(dag)) - return dag - -def clean(self, string, clean_scale:int=0): - if clean_scale == 0: - return string - elif clean_scale == 1: - return string.lower().strip() - elif clean_scale == 2: - return ' '.join(string_profiler(string)) - elif clean_scale == 3: - return ' '.join(string_profiler(string)).replace('obsolete', '').strip() - - -def string_profiler( - string: str, - start_delimiter: str='(', - end_delimiter: str=')', - remove: bool=True, - keep_delimiter: bool = True, - ) -> List[str]: - ''' - Seperates strings fragements into list based on the start and end delimiters - Args: - string: complete string you want to be broken up based on start and stop delimiters given - start_delimiter: delimiter element to start - end_delimiter: delimiter elemtent to end - remove: decide whether or not to keep strings inside the delimiters - Returns: - List[str]: list of strings that are split at start and end delimiters given and whether - or not you want to remove the string inside the delimiters - Tests: - long = '(life is is good) love world "(blah) blah" "here I am" once again "yes" blah ' - print(string_profiler(long)) - null = '' - print(string_profiler(null)) - short = '(life love) yes(and much more)' - print(string_profiler(short)) - short = 'yes "life love"' - print(string_profiler(short)) - ''' - outer_index = 0 # stepper for outer delimier string elements - inner_index = 0 # stepper for inner delimier string elements - curr_index = 0 # actual index of the current element in the string - string_list = [] # string broken up into individual elements whenever a start and end delimiter is hit - outer_string = '' # string tracked while outside the delimiters - inner_string = '' # string tracked while inside the delimiters - - for outer_index in range(len(string)): - # Actual pointer position (inner delimiter counter + outer delimiter counter) - curr_index = inner_index + outer_index - # Close once acutal index is at the end - # NOTE: outer_index will keep going till end regardless of hitting a delimiter and adding to inner stepper. - if curr_index == len(string): break - ### DELIMITER HIT ### - if string[curr_index] == start_delimiter: - # If we his a delimiter, collect the string previous to that as an element; flush - if outer_string: - # Option: .extend(outer_string.strip().split()) | If you want every word seperate. Maybe an option? - string_list.append(outer_string.strip()) - outer_string = '' - for j in range(curr_index+1, len(string)): - # Stepper that is pushed while in inner delimiter string. - inner_index += 1 - # Once we his the end delimiter, stop iterating through the inner delimiter string - if string[j] == end_delimiter: break - # String inside delimiters - inner_string += string[j] - # If you want the string inside the delimiters - if not remove: - if keep_delimiter: - inner_string = start_delimiter + inner_string + end_delimiter - string_list.append(inner_string) - # inner delimiter string restart - inner_string = '' - # String outside of the delimiters - else: outer_string += string[curr_index] - # End delimiter is either nested or not the real target; should ignore - if string[curr_index] == end_delimiter: - if string_list and outer_string: - string_list[-1] += outer_string - outer_string = '' - # In case of not hiting a delimiter at the end of the string, collect the remaining outer delimiter string - # Option: .extend(outer_string.strip().split()) | If you want every word seperate. Maybe an option? - if outer_string: string_list.append(outer_string.strip()) - return string_list - - -pp = pprint.PrettyPrinter(indent=4).pprint - - -class SetEncoder(json.JSONEncoder): - ''' Custom encoder to allow json to convert any sets in nested data to become lists ''' - def default(self, obj): - if isinstance(obj, set): - return sorted(list(obj)) - return json.JSONEncoder.default(self, obj) - -def is_file(path): - if p(path).is_file(): - return True - return False - - -def is_dict(path): - if p(path).is_dir(): - return True - return False - - -def tilda(obj): - if isinstance(obj, list): - return [str(p(o).expanduser()) if isinstance(o, str) else o for o in obj] - elif isinstance(obj, str): - return str(p(obj).expanduser()) - else: - return obj - - -def fix_path(path): - - def __fix_path(path): - if not isinstance(path, str): - return path - elif '~' == path[0]: - tilda_fixed_path = tilda(path) - if is_file(tilda_fixed_path): - return tilda_fixed_path - else: - exit(path, ': does not exit.') - elif is_file(p.home() / path): - return str(p().home() / path) - elif is_dict(p.home() / path): - return str(p().home() / path) - else: - return path - - if isinstance(path, str): - return __fix_path(path) - elif isinstance(path, list): - return [__fix_path(p) for p in path] - else: - return path - - -def compare_strings(s1, s2): - s1, s2 = degrade(s1), degrade(s2) - if s1 != s2: - return False - return True - - -def mydecoder(string): - try: - string.encode('ascii') - return string - except: - ustring = string.encode('utf-8') - string = re.sub(b"\xe2\x80\x90", b"-", ustring) - return string.decode('utf-8') - - -def __degrade(sub, var): - def helper(s): - s = str(s) - s = mydecoder(s) - s = re.sub(sub, "", s).lower().strip() - if not s: - return None - return s - - if isinstance(var, list): - return [helper(v) if v else v for v in var] - else: - if var: - return helper(var) - else: - return None - -def light_degrade(var): - sub = "\(|\)|'|"|\'|\"" - return __degrade(sub=sub, var=var) - - -def degrade(var): - sub = "\(|\)|'|"|\'|\"|-|,|_|:|\.| |;|#|>|<|`|~|@" - return __degrade(sub=sub, var=var) - - -def degrade_hash(mylist): - if not isinstance(var, list): - sys.exit('degrade_hash :: intended for lists only') - local_hash = {} - return {v: degraded(v) for v in mylist} - - -def namecheck(infilename): - if infilename == str(p.home() / 'Dropbox'): - sys.exit('DONT OVERWRITE THE DROPBOX') - - -def open_txt(infilename): - namecheck(infilename) - infilename = str(infilename) - if '.txt' not in infilename: - infilename += '.txt' - infilename = fix_path(infilename) - with open(infilename, 'r') as infile: - output = infile.read().strip() - infile.close() - return output - - -def create_txt(data, output): - namecheck(output) - output = str(output) - if '.txt' not in output: - output += '.txt' - output = fix_path(output) - with open(output, 'w') as outfile: - outfile.write(data) - outfile.close() - - -def create_json(data, output): - namecheck(output) - output = str(output) - if '.json' not in output: - output += '.json' - output = fix_path(output) - with open(output, 'w') as outfile: - json.dump(data, outfile, indent=4, cls=SetEncoder) - outfile.close() - -def open_json(infile): - namecheck(infile) - infile = str(infile) - if '.json' not in infile: - infile += '.json' - infile = fix_path(infile) - with open(infile, 'r') as _infile: - return json.load(_infile) - - -def create_pickle(data, output): - namecheck(output) - output = str(output) - if '.pickle' not in output: - output += '.pickle' - output = fix_path(output) - with open(output, 'wb') as outfile: - pickle.dump(data, outfile) - outfile.close() - - -def open_pickle(infile): - namecheck(infile) - infile = str(infile) - if '.pickle' not in infile: - infile += '.pickle' - infile = fix_path(infile) - with open(infile, 'rb') as _infile: - output = pickle.load(_infile) - _infile.close() - return output - - -def create_csv(rows, infile): - namecheck(infile) - infile = str(infile) - if '.csv' not in infile: - infile += '.csv' - infile = fix_path(infile) - with open(infile, 'wb') as csvfile: - filewriter = csv.writer(csvfile, - delimiter=',', - quotechar='|', - quoting=csv.QUOTE_MINIMAL,) - for row in rows: - filewriter.writerow(row) - csvfile.close() - - -def prettify_ontodiff_json(output): - namecheck(output) - output = str(output) # shell fixes output path itself - if '.json' not in output: - output += '.json' - shellcommand = 'ex -s +\'g/\[[\ \\n]\+"/j4\' -cwq ' + output - if call(shellcommand, shell=True) == 1: - print('Could not prettify the json file') - else: - print('Prettify Complete For:', output) From dad24e7178d8d8cd3bd60d53b9039952fa7a5a1e Mon Sep 17 00:00:00 2001 From: Troy Sincomb Date: Thu, 19 Aug 2021 13:31:23 -0700 Subject: [PATCH 16/17] - --- nifstd/resources/pax-spine-2009.txt | 230 ------------------------- nifstd/resources/pax-spine-2013.txt | 256 ---------------------------- 2 files changed, 486 deletions(-) delete mode 100644 nifstd/resources/pax-spine-2009.txt delete mode 100644 nifstd/resources/pax-spine-2013.txt diff --git a/nifstd/resources/pax-spine-2009.txt b/nifstd/resources/pax-spine-2009.txt deleted file mode 100644 index 7b06f4ca..00000000 --- a/nifstd/resources/pax-spine-2009.txt +++ /dev/null @@ -1,230 +0,0 @@ -List of Structures -; # -; A -accessory nerve nucleus 11N -adductor motoneurons of lamina 9 Ad9 -anterior spinal artery asp -axial muscle motoneurons of lamina 9 Ax9 -; B -biceps motoneurons of lamina 9 Bi9 -; C -cauda equina CE -central canal CC -central cervical nucleus CeCv -coccygeal vertebra CoVert -cremaster motoneurons of lamina 9 Cr9 -crural extensor motoneurons of lamina 9 CEx9 -crural flexor motoneurons of lamina 9 CFl9 -cuneate fasciculus cu -; D -deltoid motoneurons of lamina 9 De9 -dorsal corticospinal tract dcs -dorsal funiculus df -dorsal nucleus (Clarke) D -dorsal root dr -dorsal root ganglion DRG -dorsolateral fasciculus (Lissauer) dl -; E -external anal sphincter motoneurons of lamina 9 ExA9 -external urethral sphincter motoneurons of lamina 9 ExU9 -; F -forearm extensor motoneurons of lamina 9 FEx9 -forearm flexor motoneurons of lamina 9 FFl9 -; G -Gl9 gracile fasciculus gr -gluteal motoneurons of lamina 9 Gl9 -; H -hamstring motoneurons of lamina 9 Hm9 -; I -iliopsoas motoneurons of lamina 9 IPs9 -infrahyoid muscle motoneurons of lamina 9 IH9 -intercalated nucleus ICl -intercalated nucleus, paraependymal part IClPe -intercostal muscle motoneurons of lamina 9 ICo9 -intermediolateral column IML -intermediomedial column IMM -internal basilar nucleus IB -; J -; K -; L -lamina 1 of the spinal gray 1Sp -lamina 10 of the spinal gray 10Sp -lamina 2 of the spinal gray 2Sp -lamina 2 of the spinal gray, inner part 2SpI -lamina 2 of the spinal gray, outer part 2SpO -lamina 3 of the spinal gray 3Sp -lamina 4 of the spinal gray 4Sp -lamina 5 of the spinal gray 5Sp -lamina 5 of the spinal gray, lateral part 5SpL -lamina 6 of the spinal gray 6Sp -lamina 6 of the spinal gray, lateral part 6SpL -lamina 6 of the spinal gray, medial part 6SpM -lamina 7 of the spinal gray 7Sp -lamina 8 of the spinal gray 8Sp -lamina 9 of the spinal gray 9Sp -lamina 5 of the spinal gray, medial part 5SpM -lateral cervical nucleus LatC -lateral funiculus lf -lateral spinal nucleus LSp -latissimus dorsi motoneurons of lamina 9 LD9 -levator scapulae motoneurons of lamina 9 LS9 -lumbar dorsal commissural nucleus LDCom -lumbar precerebellar nucleus LPrCb -; M -manus motoneurons of lamina 9 Man9 -; N -; O -; P -pectoral muscle motoneurons of lamina 9 Pec9 -pes motoneurons of lamina 9 Pes9 -phrenic motoneurons of lamina 9 Ph9 -postsynaptic dorsal column pathway psdc -psoas motoneurons of lamina 9 Ps9 -; Q -quadratus lumborum motoneurons of lamina 9 QL9 -quadriceps motoneurons of lamina 9 Q9 -; R -rhomboid muscle motoneurons of lamina 9 Rh9 -rubrospinal tract rs -; S -sacral dorsal commissural nucleus SDCom -sacral parasympathetic nucleus SPSy -sacral precerebellar nucleus SPrCb -scalene muscle motoneurons of lamina 9 Sc9 -serratus anterior motoneurons in lamina 9 Sr9 -spinal accessory nerve 11n -spinous process of vertebra SpPr -sternomastoid motoneurons of lamina 9 SM9 -supraspinatus and infraspinatus motoneurons of lamina 9 SI9 -; T -tail muscle motoneurons of lamina 9 Tail9 -thoracoabdominal wall muscle motoneurons of lamina 9 ThAb9 -trapezius and sternomastoid motoneurons of lamina 9 TzSM9 -trapezius motoneurons of lamina 9 Tz9 -triceps motoneurons of lamina 9 Tr9 -; U -; V -ventral funiculus vf -ventral median fissure vmf -ventral root vr -ventral white commissure vwc -vertebral arch VAr -vertebral body VB -; W -; X -; Y -; Z - -List of Abbreviations -; # -10Sp lamina 10 of the spinal gray -11N accessory nerve nucleus -11n spinal accessory nerve -1Sp lamina 1 of the spinal gray -2Sp lamina 2 of the spinal gray -2SpI lamina 2 of the spinal gray, inner part -2SpO lamina 2 of the spinal gray, outer part -3Sp lamina 3 of the spinal gray -4Sp lamina 4 of the spinal gray -5Sp lamina 5 of the spinal gray -5SpL lamina 5 of the spinal gray, lateral part -5SpM lamina 5 of the spinal gray, medial part -6Sp lamina 6 of the spinal gray -6SpL lamina 6 of the spinal gray, lateral part -6SpM lamina 6 of the spinal gray, medial part -7Sp lamina 7 of the spinal gray -8Sp lamina 8 of the spinal gray -9Sp lamina 9 of the spinal gray -; A -Ad9 adductor motoneurons of lamina 9 -Ax9 axial muscle motoneurons of lamina 9 -asp anterior spinal artery -; B -Bi9 biceps motoneurons of lamina 9 -; C -CE cauda equina -CEx9 crural extensor motoneurons of lamina 9 -CFl9 crural flexor motoneurons of lamina 9 -CeCv central cervical nucleus -CoVert coccygeal vertebra -Cr9 cremaster motoneurons of lamina 9 -cu cuneate fasciculus -; D -D dorsal nucleus -DRG dorsal root ganglion -De9 deltoid motoneurons of lamina 9 -dcs dorsal corticospinal tract -df dorsal funiculus -dl dorsolateral fasciculus -dr dorsal root -; E -ExA9 external anal sphincter motoneurons of lamina 9 -ExU9 external urethral sphincter motoneurons of lamina 9 -; F -FEx9 forearm extensor motoneurons of lamina 9 -FFl9 forearm flexor motoneurons of lamina 9 -; G -Gl9 gluteal motoneurons of lamina 9 -gr gracile fasciculus -; H -Hm9 hamstring motoneurons of lamina 9 -; I -IB internal basilar nucleus -ICl intercalated nucleus -IClPe intercalated nucleus, paraependymal part -ICo9 intercostal muscle motoneurons of lamina 9 -IH9 infrahyoid muscle motoneurons of lamina 9 -IML intermediolateral column -IMM intermediomedial column -; J -; K -; L -LD9 latissimus dorsi motoneurons of lamina 9 -LDCom lumbar dorsal commissural nucleus -LPrCb lumbar precerebellar nucleus -LS9 levator scapulae motoneurons of lamina 9 -LSp lateral spinal nucleus -LatC lateral cervical nucleus -lf lateral funiculus -; M -Man9 manus motoneurons of lamina 9 -; N -; O -; P -Pec9 pectoral muscle motoneurons of lamina 9 -Pes9 pes motoneurons of lamina 9 -Ph9 phrenic motoneurons of lamina 9 -Ps9 psoas motoneurons of lamina 9 -psdc postsynaptic dorsal column pathway -; Q -Q9 quadriceps motoneurons of lamina 9 -QL9 quadratus lumborum motoneurons of lamina 9 -; R -Rh9 rhomboid muscle motoneurons of lamina 9 -rs rubrospinal tract -; S -SDCom sacral dorsal commissural nucleus -SI9 supraspinatus and infraspinatus motoneurons of lamina 9 -SM9 sternomastoid motoneurons of lamina 9 -SPSy sacral parasympathetic nucleus -SPrCb sacral precerebellar nucleus -Sc9 scalene muscle motoneurons of lamina 9 -SpPr spinous process of vertebra -Sr9 serratus anterior motoneurons in lamina 9 -; T -Tail9 tail muscle motoneurons of lamina 9 -ThAb9 thoracoabdominal wall muscle motoneurons of lamina 9 -Tr9 triceps motoneurons of lamina 9 -TzSM9 trapezius and sternomastoid motoneurons of lamina 9 -; U -; V -VAr vertebral arch -VB vertebral body -vf ventral funiculus -vmf ventral median fissure -vr ventral root -vwc ventral white commissure -; W -; X -; Y -; Z diff --git a/nifstd/resources/pax-spine-2013.txt b/nifstd/resources/pax-spine-2013.txt deleted file mode 100644 index 6143055d..00000000 --- a/nifstd/resources/pax-spine-2013.txt +++ /dev/null @@ -1,256 +0,0 @@ -List of Structures -; # -; A -adductor motoneurons of lamina 9 Ad9 -anterior corticospinal tract acs -anterior spinal artery asp -area 10 of the spinal gray 10Sp -axial muscle motoneurons of lamina 9 Ax9 -; B -biceps motoneurons of lamina 9 Bi9 -; C -cauda equina CE -caudal reticulospinal tract crsp -central canal CC -central cervical nucleus CeCv -coccygeal vertebra CoVert -cremaster motoneurons of lamina 9 Cr9 -crural extensor motoneurons of lamina 9 CEx9 -crural flexor motoneurons of lamina 9 CF19 -cuneate fasciculus cu -; D -deltoid motoneurons of lamina 9 De9 -dorsal corticospinal tract dcs -dorsal funiculus df -dorsal nucleus (clarke) D -dorsal root dr -dorsal root dr -dorsal root ganglion DRG -dorsal spinocerebellar tract dsc -dorsolateral fasciculus (lissauer) dl -dorsolateral spinothalamic tract dlst -; E -external anal sphincter motoneurons of lamina 9 ExA9 -external urethral sphincter motoneurons of lamina 9 ExU9 -; F -forearm extensor motoneurons of lamina 9 FEx9 -forearm flexor motoneurons of lamina 9 FFl9 -; G -gluteal motoneurons of lamina 9 Gl9 -gracile fasciculus gr -; H -hamstring motoneurons of lamina 9 Hm9 -; I -iliopsoas motoneurons of lamina 9 IPs9 -infrahyoid muscle motoneurons of lamina 9 IH9 -intercalated nucleus ICl -intercalated nucleus, paraependymal part ICIPe -intercostal muscle motoneurons of lamina 9 ICo9 -intermediolateral column IML -intermediolateral column, funicular part IMFL -intermediomedial column IMM -internal basilar nucleus IB -; J -; K -; L -lamina 1 of the spinal gray 1Sp -lamina 2 of the spinal gray 2Sp -lamina 2 of the spinal gray, inner part 2SpI -lamina 2 of the spinal gray, outer part 2SpO -lamina 3 of the spinal gray 3Sp -lamina 4 of the spinal gray 4Sp -lamina 5 of the spinal gray 5Sp -lamina 5 of the spinal gray, lateral part 5SpL -lamina 5 of the spinal gray, medial part 5SpM -lamina 6 of the spinal gray 6Sp -lamina 6 of the spinal gray, lateral part 6SpL -lamina 6 of the spinal gray, medial part 6SpM -lamina 7 of the spinal gray 7Sp -lamina 8 of the spinal gray 8Sp -lamina 9 of the spinal gray 9Sp -lateral cervical nucleus LatC -lateral corticospinal tract lcs -lateral funiculus lf -lateral spinal nucleus LSp -lateral spinothalamic tract lst -lateral vestibulospinal tract lvs -latissimus dorsi motoneurons of lamina 9 LD9 -levator scapulae motoneurons of lamina 9 LS9 -lumbar dorsal commissural nucleus LDCom -lumbar precerebellar nucleus LPrCb -; M -manus motoneurons of lamina 9 Man9 -medial vestibulospinal tract mvs -; N -; O -; P -pectoral muscle motoneurons of lamina 9 Pec9 -pes motoneurons of lamina 9 Pes9 -phrenic motoneurons of lamina 9 Ph9 -postsynaptic dorsal column pathway psdc -psoas muscle motoneurons of lamina 9 Ps9 -; Q -quadratus lumborum motoneurons of lamina 9 QL9 -quadriceps motoneurons of lamina 9 Q9 -; R -rhomboid muscle motoneurons of lamina 9 Rh9 -rostral reticulospinal tract rrsp -rubrospinal tract rs -; S -sacral dorsal commissural nucleus SDCom -sacral parasympathetic nucleus SPSy -sacral precerebellar nucleus SPrCb -scalene muscle motoneurons of lamina 9 Sc9 -serratus anterior motoneurons in lamina 9 Sr9 -spinal accessory nerve 11n -spinous process of vertebra SpPr -sternomastoid motoneurons of lamina 9 SM9 -supraspinatus and infraspinatus motoneurons of lamina 9 SI9 -; T -tail muscle motoneurons of lamina 9 Tail9 -thoracoabdominal wall muscle motoneurons of lamina 9 ThAb9 -trapezius and sternomastoid motoneurons of lamina 9 TzSM9 -trapezius motoneurons of lamina 9 Tz9 -triceps motoneurons of lamina 9 Tr9 -; U -; V -ventral funiculus vf -ventral median fissure vmf -ventral root vr -ventral spinocerebellar tract vsc -ventral spinothalamic tract vst -ventral white commissure vwc -vertebral arch VAr -vertebral body VB -; W -; X -; Y -; Z - -List of Abbreviations -; # -; A -adductor motoneurons of lamina 9 Ad9 -anterior corticospinal tract acs -anterior spinal artery asp -area 10 of the spinal gray 10Sp -axial muscle motoneurons of lamina 9 Ax9 -; B -biceps motoneurons of lamina 9 Bi9 -; C -cauda equina CE -caudal reticulospinal tract crsp -central canal CC -central cervical nucleus CeCv -coccygeal vertebra CoVert -cremaster motoneurons of lamina 9 Cr9 -crural extensor motoneurons of lamina 9 CEx9 -crural flexor motoneurons of lamina 9 CF19 -cuneate fasciculus cu -; D -deltoid motoneurons of lamina 9 De9 -dorsal corticospinal tract dcs -dorsal funiculus df -dorsal nucleus (clarke) D -dorsal root dr -dorsal root ganglion DRG -dorsal spinocerebellar tract dsc -dorsolateral fasciculus (lissauer) dl -dorsolateral spinothalamic tract dlst -; E -external anal sphincter motoneurons of lamina 9 ExA9 -external urethral sphincter motoneurons of lamina 9 ExU9 -; F -forearm extensor motoneurons of lamina 9 FEx9 -forearm flexor motoneurons of lamina 9 FFl9 -; G -gluteal motoneurons of lamina 9 Gl9 -gracile fasciculus gr -; H -hamstring motoneurons of lamina 9 Hm9 -; I -iliopsoas motoneurons of lamina 9 IPs9 -infrahyoid muscle motoneurons of lamina 9 IH9 -intercalated nucleus ICl -intercalated nucleus, paraependymal part ICIPe -intercostal muscle motoneurons of lamina 9 ICo9 -intermediolateral column IML -intermediolateral column, funicular part IMFL -intermediomedial column IMM -internal basilar nucleus IB -; J -; K -; L -lamina 1 of the spinal gray 1Sp -lamina 2 of the spinal gray 2Sp -lamina 2 of the spinal gray, inner part 2SpI -lamina 2 of the spinal gray, outer part 2SpO -lamina 3 of the spinal gray 3Sp -lamina 4 of the spinal gray 4Sp -lamina 5 of the spinal gray 5Sp -lamina 5 of the spinal gray, lateral part 5SpL -lamina 5 of the spinal gray, medial part 5SpM -lamina 6 of the spinal gray 6Sp -lamina 6 of the spinal gray, lateral part 6SpL -lamina 6 of the spinal gray, medial part 6SpM -lamina 7 of the spinal gray 7Sp -lamina 8 of the spinal gray 8Sp -lamina 9 of the spinal gray 9Sp -lateral cervical nucleus LatC -lateral corticospinal tract lcs -lateral funiculus lf -lateral spinal nucleus LSp -lateral spinothalamic tract lst -lateral vestibulospinal tract lvs -latissimus dorsi motoneurons of lamina 9 LD9 -levator scapulae motoneurons of lamina 9 LS9 -lumbar dorsal commissural nucleus LDCom -lumbar precerebellar nucleus LPrCb -; M -manus motoneurons of lamina 9 Man9 -medial vestibulospinal tract mvs -; N -; O -; P -pectoral muscle motoneurons of lamina 9 Pec9 -pes motoneurons of lamina 9 Pes9 -phrenic motoneurons of lamina 9 Ph9 -postsynaptic dorsal column pathway psdc -psoas muscle motoneurons of lamina 9 Ps9 -; Q -quadratus lumborum motoneurons of lamina 9 QL9 -quadriceps motoneurons of lamina 9 Q9 -; R -rhomboid muscle motoneurons of lamina 9 Rh9 -rostral reticulospinal tract rrsp -rubrospinal tract rs -; S -sacral dorsal commissural nucleus SDCom -sacral parasympathetic nucleus SPSy -sacral precerebellar nucleus SPrCb -scalene muscle motoneurons of lamina 9 Sc9 -serratus anterior motoneurons in lamina 9 Sr9 -spinal accessory nerve 11n -spinous process of vertebra SpPr -sternomastoid motoneurons of lamina 9 SM9 -supraspinatus and infraspinatus motoneurons of lamina 9 SI9 -; T -tail muscle motoneurons of lamina 9 Tail9 -thoracoabdominal wall muscle motoneurons of lamina 9 ThAb9 -trapezius and sternomastoid motoneurons of lamina 9 TzSM9 -trapezius motoneurons of lamina 9 Tz9 -triceps motoneurons of lamina 9 Tr9 -; U -; V -ventral funiculus vf -ventral median fissure vmf -ventral root vr -ventral spinocerebellar tract vsc -ventral spinothalamic tract vst -ventral white commissure vwc -vertebral arch VAr -vertebral body VB -; W -; X -; Y -; Z From 4444ec1e8d903fdf61465b19c4157d52376e866a Mon Sep 17 00:00:00 2001 From: tmsincomb Date: Mon, 6 Mar 2023 08:58:03 -0800 Subject: [PATCH 17/17] sql update --- ilxutils/ilxutils/backup_ilx.py | 44 +++-- ilxutils/ilxutils/interlex_sql.py | 288 ++++++++++++++++++------------ 2 files changed, 197 insertions(+), 135 deletions(-) diff --git a/ilxutils/ilxutils/backup_ilx.py b/ilxutils/ilxutils/backup_ilx.py index f0b2faf2..6e15b868 100644 --- a/ilxutils/ilxutils/backup_ilx.py +++ b/ilxutils/ilxutils/backup_ilx.py @@ -1,49 +1,59 @@ from pathlib import Path as p from .interlex_sql import IlxSql + # from tools import create_pickle import pickle import os +HOME = p.home() / "DropboxPersonal/.interlex_backups" +if HOME.exists is False: + HOME.mkdir() + + def create_pickle(data, outfilename): - with open(outfilename, 'wb') as outfile: + with open(outfilename, "wb") as outfile: pickle.dump(data, outfile) def main(): - sql = IlxSql(db_url=os.environ.get('SCICRUNCH_DB_URL_PRODUCTION')) + sql = IlxSql(db_url=os.environ.get("SCICRUNCH_DB_URL_PRODUCTION")) + + users = sql.get_users() + create_pickle(users, HOME / "ilx_db_users_backup.pickle") + print("=== Users backup complete ===") + del users terms = sql.get_terms() - create_pickle(terms, p.home() / 'Dropbox/interlex_backups/ilx_db_terms_backup.pickle') - print('=== terms backup complete ===') + create_pickle(terms, HOME / "ilx_db_terms_backup.pickle") + print("=== terms backup complete ===") del terms annos = sql.get_annotations() - create_pickle(annos, p.home() / 'Dropbox/interlex_backups/ilx_db_annos_backup.pickle') - print('=== annotations backup complete ===') + create_pickle(annos, HOME / "ilx_db_annos_backup.pickle") + print("=== annotations backup complete ===") del annos ex = sql.get_existing_ids() - create_pickle(ex, p.home() / 'Dropbox/interlex_backups/ilx_db_ex_backup.pickle') - print('=== existing ids backup complete ===') + create_pickle(ex, HOME / "ilx_db_ex_backup.pickle") + print("=== existing ids backup complete ===") del ex synonyms = sql.get_synonyms() - create_pickle(synonyms, p.home() / 'Dropbox/interlex_backups/ilx_db_synonyms_backup.pickle') - print('=== synonyms backup complete ===') + create_pickle(synonyms, HOME / "ilx_db_synonyms_backup.pickle") + print("=== synonyms backup complete ===") del synonyms superclasses = sql.get_superclasses() - create_pickle(superclasses, p.home() / 'Dropbox/interlex_backups/ilx_db_superclasses_backup.pickle') - print('=== superclasses backup complete ===') + create_pickle(superclasses, HOME / "ilx_db_superclasses_backup.pickle") + print("=== superclasses backup complete ===") del superclasses relationships = sql.get_relationships() - create_pickle(relationships, p.home() / 'Dropbox/interlex_backups/ilx_db_relationships_backup.pickle') - print('=== relationships backup complete ===') + create_pickle(relationships, HOME / "ilx_db_relationships_backup.pickle") + print("=== relationships backup complete ===") del relationships - -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/ilxutils/ilxutils/interlex_sql.py b/ilxutils/ilxutils/interlex_sql.py index 544960e4..522fc1e5 100755 --- a/ilxutils/ilxutils/interlex_sql.py +++ b/ilxutils/ilxutils/interlex_sql.py @@ -5,33 +5,45 @@ from collections import defaultdict from .tools import light_degrade, open_pickle, create_pickle import os -#ELASTIC = 'https://5f86098ac2b28a982cebf64e82db4ea2.us-west-2.aws.found.io:9243/interlex/term/' -TERMS_COMPLETE_BACKUP_PATH = Path.home( -)/'Dropbox/interlex_backups/ilx_db_terms_complete_backup.pickle' -TERMS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_terms_backup.pickle' -ANNOS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_annotations_backup.pickle' -RELAS_BACKUP_PATH = Path.home( -)/'Dropbox/interlex_backups/ilx_db_relationships_backup.pickle' -SUPER_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_superclasses_backup.pickle' -SYNOS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_synonyms_backup.pickle' -EXIDS_BACKUP_PATH = Path.home()/'Dropbox/interlex_backups/ilx_db_ex_backup.pickle' +HOME = Path.home() / "DropboxPersonal" +# ELASTIC = 'https://5f86098ac2b28a982cebf64e82db4ea2.us-west-2.aws.found.io:9243/interlex/term/' +TERMS_COMPLETE_BACKUP_PATH = HOME / ".interlex_backups/ilx_db_terms_complete_backup.pickle" +USERS_BACKUP_PATH = HOME / ".interlex_backups/ilx_db_users_backup.pickle" +TERMS_BACKUP_PATH = HOME / ".interlex_backups/ilx_db_terms_backup.pickle" +ANNOS_BACKUP_PATH = HOME / ".interlex_backups/ilx_db_annotations_backup.pickle" +RELAS_BACKUP_PATH = HOME / ".interlex_backups/ilx_db_relationships_backup.pickle" +SUPER_BACKUP_PATH = HOME / ".interlex_backups/ilx_db_superclasses_backup.pickle" +SYNOS_BACKUP_PATH = HOME / ".interlex_backups/ilx_db_synonyms_backup.pickle" +EXIDS_BACKUP_PATH = HOME / ".interlex_backups/ilx_db_ex_backup.pickle" -class IlxSql(): +class IlxSql: def __init__(self, db_url, pre_load=False, from_backup=False): self.db_url = db_url self.engine = create_engine(self.db_url) # current degrade of choice for sql self.local_degrade = lambda string: string.lower().strip() self.from_backup = from_backup - self.terms_complete = self.get_terms_complete() if pre_load else pd.DataFrame - self.terms = self.get_terms() if pre_load else pd.DataFrame - self.superclasses = self.get_superclasses if pre_load else pd.DataFrame - self.annotations = self.get_annotations() if pre_load else pd.DataFrame - self.existing_ids = self.get_existing_ids() if pre_load else pd.DataFrame - self.relationships = self.get_relationships() if pre_load else pd.DataFrame - self.synonyms = self.get_synonyms() if pre_load else pd.DataFrame + # self.terms_complete = self.get_terms_complete() if pre_load else pd.DataFrame + self.users = pd.DataFrame + self.terms = pd.DataFrame + self.superclasses = pd.DataFrame + self.annotations = pd.DataFrame + self.existing_ids = pd.DataFrame + self.relationships = pd.DataFrame + self.synonyms = pd.DataFrame + if pre_load: + self.__pre_load_db() + + def __pre_load_db(self): + self.users = self.get_users() + self.terms = self.get_terms() + self.superclasses = self.get_superclasses() + self.annotations = self.get_annotations() + self.existing_ids = self.get_existing_ids() + self.relationships = self.get_relationships() + self.synonyms = self.get_synonyms() def fix_df_types(self, df: pd.DataFrame) -> pd.DataFrame: """ @@ -46,7 +58,7 @@ def fix_df_types(self, df: pd.DataFrame) -> pd.DataFrame: dataframe 0 and None in place of nans """ - float_col = df.select_dtypes(include=['float64']) + float_col = df.select_dtypes(include=["float64"]) for col in float_col: df[col] = df[col].replace({np.nan: 0}).astype(int) df = df.replace({np.nan: None}) @@ -57,6 +69,11 @@ def fetch_terms_complete(self): return self.get_terms_complete() return self.terms_complete + def fetch_users(self): + if self.users.empty: + return self.get_users() + return self.users + def fetch_terms(self): if self.terms.empty: return self.get_terms() @@ -87,21 +104,44 @@ def fetch_superclasses(self): return self.get_superclasses() return self.superclasses + def get_users(self, refresh=False) -> pd.DataFrame: + """GROUP BY is a shortcut to only getting the first in every list of group""" + if not self.users.empty and not refresh: + return self.users + if self.from_backup: + self.users = open_pickle(USERS_BACKUP_PATH) + self.users.name = "users" + return self.users + engine = create_engine(self.db_url) + data = f""" + SELECT u.* + FROM users u + """ + self.users = pd.read_sql(data, engine) + self.users = self.fix_df_types(self.users) + self.users.name = "users" + create_pickle(self.users, USERS_BACKUP_PATH) + return self.users + def get_terms(self, status=0, refresh=False): - ''' GROUP BY is a shortcut to only getting the first in every list of group ''' + """GROUP BY is a shortcut to only getting the first in every list of group""" if not self.terms.empty and not refresh: return self.terms if self.from_backup: self.terms = open_pickle(TERMS_BACKUP_PATH) + self.terms.name = "terms" return self.terms engine = create_engine(self.db_url) data = f""" - SELECT t.* + SELECT t.*, u.* FROM terms t + JOIN users u + ON t.orig_uid = u.guid WHERE t.status = '{status}' """ self.terms = pd.read_sql(data, engine) self.terms = self.fix_df_types(self.terms) + self.terms.name = "terms" create_pickle(self.terms, TERMS_BACKUP_PATH) return self.terms @@ -110,11 +150,12 @@ def get_annotations(self, status=0, withdrawn=0, refresh=False): return self.fetch_annotations() if self.from_backup: self.annotations = open_pickle(ANNOS_BACKUP_PATH) + self.annotations.name = "annotations" return self.annotations engine = create_engine(self.db_url) data = f""" SELECT - t1.ilx as term_ilx, t1.label as term_label, t1.type as term_type, + t1.ilx as term_ilx, t1.label as term_label, t1.type as term_type, t1.orig_cid as term_orig_cid, t2.ilx as annotation_type_ilx, t2.label as annotation_type_label, ta.* FROM term_annotations AS ta @@ -132,6 +173,7 @@ def get_annotations(self, status=0, withdrawn=0, refresh=False): """ self.annotations = pd.read_sql(data, engine) self.annotations = self.fix_df_types(self.annotations) + self.annotations.name = "annotations" create_pickle(self.annotations, ANNOS_BACKUP_PATH) return self.annotations @@ -140,10 +182,11 @@ def get_existing_ids(self, status=0, refresh=False): return self.existing_ids if self.from_backup: self.existing_ids = open_pickle(EXIDS_BACKUP_PATH) + self.existing_ids.name = "existing_ids" return self.existing_ids engine = create_engine(self.db_url) data = f""" - SELECT tei.*, t.ilx, t.label, t.type + SELECT tei.*, t.ilx, t.label, t.type, t.orig_cid FROM ( SELECT * FROM terms @@ -154,6 +197,7 @@ def get_existing_ids(self, status=0, refresh=False): """ self.existing_ids = pd.read_sql(data, engine) self.existing_ids = self.fix_df_types(self.existing_ids) + self.existing_ids.name = "existing_ids" create_pickle(self.existing_ids, EXIDS_BACKUP_PATH) return self.existing_ids @@ -162,12 +206,13 @@ def get_relationships(self, status=0, withdrawn=0, refresh=False): return self.relationships if self.from_backup: self.relationships = open_pickle(RELAS_BACKUP_PATH) + self.relationships.name = "relationships" return self.relationships engine = create_engine(self.db_url) data = f""" SELECT - t1.ilx AS term1_ilx, t1.type as term1_type, t1.label as term1_label, - t2.ilx AS term2_ilx, t2.type as term2_type, t2.label as term2_label, + t1.ilx AS term1_ilx, t1.type as term1_type, t1.label as term1_label, t1.orig_cid as term1_orig_cid, + t2.ilx AS term2_ilx, t2.type as term2_type, t2.label as term2_label, t2.orig_cid as term2_orig_cid, t3.ilx AS relationship_ilx, t3.label as relationship_label, tr.* FROM term_relationships AS tr @@ -190,6 +235,7 @@ def get_relationships(self, status=0, withdrawn=0, refresh=False): """ self.relationships = pd.read_sql(data, engine) self.relationships = self.fix_df_types(self.relationships) + self.relationships.name = "relationships" create_pickle(self.relationships, RELAS_BACKUP_PATH) return self.relationships @@ -198,6 +244,7 @@ def get_superclasses(self, status=0, withdrawn=0, refresh=False): return self.superclasses if self.from_backup: self.superclasses = open_pickle(SUPER_BACKUP_PATH) + self.superclasses.name = "superclasses" return self.superclasses engine = create_engine(self.db_url) data = f""" @@ -221,6 +268,7 @@ def get_superclasses(self, status=0, withdrawn=0, refresh=False): """ self.superclasses = pd.read_sql(data, engine) self.superclasses = self.fix_df_types(self.superclasses) + self.superclasses.name = "superclasses" create_pickle(self.superclasses, SUPER_BACKUP_PATH) return self.superclasses @@ -229,6 +277,7 @@ def get_synonyms(self, status=0, refresh=False): return self.synonyms if self.from_backup: self.synonyms = open_pickle(SYNOS_BACKUP_PATH) + self.synonyms.name = "synonyms" return self.synonyms engine = create_engine(self.db_url) data = f""" @@ -243,33 +292,34 @@ def get_synonyms(self, status=0, refresh=False): """ self.synonyms = pd.read_sql(data, engine) self.synonyms = self.fix_df_types(self.synonyms) + self.synonyms.name = "synonyms" create_pickle(self.synonyms, SYNOS_BACKUP_PATH) return self.synonyms - def get_terms_complete(self, status=0, withdrawn=0, refresh=False) -> pd.DataFrame: - ''' Gets complete entity data like term/view ''' - if not self.terms_complete.empty and not refresh: - return self.terms_complete - if self.from_backup: - self.terms_complete = open_pickle(TERMS_COMPLETE_BACKUP_PATH) - return self.terms_complete - ilx2synonyms = self.get_ilx2synonyms() - ilx2existing_ids = self.get_ilx2existing_ids() - ilx2annotations = self.get_ilx2annotations() - ilx2superclass = self.get_ilx2superclass() - ilx_complete = [] - header = ['Index'] + list(self.fetch_terms().columns) - for row in self.fetch_terms().itertuples(): - row = {header[i]: val for i, val in enumerate(row)} - row['synonyms'] = ilx2synonyms.get(row['ilx']) - # if breaks we have worse problems - row['existing_ids'] = ilx2existing_ids[row['ilx']] - row['annotations'] = ilx2annotations.get(row['ilx']) - row['superclass'] = ilx2superclass.get(row['ilx']) - ilx_complete.append(row) - terms_complete = pd.DataFrame(ilx_complete) - create_pickle(terms_complete, TERMS_COMPLETE_BACKUP_PATH) - return terms_complete + # def get_terms_complete(self, status=0, withdrawn=0, refresh=False) -> pd.DataFrame: + # ''' Gets complete entity data like term/view ''' + # if not self.terms_complete.empty and not refresh: + # return self.terms_complete + # if self.from_backup: + # self.terms_complete = open_pickle(TERMS_COMPLETE_BACKUP_PATH) + # return self.terms_complete + # ilx2synonyms = self.get_ilx2synonyms() + # ilx2existing_ids = self.get_ilx2existing_ids() + # ilx2annotations = self.get_ilx2annotations() + # ilx2superclass = self.get_ilx2superclass() + # ilx_complete = [] + # header = ['Index'] + list(self.fetch_terms().columns) + # for row in self.fetch_terms().itertuples(): + # row = {header[i]: val for i, val in enumerate(row)} + # row['synonyms'] = ilx2synonyms.get(row['ilx']) + # # if breaks we have worse problems + # row['existing_ids'] = ilx2existing_ids[row['ilx']] + # row['annotations'] = ilx2annotations.get(row['ilx']) + # row['superclass'] = ilx2superclass.get(row['ilx']) + # ilx_complete.append(row) + # terms_complete = pd.DataFrame(ilx_complete) + # create_pickle(terms_complete, TERMS_COMPLETE_BACKUP_PATH) + # return terms_complete def get_label2id(self): self.terms = self.fetch_terms() @@ -278,14 +328,14 @@ def get_label2id(self): for row in self.terms.itertuples(): label = self.local_degrade(row.label) if not visited.get((label, row.type, row.ilx)): - if row.type == 'term': - label_to_id[label]['term'].append(int(row.id)) + if row.type == "term": + label_to_id[label]["term"].append(int(row.id)) visited[(label, row.type, row.ilx)] = True - elif row.type == 'cde': - label_to_id[label]['cde'].append(int(row.id)) + elif row.type == "cde": + label_to_id[label]["cde"].append(int(row.id)) visited[(label, row.type, row.ilx)] = True - elif row.type == 'fde': - label_to_id[label]['fde'].append(int(row.id)) + elif row.type == "fde": + label_to_id[label]["fde"].append(int(row.id)) visited[(label, row.type, row.ilx)] = True return label_to_id @@ -304,172 +354,172 @@ def get_label2rows(self): self.terms_complete = self.fetch_terms_complete() visited = {} label2rows = defaultdict(list) - header = ['Index'] + list(self.terms_complete.columns) + header = ["Index"] + list(self.terms_complete.columns) for row in self.terms_complete.itertuples(): row = {header[i]: val for i, val in enumerate(row)} - label = self.local_degrade(row['label']) - if not visited.get((label, row['type'], row['ilx'])): + label = self.local_degrade(row["label"]) + if not visited.get((label, row["type"], row["ilx"])): label2rows[label].append(row) - visited[(label, row['type'], row['ilx'])] = True + visited[(label, row["type"], row["ilx"])] = True return label2rows def get_definition2rows(self): self.terms = self.fetch_terms() visited = {} definition2rows = defaultdict(list) - header = ['Index'] + list(self.terms.columns) + header = ["Index"] + list(self.terms.columns) for row in self.terms.itertuples(): row = {header[i]: val for i, val in enumerate(row)} - definition = self.local_degrade(row['definition']) - if not definition or definition == ' ': + definition = self.local_degrade(row["definition"]) + if not definition or definition == " ": continue - if not visited.get((definition, row['type'], row['ilx'])): + if not visited.get((definition, row["type"], row["ilx"])): definition2rows[definition].append(row) - visited[(definition, row['type'], row['ilx'])] = True + visited[(definition, row["type"], row["ilx"])] = True return definition2rows def get_tid2row(self): tid2row = {} - header = ['Index'] + list(self.fetch_terms().columns) + header = ["Index"] + list(self.fetch_terms().columns) for row in self.fetch_terms().itertuples(): row = {header[i]: val for i, val in enumerate(row)} - tid2row[row['tid']] = row + tid2row[row["tid"]] = row return tid2row def get_ilx2row(self): ilx2row = {} - header = ['Index'] + list(self.fetch_terms().columns) + header = ["Index"] + list(self.fetch_terms().columns) for row in self.fetch_terms().itertuples(): row = {header[i]: val for i, val in enumerate(row)} - ilx2row[row['ilx']] = row + ilx2row[row["ilx"]] = row return ilx2row def get_ilx2superclass(self, clean: bool = True): - ''' clean: for list of literals only ''' + """clean: for list of literals only""" ilx2superclass = defaultdict(list) - header = ['Index'] + list(self.fetch_superclasses().columns) + header = ["Index"] + list(self.fetch_superclasses().columns) for row in self.fetch_superclasses().itertuples(): row = {header[i]: val for i, val in enumerate(row)} if clean: superclass = { - 'tid': row['superclass_tid'], - 'ilx': row['superclass_ilx'], + "tid": row["superclass_tid"], + "ilx": row["superclass_ilx"], } - ilx2superclass[row['term_ilx']].append(superclass) + ilx2superclass[row["term_ilx"]].append(superclass) elif not clean: - ilx2superclass[row['term_ilx']].append(row) + ilx2superclass[row["term_ilx"]].append(row) return ilx2superclass def get_tid2annotations(self, clean: bool = True): - ''' clean: for list of literals only ''' + """clean: for list of literals only""" tid2annotations = defaultdict(list) - header = ['Index'] + list(self.fetch_annotations().columns) + header = ["Index"] + list(self.fetch_annotations().columns) for row in self.fetch_annotations().itertuples(): row = {header[i]: val for i, val in enumerate(row)} if clean: annotation = { - 'tid': row['tid'], - 'annotation_type_tid': row['annotation_type_tid'], - 'value': row['value'], - 'annotation_type_label': row['annotation_type_label'], + "tid": row["tid"], + "annotation_type_tid": row["annotation_type_tid"], + "value": row["value"], + "annotation_type_label": row["annotation_type_label"], } - tid2annotations[row['tid']].append(annotation) + tid2annotations[row["tid"]].append(annotation) elif not clean: - tid2annotations[row['tid']].append(row) + tid2annotations[row["tid"]].append(row) return tid2annotations def get_ilx2annotations(self, clean: bool = True): - ''' clean: for list of literals only ''' + """clean: for list of literals only""" ilx2annotations = defaultdict(list) - header = ['Index'] + list(self.fetch_annotations().columns) + header = ["Index"] + list(self.fetch_annotations().columns) for row in self.fetch_annotations().itertuples(): row = {header[i]: val for i, val in enumerate(row)} if clean: annotation = { - 'tid': row['tid'], - 'annotation_type_tid': row['annotation_type_tid'], - 'value': row['value'], - 'annotation_type_label': row['annotation_type_label'], + "tid": row["tid"], + "annotation_type_tid": row["annotation_type_tid"], + "value": row["value"], + "annotation_type_label": row["annotation_type_label"], } - ilx2annotations[row['term_ilx']].append(annotation) + ilx2annotations[row["term_ilx"]].append(annotation) elif not clean: - ilx2annotations[row['term_ilx']].append(row) + ilx2annotations[row["term_ilx"]].append(row) return ilx2annotations def get_tid2synonyms(self, clean: bool = True): - ''' clean: for list of literals only ''' + """clean: for list of literals only""" tid2synonyms = {} - header = ['Index'] + list(self.fetch_synonyms().columns) + header = ["Index"] + list(self.fetch_synonyms().columns) for row in self.fetch_synonyms().itertuples(): row = {header[i]: val for i, val in enumerate(row)} if clean: - synonym = {'literal': row['literal'], 'type': row['type']} - tid2synonyms[row['tid']].append(synonym) + synonym = {"literal": row["literal"], "type": row["type"]} + tid2synonyms[row["tid"]].append(synonym) elif not clean: - tid2synonyms[row['tid']].append(row) + tid2synonyms[row["tid"]].append(row) return tid2synonyms def get_ilx2synonyms(self, clean: bool = True): - ''' clean: for list of literals only ''' + """clean: for list of literals only""" ilx2synonyms = defaultdict(list) - header = ['Index'] + list(self.fetch_synonyms().columns) + header = ["Index"] + list(self.fetch_synonyms().columns) for row in self.fetch_synonyms().itertuples(): row = {header[i]: val for i, val in enumerate(row)} if clean: - synonym = {'literal': row['literal'], 'type': row['type']} - ilx2synonyms[row['ilx']].append(synonym) + synonym = {"literal": row["literal"], "type": row["type"]} + ilx2synonyms[row["ilx"]].append(synonym) elif not clean: - ilx2synonyms[row['ilx']].append(row) + ilx2synonyms[row["ilx"]].append(row) return ilx2synonyms def get_iri2row(self): iri2row = {} - header = ['Index'] + list(self.fetch_existing_ids().columns) + header = ["Index"] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): row = {header[i]: val for i, val in enumerate(row)} - iri2row[row['iri']] = row + iri2row[row["iri"]] = row return iri2row def get_tid2existing_ids(self, clean=True): tid2existing_ids = defaultdict(list) - header = ['Index'] + list(self.fetch_existing_ids().columns) + header = ["Index"] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): row = {header[i]: val for i, val in enumerate(row)} if clean: - existing_id = {'iri': row['iri'], 'curie': row['curie']} - tid2existing_ids[row['tid']].append(existing_id) + existing_id = {"iri": row["iri"], "curie": row["curie"]} + tid2existing_ids[row["tid"]].append(existing_id) elif not clean: - tid2existing_ids[row['tid']].append(row) + tid2existing_ids[row["tid"]].append(row) return tid2existing_ids def get_ilx2existing_ids(self, clean=True): ilx2existing_ids = defaultdict(list) - header = ['Index'] + list(self.fetch_existing_ids().columns) + header = ["Index"] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): row = {header[i]: val for i, val in enumerate(row)} if clean: - existing_id = {'iri': row['iri'], 'curie': row['curie']} - ilx2existing_ids[row['ilx']].append(existing_id) + existing_id = {"iri": row["iri"], "curie": row["curie"]} + ilx2existing_ids[row["ilx"]].append(existing_id) elif not clean: - ilx2existing_ids[row['ilx']].append(row) + ilx2existing_ids[row["ilx"]].append(row) return ilx2existing_ids def get_curie2row(self): curie2row = {} - header = ['Index'] + list(self.fetch_existing_ids().columns) + header = ["Index"] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): row = {header[i]: val for i, val in enumerate(row)} - curie2row[row['curie']] = row + curie2row[row["curie"]] = row return curie2row def get_fragment2rows(self): fragement2rows = defaultdict(list) - header = ['Index'] + list(self.fetch_existing_ids().columns) + header = ["Index"] + list(self.fetch_existing_ids().columns) for row in self.fetch_existing_ids().itertuples(): row = {header[i]: val for i, val in enumerate(row)} - if not row['curie']: # there are a few with no curies that will cause a false positive + if not row["curie"]: # there are a few with no curies that will cause a false positive continue - fragment = row['curie'].split(':')[-1] + fragment = row["curie"].split(":")[-1] fragement2rows[fragment].append(row) return fragement2rows @@ -482,7 +532,9 @@ def get_table(self, tablename, limit=5): SELECT * FROM {tablename} LIMIT {limit} - """.format(tablename=tablename, limit=limit) + """.format( + tablename=tablename, limit=limit + ) return pd.read_sql(data, self.engine) def get_custom(self, data): @@ -490,11 +542,11 @@ def get_custom(self, data): def main(): - db_url = os.environ.get('SCICRUNCH_DB_URL_PRODUCTION') + db_url = os.environ.get("SCICRUNCH_DB_URL_PRODUCTION") sql = IlxSql(db_url) rels = sql.get_relationships() print(rels.head()) -if __name__ == '__main__': +if __name__ == "__main__": main()