[pull] master from biothings:master

.github/workflows/deploy.yml

@@ -1,4 +1,4 @@
-name: ci-cd
+name: deploy to production
 
 on:
   push:
@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x]
+        node-version: [14.x]
 
     steps:
     - uses: actions/checkout@v2
@@ -33,12 +33,12 @@ jobs:
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
-      - 
+      -
         name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
 
-      - 
+      -
         name: Cache Docker layers
         uses: actions/cache@v2
         with:
@@ -52,7 +52,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-      - 
+      -
         name: Get the version
         id: vars
         run: echo ::set-output name=tag::$(echo ${GITHUB_REF:10})
@@ -70,7 +70,7 @@ jobs:
           builder: ${{ steps.buildx.outputs.name }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
-      - 
+      -
         name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
 
@@ -87,7 +87,7 @@ jobs:
         key: ${{ secrets.SSH_KEY }}
         port: ${{ secrets.SSH_PORT }}
         script: |
-          cd opt/single-hop-app.js
+          cd ~/opt/BioThings_Explorer_TRAPI
           git stash && git pull
           rm docker-compose.yml
           cp docker-compose-prod.yml docker-compose.yml
@@ -99,7 +99,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [12.x]
+        node-version: [14.x]
 
     steps:
     - uses: actions/checkout@v2
@@ -122,5 +122,5 @@ jobs:
         key: ${{ secrets.SSH_KEY }}
         port: ${{ secrets.SSH_PORT }}
         source: "performance-test/report.html"
-        target: "opt/single-hop-app.js/performance-test"
+        target: "~/opt/BioThings_Explorer_TRAPI/performance-test"
         strip_components: 1

.github/workflows/development-deploy-revert.yml

@@ -1,10 +1,10 @@
 name: deploy to dev server with git ref
 
-on: 
+on:
   workflow_dispatch:
     inputs:
       git-ref:
-        description: Git Ref (Optional)    
+        description: Git Ref (Optional)
         required: false
     branches:
         - development
@@ -22,7 +22,13 @@ jobs:
         key: ${{ secrets.SSH_KEY_DEV }}
         port: ${{ secrets.SSH_PORT }}
         script: |
-          cd opt/BioThings_Explorer_TRAPI
+          cd ~/opt/BioThings_Explorer_TRAPI
+          git fetch
           git reset ${{ github.event.inputs.git-ref }}
-          npm i --production
-          pm2 restart all
+          cp docker-compose-dev.yml docker-compose.yml
+          docker-compose up --force-recreate --build -d
+          docker image prune -f
+
+          # alternative way to start the app:
+          # npm i --production
+          # pm2 restart all

.github/workflows/development-deploy.yml

@@ -13,7 +13,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [13.x]
+        node-version: [14.x]
 
     steps:
     - uses: actions/checkout@v2
@@ -37,7 +37,7 @@ jobs:
         key: ${{ secrets.SSH_KEY_DEV }}
         port: ${{ secrets.SSH_PORT }}
         script: |
-          cd opt/BioThings_Explorer_TRAPI
+          cd ~/opt/BioThings_Explorer_TRAPI
           rm docker-compose.yml
           git stash
           git pull
@@ -50,7 +50,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [12.x]
+        node-version: [14.x]
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/test.yml

@@ -3,7 +3,7 @@
 
 on:
   push:
-    branches: 
+    branches:
       - "**"
   pull_request:
     branches: [ master ]
@@ -19,10 +19,10 @@ jobs:
 
     - uses: actions/checkout@v1
 
-    - name: Use Node.js 12.x
+    - name: Use Node.js 14.x
       uses: actions/setup-node@v1
       with:
-        node-version: 12.x
+        node-version: 14.x
 
     - name: npm install, generate coverage report
       run: |

.gitignore

@@ -115,3 +115,5 @@ processed*.txt
 report*.json
 
 .DS_Store
+
+built

.jest/setEnvVars.js

@@ -0,0 +1 @@
+process.env.MAX_QUERIES_PER_MIN = 20;

CHANGELOG.md

@@ -2,6 +2,75 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [2.8.1](https://github.com/biothings/BioThings_Explorer_TRAPI/compare/v2.8.0...v2.8.1) (2021-06-05)
+
+
+### Features
+
+* :sparks: update to consume trapi v1.1 apis ([5b2f465](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/5b2f465483b7e817a0d75721ea16e6655b4bd215))
+* transform graph data from v1.1 to legacy format, uncomment code ([b1d2413](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/b1d24135c7831041a17eda277bfb6264959d957a))
+
+
+### Bug Fixes
+
+* add details to meta graph error ([1d36a18](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/1d36a18694e4ca291afda8d25ddc878e33bf1b16))
+
+## [2.8.0](https://github.com/biothings/BioThings_Explorer_TRAPI/compare/v2.7.0...v2.8.0) (2021-05-25)
+
+
+### Features
+
+* :sparkles: added meta_knowledge_graph endpoints by team and ip ([8882df4](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/8882df487e7348918e437fa6f971fadd7b24cb3f))
+
+
+### Bug Fixes
+
+* :bug: fix meta_kg endpoint for a given API ([85ff733](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/85ff733bbaefec91c55d36b9bf62bc4bc16259de))
+
+## [2.7.0](https://github.com/biothings/BioThings_Explorer_TRAPI/compare/v2.6.0...v2.7.0) (2021-05-16)
+
+
+### Features
+
+* :sparkles: add handler logics for creating v1 metakg ([c1c5011](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/c1c50116dcec2c131c36c8c3ec52502f49e65d2d))
+* :sparkles: add metakg routes to index ([7c7b8e1](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/7c7b8e1fed0f195c8ab5854b8a49d0cf9b71ae96))
+* :sparkles: create route for metakg ([cd379bc](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/cd379bc8f16c491090c2982f5c0fb5fbbc4dda3d))
+
+## [2.6.0](https://github.com/biothings/BioThings_Explorer_TRAPI/compare/v2.5.0...v2.6.0) (2021-05-11)
+
+
+### Features
+
+* :sparkles: upgrade handler to support trapi 1.1.0 ([3cec3f2](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/3cec3f2868864cd4a6501ba441ac4927b8034250))
+* trapi v1.1.0 support, update handler ([3109a8a](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/3109a8a14071198ae038c5001f0d1bd76c566ad2))
+
+
+### Bug Fixes
+
+* :bug: change from NCBIGENE to NCBIGene ([88d6035](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/88d6035d9c9e76451be5514409f3a416da062c55))
+
+## [2.5.0](https://github.com/biothings/BioThings_Explorer_TRAPI/compare/v2.4.0...v2.5.0) (2021-04-23)
+
+
+### Bug Fixes
+
+* :bug: fix issue when output ids contain special characters ([898483a](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/898483a89d678e7fc4c95dd43ab607e3bbc4ae46))
+
+## [2.4.0](https://github.com/biothings/BioThings_Explorer_TRAPI/compare/v2.3.0...v2.4.0) (2021-04-23)
+
+
+### Features
+
+* :sparkles: allow user to set MAX_QUERIES_PER_MIN as env ([8686092](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/8686092a8cbeedacbc38f8670a7be6470eeafc9d))
+* :sparkles: set rate limit to max 3 queries per min ([aeea89c](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/aeea89c867eace66ba485b1c8b146af2a4328912))
+
+
+### Bug Fixes
+
+* :bug: fix unable to resolve biocarta id issue ([551e4b1](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/551e4b1f938eb08054f303ec6d2460f558e32d08))
+* :fire: temporarily disable text mining co-occurrence kp ([d214a4d](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/d214a4dbec36a60d6df7d150f89835a6ea32bac8))
+* :sparkles: re-enable clinical risk kp ([4464dfc](https://github.com/biothings/BioThings_Explorer_TRAPI/commits/4464dfcffcb470ee7478cd16b2d93d712367ebd7))
+
 ## [2.3.0](https://github.com/biothings/BioThings_Explorer_TRAPI/compare/v2.2.0...v2.3.0) (2021-04-20)
 
 

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# How to Contribute
+
+See the [main README](https://github.com/biothings/BioThings_Explorer_TRAPI#biothings-explorer-trapi-api). This file just has additional details.
+
+## Set up Dev Environment
+
+### Small Updates
+
+If you're just updating one package and your changes won't affect other packages, you don't have to worry about any of this. If your changes will affect multiple packages but primarily affect just one, you may be able to get away with just using [`npm link`](https://docs.npmjs.com/cli/v7/commands/npm-link).
+
+### Larger Updates
+
+If your updates will affect multiple packages, you may find it helpful to use [NPM workspaces](https://docs.npmjs.com/cli/v7/using-npm/workspaces). This feature is better than just `npm link` for handling the case of simultaneously updating multiple linked packages. Note it requires NPM v7+. Steps to get this working:
+
+1. Create a new directory and `cd` into it
+2. Add a top-level package.json with all the packages and their dependencies hoisted up to the top level ([something like this](https://www.dropbox.com/s/izsofa7r5alwwfd/package.json?dl=0) - notice the `workspaces` key)
+3. For each package to be affected by your updates, clone its repo into subdirectory `./packages` ([demo bash script](https://www.dropbox.com/s/upggzaby7b978z8/clone_packages.sh?dl=0) to do this).
+4. From the top-level directory, run the following:
+
+- `npm install`
+- `npm run build --workspaces`
+- `npm run test --workspaces`
+
+5. To run an NPM script for a specific package, try `npm run <script-name> --workspace=package-a` from the top-level directory, e.g., to start the BioThings_Explorer_TRAPI endpoint: `npm run start --workspace='@biothings-explorer/single-hop-app'`.
+6. From the directories in `./packages`, you can edit code and run git commands, e.g., `cd './packages/@biothings-explorer/single-hop-app'` and `git pull`.
+7. For now, you still need to run the build step for each package affected by your updates: `npm run build --workspaces` from the top-level directory. If it doesn't work immediately, try running it again. (We can automate this in the future.)

README.md

@@ -5,44 +5,29 @@
 
 ## Introduction
 
-This GitHub repo serves as the development repo for the TRAPI API implementation of BioThings Explorer.
+This GitHub repo serves as the development repo for the TRAPI API implementation of **BioThings Explorer (BTE)**. BTE is an engine for autonomously querying a distributed knowledge graph. The distributed knowledge graph is made up of biomedical APIs that have been annotated with semantically-precise descriptions of their inputs and outputs in the [SmartAPI registry](https://smart-api.info/). This project is primarily funded by the [NCATS Translator project](https://ncats.nih.gov/translator).  There is also an older [python version of BioThings Explorer](https://github.com/biothings/biothings_explorer) that is currently not being actively developed.
 
-### What's BioThings Explorer
+An older version of the meta knowledge graph that is consumed by BTE is in this figure (which, although older, gives a nice conceptual visualization of API interoperability):
 
-BioThings Explorer aims at helping users querying and linking results from a variety of biomedical relevant APIs through one interface. The project is funded by the [NCATS Translator project](https://ncats.nih.gov/translator).
+![BTE Meta-KG](diagrams/smartapi_metagraph.png "BioThings Explorer metagraph")
 
-### What's TRAPI
+### What's TRAPI?
 
-TRAPI stands for [Translator Reasoner API](https://github.com/NCATSTranslator/ReasonerAPI). It is a standard defined for APIs developed within NCATS Biomedical Translator project to allow easy information exchange among them.
+TRAPI stands for [Translator Reasoner API](https://github.com/NCATSTranslator/ReasonerAPI). It is a standard defined for APIs developed within NCATS Biomedical Translator project to facilitate information exchange between resources.  BTE exports results via TRAPI to maintain interoperability with other Translator tools.  BTE can also _consume_ knowledge resources that expose the TRAPI interface, but it also can consume APIs that have been annotated in the [SmartAPI registry](https://smart-api.info/) using the [x-bte extension](https://x-bte-extension.readthedocs.io/en/latest/index.html) to the OpenAPI specification.
 
-### Our TRAPI Interface
+### Live TRAPI Instance
 
-Please visit our TRAPI API landing page [here](http://bte_trapi.smart-api.info/).
+We maintain a live instance of this application at https://api.bte.ncats.io/ that can be used for testing.  Query Examples can be found [here](/examples).
 
 
-### Some Query Examples
-
-Query Examples can be found [here](https://github.com/kevinxin90/BioThings_Explorer_TRAPI/tree/master/examples).
-
----
-
-
-## Current Development Status
-
-Currently, the BioThings Explorer TRAPI API implements TRAPI v1.0.0 standard, and can be queried at https://api.bte.ncats.io/v1/query endpoint.
-
-
-The TRAPI v0.9.2 standard is deprecated. But it can still be tested using https://api.bte.ncats.io/query endpoint.
-
 ---
 
-## Requirements
 
-For development, you will only need Node.js and a node global package, e.g. npm, installed in your environment.
+## Local installations
 
-### Node
+### Requirements
 
-NOTE: Node version must be higher than v12.
+For development, you will only need Node.js and a node global package, e.g. npm, installed in your environment. Your Node version must be higher than v12.
 
 
 - #### Node installation on Windows
@@ -75,14 +60,14 @@ If you need to update `npm`, you can make it using `npm`! Cool right? After runn
 
 ---
 
-## Install
+### Installation
 
-    $ git clone https://github.com/biothings/BioThings_Explorer_TRAPI`
-    $ cd BioThings_Explorer_TRAPI`
-    $ npm install`
+    $ git clone https://github.com/biothings/BioThings_Explorer_TRAPI
+    $ cd BioThings_Explorer_TRAPI
+    $ npm install
 
 
-## Running the project
+### Running the project
 
     $ npm start
 
@@ -92,11 +77,15 @@ To enable debug mode, which outputs logging statements to the terminal in real t
 
 `$ DEBUG=biothings-explorer-trapi:*,smartapi-kg:*,call-apis:*,biomedical-id-resolver:* npm start` (also outputs debug statements from dependencies)
 
-## Simple build for production
+By default, the `/v1/query` endpoint only supports 3 queries per min, you could modify this behavior by setting MAX_QUERIES_PER_MIN environment variable when starting the service
+
+`$ MAX_QUERIES_PER_MIN=5 npm start`
+
+### Simple build for production
 
     $ npm build
 
-## Deploy
+### Deploy
 
 A docker file is included in the base directory and can be used to build the customized container
 
@@ -112,11 +101,14 @@ docker-compose up
 
 Public Docker image located at [link](https://hub.docker.com/repository/docker/biothings/bte_reasoner_api)
 
-## Usage
+### Usage
+
+You now can POST queries to `http://<HOST>:3000/v1/query`.  
 
-`http://<HOST>:3000`
+Query Examples can be found [here](/examples).
 
-## Test with local SmartAPI spec
+
+### Test with local SmartAPI spec
 
 The TRAPI interface has `/test/query` endpoint which uses a SmartAPI spec stored at **test** folder named **smartapi.json**
 
@@ -134,3 +126,8 @@ Then, Run the image and mount your local smartapi spec folder
 Now, you should be able to test your local smartapi using POST queries at:
 
 `http://localhost:3000/test/query`
+
+### Testing on a specific SmartAPI API
+
+By default, BTE queries all APIs specified in the the[ config.js file](https://github.com/biothings/BioThings_Explorer_TRAPI/blob/master/src/routes/v1/config.js).  In some cases, you may want to override that default to specifically query a single API.  For example, the SmartAPI record for the "EBI Proteins API) is [43af91b3d7cae43591083bff9d75c6dd](https://smart-api.info/registry?q=43af91b3d7cae43591083bff9d75c6dd). To instruct BTE to query that API only, you can POST your query to http://localhost:3000/v1/smartapi/43af91b3d7cae43591083bff9d75c6dd/query
+