Gcore Rest Api/ sHINER Tab

config/gcore.env.example

@@ -0,0 +1 @@
+GCORE_ENDPOINT=http://diascld32.iccluster.epfl.ch:6080/

docker-compose.yml

@@ -3,6 +3,7 @@ version: '3.7'
 networks:
   sdl_net:
    name: sdl_net
+   external: true
 
 services:
   #######################
@@ -20,8 +21,9 @@ services:
     command: ["sh", "-c", "npm install && npm rebuild node-sass && npm run-script start"]
     depends_on:
       - visual-analytics-engine
-    networks:
-      - sdl_net
+    network_mode: host
+    #networks:
+    #  - sdl_net
 
   ###############################
   # SDL Visual Analytics Engine #
@@ -33,7 +35,8 @@ services:
       context: ./visual-analytics-engine/
       dockerfile: ./Dockerfile
     ports:
-      - "3001:8080"
+    - "3001:8080"
+    # - "3001:3001"
     volumes:
       - ./visual-analytics-engine/app:/usr/src/app
       - ./visual-analytics-engine/data/input:/data/input
@@ -42,12 +45,15 @@ services:
       - ./config/postgres.env
       - ./config/proteus.env
       - ./config/qal.env
+      - ./config/gcore.env
     working_dir: /usr/src/app
+    #command: ["python", "app.py", "--port", "3001", "--dev"]
     command: ["python", "app.py", "--port", "8080", "--dev"]
     depends_on:
       - redis
-    networks:
-      - sdl_net
+    network_mode: host
+    #networks:
+    #  - sdl_net
 
   #########
   # Redis #
@@ -59,8 +65,9 @@ services:
     hostname: redis
     volumes:
       - redis-data:/data
-    networks:
-      - sdl_net
+    network_mode: host
+    #networks:
+    #  - sdl_net
 
   ###############
   # JUPYTER LAB #
@@ -79,8 +86,9 @@ services:
       - ./config/proteus.env
     working_dir: /home/jovyan
     command: ["sh", "-c", "cd /home/jovyan/notebooks && jupyter-lab --port=3002"]
-    networks:
-      - sdl_net
+    network_mode: host
+    #networks:
+    #  - sdl_net
 
   ###########
   # V-Plots #
@@ -94,9 +102,10 @@ services:
       - ./v-plots/app:/usr/src/app
     working_dir: /usr/src/app
     command: ["sh", "-c", "npm install && npx bower --allow-root install && npx grunt serve"]
-    networks:
-      - sdl_net
-
+    network_mode: host
+    #networks:
+    # - sdl_net
+
   ####################
   # Postgres Test DB #
   ####################
@@ -110,12 +119,13 @@ services:
       - ./config/postgres.env
     ports:
       - "3004:5432"
-    networks:
-      - sdl_net
+    network_mode: host
+    #networks:
+    #  - sdl_net
 
 
 volumes:
   vae-data:
     external: false
   redis-data:
-    external: false
+    external: false

visual-analytics-engine/app/app.py

@@ -17,6 +17,7 @@
 from tools.data_transformer import transform
 from tools.simsearch_manager import SimSearchManager
 from tools.timeseries_manager import TimeSeriesManager
+from tools.gcore_manager import GCoreManager
 
 warnings.filterwarnings("ignore", category=UserWarning)
 
@@ -36,6 +37,7 @@ def create_paths():
 PROTEUS_PASSWORD = os.environ["PROTEUS_PASSWORD"]
 
 QAL_ENDPOINT = os.environ["QAL_ENDPOINT"]
+GCORE_ENDPOINT = os.environ["GCORE_ENDPOINT"]
 
 
 def make_flask_app() -> Flask:
@@ -66,6 +68,7 @@ def make_cache_key(*args, **kwargs):
     proteus_manager = ProteusManager(PROTEUS_URL, PROTEUS_USER, PROTEUS_PASSWORD)
     qal_manager = QALManager(QAL_ENDPOINT)
     timeseries_manager = TimeSeriesManager()
+    gcore_manager = GCoreManager(GCORE_ENDPOINT)
 
     with app.app_context():
         graph = nx.read_gpickle("/data/input/graph/graph.gpickle")
@@ -88,6 +91,79 @@ def index():
     def graph_route():
         return clustered_graph["hierarchical"]
 
+    ############graph g-core entity resolution - routes###############
+
+    #graph schema
+    @app.route("/gcore/schema/<string:graph_name>")  # ok
+    def get_schema(graph_name):
+        return gcore_manager.graph_schema(graph_name)
+
+    @app.route("/gcore/availableGraphs")
+    def get_graphdb():
+        return gcore_manager.get_graphs()
+
+    @app.route("/gcore/er/setggds", methods=['POST'])
+    def set_ggds():
+        args = request.json
+        return gcore_manager.set_ggds(args)
+
+    @app.route("/gcore/er/getggds")
+    def get_ggds():
+        return gcore_manager.get_ggds()
+
+    @app.route("/gcore/er/run")
+    def run_er():
+        return gcore_manager.run_ER()
+
+    @app.route("/gcore/query/select", methods=['POST'])
+    def select_query():
+        args = request.json
+        query = args["query"]
+        limit = args["limit"]
+        print("here select panel" + query)
+        return gcore_manager.selectQuery(query, limit)
+
+    @app.route("/gcore/query/construct", methods=['POST'])
+    def construct_query():
+        args = request.json
+        query = args["query"]
+        limit = args["limit"]
+        print("here construct panel" + query)
+        return gcore_manager.constructQuery(query, limit)
+
+    #args for both select and graph neighbor
+    # json format for "passing node information"
+    # {
+    #        "nodeLabel": "ProductAmazon",
+    #        "id": "1",
+    #        "edgeLabel": "",
+    #        "graphName": "Amazon",
+    #        "limit": -1
+    #    }
+    @app.route("/gcore/query/select-neighbor", methods=['POST'])
+    def select_neighbor():
+        args = request.json
+        return gcore_manager.getNeighbors(args)
+
+    @app.route("/gcore/query/graph-neighbor", methods=['POST'])
+    def graph_neighbor():
+        args = request.json
+        return gcore_manager.getNeighborsGraph(args)
+
+    ###############graph visualization#############
+
+    @app.route("/graphvis/initialvis", methods=['POST'])
+    def graph_initialvis():
+        args = request.json
+        return gcore_manager.startVisualization(args)
+
+    @app.route("/graphvis/nextlevel", methods=['POST'])
+    def graph_nextlevel():
+        args = request.json
+        return gcore_manager.nextLevelGraph(args)
+
+    ##############################
+
     # Renamed from /tables to /schema to avoid confusion
     @app.route('/schema', methods=['POST'])
     @cache.cached(timeout=432000, key_prefix=make_cache_key)

visual-analytics-engine/app/tools/gcore_manager.py

@@ -0,0 +1,111 @@
+import requests
+import json
+import pandas as pd
+import networkx as nx
+
+
+class GCoreManager:
+    _service_url = None
+
+    def __init__(self, gcore_endpoint):
+        self._service_url = gcore_endpoint
+        r = requests.get(self._service_url + 'graphDB')
+        print(r.content)
+
+    def get_graphs(self):
+        print("GCore service url::" + self._service_url)
+        r = requests.get(self._service_url + 'graphDB')
+        print(r.content)
+        return r.content
+
+    def graph_schema(self, name=None):
+        urlGCore = self._service_url + 'graphDB/' + name
+        r = requests.get(url=urlGCore)
+        return r.content
+
+    def set_ggds(self, ggds):
+        r = requests.post(url=self._service_url + 'ggds/setGGDs', json=ggds)
+        print("Here answer to content:" + r.content)
+        return r.content
+
+    def get_ggds(self):
+        r = requests.get(url=self._service_url+'ggds/getGGDs')
+        return r.content
+
+    def run_ER(self):
+        r = requests.get(url=self._service_url+'ggds/runER')
+        print(r.content)
+        return r.content
+
+    def selectQuery(self, query, limit):
+        #json = {"key": "value"}
+        r = requests.post(url=self._service_url + 'gcore/select', json={"query": query, "limit": limit})
+        #r = requests.post(url=service_url + 'gcore/select', data=query)
+        #tableData = pd.DataFrame.from_dict(json.loads(r['content']))
+        tableData = r.content
+        return tableData
+
+    def constructQuery(self, query, limit):
+        print(query)
+        r = requests.post(url=self._service_url + 'gcore/construct', json={"query": query, "limit": limit})
+        #r = requests.post(url=service_url + "gcore/construct", data=query)
+        jsonobj = json.loads(r.content)
+        print(r.content)
+        graph = r.content#nx.readwrite.json_graph.node_link_graph(r.content)
+        return graph
+
+    def selectQuery_table(self, tableName, graphName=None):
+        if graphName is None:
+            query = "SELECT * MATCH (a:" + tableName + ")"
+        else:
+            query = "SELECT * MATCH (a:" + tableName + ") ON " + graphName
+        r = requests.post(url=self._service_url+'gcore/select', data=query)
+        tableData = pd.DataFrame.from_dict(json.loads(r['content']))
+        return tableData
+
+    def constructQuery_graph(self, resPattern, matchPattern, graphName=None):
+        if graphName is None:
+            query = "CONSTRUCT " + resPattern + " MATCH " + matchPattern
+        else:
+            query = "CONSTRUCT " + resPattern + " MATCH " + matchPattern + " ON " + graphName
+        r = requests.post(url=self._service_url+"gcore/construct", data=query)
+        graph = nx.readwrite.json_graph.node_link_graph(r.content)
+        return graph
+
+#example json file
+#    with open("data/graph.txt") as json_file:
+#        jsonParam = json.load(json_file)
+#    print(jsonParam)
+#    r = requests.post(url='http://localhost:8080/gcore/construct-neighbor', json=jsonParam)
+
+#json format for "passing node information"
+#{
+#        "nodeLabel": "ProductAmazon",
+#        "id": "1",
+#        "edgeLabel": "",
+#        "graphName": "Amazon",
+#        "limit": -1
+#    }
+    def getNeighbors(self, nodeParam):
+        r = requests.post(url=self._service_url+"gcore/select-neighbor",json=nodeParam)
+        neighborData = pd.DataFrame.from_dict(json.loads(r['content']))
+        return neighborData
+
+    def getNeighborsGraph(self, nodeParam):
+        r = requests.post(url=self._service_url+"gcore/construct-neighbor",json=nodeParam)
+        graphNeighbor = nx.readwrite.json_graph.node_link_graph(r.content)
+        return graphNeighbor
+
+    def startVisualization(self, param):
+        print(param)
+        r = requests.post(url=self._service_url+"graphvis/initialvis", json=param)
+        print(r.content)
+        return r.content#jsonify(data=r.content)
+        #initialGraph = nx.readwrite.json_graph.node_link_graph(r.content)
+        #return initialGraph
+
+    def nextLevelGraph(self, param):
+        print(param)
+        r = requests.post(url=self._service_url+"graphvis/nextlevel", json=param)
+        print(r.content)
+        return r.content#jsonify(data=r.content)