run.sh v0

Author: sirimeraoui

.gitignore

@@ -165,3 +165,6 @@ cython_debug/
 #developement
 notes.md
 debug.md
+tests/api_logs.json
+
+docker-compose.yml

README.md

@@ -1,59 +1,35 @@
-# PG_MFSERVER
-
+# MobilityAPI
 This server is a Python API that allows GET, POST, PUT, and DELETE operations on MobilityDB. The server utilizes the [PyMEOS](https://github.com/MobilityDB/PyMEOS) library.
-
 This implementation follows the OGC API - Moving Features Standard
 
 ## Introduction
-
 This Python API server provides endpoints for interacting with MobilityDB, a temporal extension for PostgreSQL. It allows users to perform CRUD operations (Create, Read, Update, Delete) on MobilityDB data using HTTP methods.
 
 ## Features
-
 - Supports GET, POST, PUT, and DELETE operations.
 - Integrates the PyMEOS library for seamless interaction with MobilityDB.
 - Provides endpoints for managing data stored in MobilityDB.
-## To test
-- create env from requirements.txt
-- docker pull --platform=linux/amd64 mobilitydb/mobilitydb
-- docker volume create mobilitydb_data
-- docker run --name mobilitydb -e POSTGRES_PASSWORD=mysecretpassword -p 25431:5432 -v mobilitydb_data:/var/lib/postgresql -d mobilitydb/mobilitydb
-- python server.py
-- pytest -v -s
+
 ## Prerequisites
 - Linux (ubuntu)
 - A recent version of Pyhton
-- A MobilityDB running locally or on a server
-
-## Installation
 
-To install and run the server, follow these steps:
 
-1. Download the server.py and utils.py file in the same folder.
-2. Dowload the rest-clients and change the queries to match your MOBILITYDB collections.
-3. Change the connection parameters in the server.py file.
-4. Install [PyMEOS](https://github.com/MobilityDB/PyMEOS) 
-    ```bash
-    python3 server.py
-6. Enjoy !
+## RUN SERVER
+- Make script executable: chmod +x run.sh
+- Run only the server: ./run.sh
+#### RUN SERVER WITH TESTS
+- Download ships datasets from: [Denmark Ships DataSets](http://aisdata.ais.dk/?prefix=2024/) aisdk_2024-08-07.zip in data folder
+- Run with integration tests: ./run.sh --with-tests (note: this takes a while due to data preprocessing - expect 23 min )
 
 ## Usage
-
 Send http requests to the api using any http service.
-
 As an example, your can use the ais.sql that will create ships and ship2 tables containing ships data.
 To do that you will have to change the path in the script to the path of your .csv file.
- 
 Here is a link to download ships datasets: [Denmark Ships DataSets](http://aisdata.ais.dk/?prefix=2024/)
-
 ## Developement
-
 This project is in progress.
-
 ## License
-
-
-
 ##Poetry
 poetry install
 

data/ais_to_json.py

@@ -7,21 +7,24 @@
 import geopandas as gpd
 import pandas as pd
 import numpy as np
-
 from utils import get_csv_from_zip
-# Load database configuration from config.json
-with open("../config.json", "r") as file:
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+output_file= os.path.join(script_dir, ".", "trajectories_mf1.json")
+config_path = os.path.join(script_dir, "..", "config.json")
+with open(config_path, "r") as file:
     config = json.load(file)
 
-# Construct the database URL for SQLAlchemy
+
 database_url = (
     f"postgresql://{config['DB_USER']}:{config['DB_PASS']}@"
     f"{config['DB_HOST']}:{config['DB_PORT']}/{config['DB_NAME']}"
 )
 
 # Create the SQLAlchemy engine
 engine = create_engine(database_url)
-data_csv_path = get_csv_from_zip("aisdk-2024-08-07.zip")
+zip_path = os.path.join(script_dir, "aisdk-2024-08-07.zip")
+data_csv_path = get_csv_from_zip(zip_path,script_dir)
 data_csv_path=os.path.abspath(data_csv_path)
 print(data_csv_path)
 #update this based on whether you're running your db from docker or local/ this is a docker version, otherwise comment it out
@@ -33,64 +36,52 @@
 
 # Data collection and 9.3 Cleaning Static Attribute +9.4 Voyage related Attribbutes:
 try:
-    # Read the SQL file CleaningStaticAttributes.sql
-    with open("init.sql", "r") as sql_file:
+    init_sql= os.path.join(script_dir, ".", "init.sql")
+    with open(init_sql, "r") as sql_file:
         sql_query = sql_file.read()
 
-    # Create a text object with parameters
     stmt = text(sql_query).bindparams(data_csv_path=data_csv_path)
 
-    # Execute with parameter
     with engine.connect() as conn:
-        # Start a transaction
+    
         with conn.begin():
-            # Execute the COPY command with parameter
             conn.execute(stmt)
             print(f"Successfully loaded data from {data_csv_path}")
 
 except Exception as e:
     print(f"Error loading data: {e}")
     raise
 
-
-# After your existing data loading code, add this:
-
-def export_json_from_view(engine, output_file="trajectories_mf1.json"):
+def export_json_from_view(engine, output_file=output_file):
     query = "SELECT json_data FROM ships_json ORDER BY mmsi;"
 
     try:
-        # Fetch all JSON data from the view
-               # for df in pd.read_sql(query, engine,chunksize=686):
-        
+    
         df = pd.read_sql(query, engine)
 
         if df.empty:
             print("No data found in ships_json view")
             return []
 
-        # Extract the json_data column as a list of dicts
+     
         json_ready = df['json_data'].tolist()
-        print('777777777777777777777777777777777777777777777')
-        # Ensure output directory exists
-        # os.makedirs(os.path.dirname(output_file), exist_ok=True)
-        
-        # Write to JSON file
+   
         with open(output_file, "w") as f:
             json.dump(json_ready, f, indent=2, default=str)
 
-        print(f"Successfully exported {len(json_ready)} vessels to {output_file}")
+        print(f"Successfully exported {len(json_ready)} ais vessels to {output_file}")
         return json_ready
 
     except Exception as e:
         print(f"Error exporting JSON: {e}")
         raise
 
-# Call the function after your data loading
+
 if __name__ == "__main__":
 
 
-    # Export to JSON
-    export_json_from_view(engine, "trajectories_mf1.json")
+    #to JSON
+    export_json_from_view(engine, output_file=output_file)
 
 
     # DROP VIEW IF EXISTS ships_json CASCADE;

data/init.sql

@@ -2,8 +2,8 @@
 -- ACTIVITY 2: VESSELS - Complete Processing Pipeline
 -- Based on Assignment 2 solutions from ULB
 -- ============================================================================
-
--- 1. Create the MobilityDB extension if not exists
+SET DATESTYLE = 'ISO, DMY';
+ -- 1. Create the MobilityDB extension if not exists
 CREATE EXTENSION IF NOT EXISTS MobilityDB CASCADE;
 
 -- 2. Create the raw AIS input table (based on PDF page 2)
@@ -198,7 +198,7 @@ FROM ShipsBelt;
 -- Envelope: ST_MakeEnvelope(644339, 6042108, 644896, 6042487, 25832)
 
 -- 14. Find ships traveling between Rodby and Puttgarden
-SELECT s.MMSI, s.Name, length(s.Trip)/1000 AS length_km
-FROM Ships s
-WHERE eintersects(s.Trip, ST_MakeEnvelope(651135, 6058230, 651422, 6058548, 25832))
-  AND eintersects(s.Trip, ST_MakeEnvelope(644339, 6042108, 644896, 6042487, 25832));
+-- SELECT s.MMSI, s.Name, length(s.Trip)/1000 AS length_km
+-- FROM Ships s
+-- WHERE eintersects(s.Trip, ST_MakeEnvelope(651135, 6058230, 651422, 6058548, 25832))
+--   AND eintersects(s.Trip, ST_MakeEnvelope(644339, 6042108, 644896, 6042487, 25832));