Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions Importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
from pathlib import Path
import threading
import subprocess as sp
import platform
import os
import ntpath

home_dir = str(Path.home())
server_started = False
Expand Down Expand Up @@ -110,6 +113,8 @@ class AbstractImporter(object):
def __init__(self, tombolo_path, print_data=False):
global gateway
self._tombolo_path = tombolo_path
if platform.system() == 'Windows':
self._tombolo_path = self._tombolo_path.replace(os.sep, ntpath.sep)
self._print_data = print_data
self._data = None
self.start_server()
Expand Down Expand Up @@ -229,16 +234,18 @@ def run(self):
global server_started
if not server_started:
jars_for_classpath = self.class_path_files()
args = ['java', '-cp',
home_dir + self._tombolo_path + 'build/classes/java/main:' +
':'.join(jars_for_classpath), 'uk.org.tombolo.Py4jServer']
p = sp.Popen(args, cwd=home_dir + self._tombolo_path)
build_path = os.path.join('build', 'classes', 'java', 'main')
class_path = os.path.join(home_dir, self._tombolo_path, build_path)
args = ['java', '-cp', class_path + ':' + ':'.join(jars_for_classpath),
'uk.org.tombolo.Py4jServer']
p = sp.Popen(args, cwd=os.path.join(home_dir, self._tombolo_path))
server_started = True

def class_path_files(self):
import os, os.path
dirs = []
for directory_path, _, file_names in os.walk(home_dir + "/.gradle/caches/modules-2/files-2.1"):
to_walk = os.path.join(home_dir, '.gradle', 'caches', 'modules-2', 'files-2.1')
for directory_path, _, file_names in os.walk(to_walk):
for file_name in [f_names for f_names in file_names if f_names.endswith(".jar")]:
dirs.append(os.path.join(directory_path, file_name))
return dirs
Expand Down
2 changes: 1 addition & 1 deletion Importers/importer_london_air_quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
data = Utils.download_data(data_url=data_url, suffix='json')

from Importer import Provider, SubjectType, AbstractImporter, home_dir, Attribute, Subject, Geometry, FixedValue, TimedValue
importer = AbstractImporter(tombolo_path='/Desktop/UptodateProject/TomboloDigitalConnector/')
importer = AbstractImporter(tombolo_path='Desktop/UptodateProject/TomboloDigitalConnector/')

# Creating Provider
provider = Provider(label='erg.kcl.ac.uk', name='Environmental Research Group Kings College London')
Expand Down
20 changes: 15 additions & 5 deletions Utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import urllib.request as url
import hashlib
from pathlib import Path
import os
import ntpath
import platform


class Utils(object):
Expand All @@ -9,14 +12,21 @@ def __init__(self):

@classmethod
def download_data(self, data_url, suffix, data_cache_directory='/tmp'):
if Path(data_url).is_file():
print('Reading from Local Data Store', data_url)
with open(data_url, 'r') as data:
return data.read()
if platform.system() == 'Windows':
data_url = data_url.replace(os.sep, ntpath.sep)

try:
if Path(data_url).is_file():
print('Reading from Local Data Store', data_url)
with open(data_url, 'r') as data:
return data.read()
except OSError:
print('Not a system path, downloading data...')

_data = None
encode_url = hashlib.md5(data_url.encode())
local_dataset = Path(data_cache_directory + '/TomboloData/' + encode_url.hexdigest() + '.' + suffix)
path = os.path.join(data_cache_directory, 'TomboloData', encode_url.hexdigest() + '.' + suffix)
local_dataset = Path(path)
if local_dataset.is_file():
print('Reading from Local Data Store', local_dataset)
with open(local_dataset, 'r') as data:
Expand Down