diff --git a/application_istsoslib.py b/application_istsoslib.py index 726ba459..3f038987 100755 --- a/application_istsoslib.py +++ b/application_istsoslib.py @@ -22,6 +22,7 @@ # ============================================================================= import sys +import os from os import path import traceback import waconf2sos @@ -57,11 +58,11 @@ def executeSos(environ, start_response): sys.path.insert(0, sosConfig.istsos_librarypath) pgdb = sosDatabase.PgDB( - sosConfig.connection["user"], - sosConfig.connection["password"], - sosConfig.connection["dbname"], - sosConfig.connection["host"], - sosConfig.connection["port"] + os.environ["GROUNDWATER_DATABASE_USER"], + os.environ["GROUNDWATER_DATABASE_PASSWORD"], + os.environ["GROUNDWATER_DATABASE"], + os.environ["GROUNDWATER_DATABASE_HOST"], + os.environ["GROUNDWATER_DATABASE_PORT"] ) req_filter = FF.sosFactoryFilter(environ, sosConfig) diff --git a/istsoslib/filters/factory_filters.py b/istsoslib/filters/factory_filters.py index 3f65695d..646985c3 100755 --- a/istsoslib/filters/factory_filters.py +++ b/istsoslib/filters/factory_filters.py @@ -29,6 +29,7 @@ __maintainer__ = 'Massimiliano Cannata, Milan Antonovic' __email__ = 'milan.antonovic@gmail.com' +from html import unescape from istsoslib import sosException import cgi from urllib.parse import parse_qs, unquote @@ -57,7 +58,7 @@ def sosFactoryFilter(environ, sosConfig): # > keep_blank_values used in version 2.0.0 to check # > null parameter exceptions rect = parse_qs( - environ['QUERY_STRING'], + unescape(environ['QUERY_STRING']), keep_blank_values=True ) requestObject = {} diff --git a/istsoslib/renderers/DSresponseRender.py b/istsoslib/renderers/DSresponseRender.py index 3aab8dcf..21676547 100755 --- a/istsoslib/renderers/DSresponseRender.py +++ b/istsoslib/renderers/DSresponseRender.py @@ -97,7 +97,10 @@ def render(DS,sosConfig): constraint = et.SubElement(time, "{%s}constraint" % ns['swe']) allowedTimes = et.SubElement(constraint, "{%s}AllowedTimes" % ns['swe']) interval = et.SubElement(allowedTimes, "{%s}interval" % ns['swe']) - interval.text = "%s %s" %(DS.stime.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), DS.etime.strftime("%Y-%m-%dT%H:%M:%S.%fZ")) + begin = et.SubElement(interval, "{%s}begin" % ns['swe']) + begin.text = DS.stime.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + end = et.SubElement(interval, "{%s}end" % ns['swe']) + end.text = DS.etime.strftime("%Y-%m-%dT%H:%M:%S.%fZ") if DS.procedureType=="insitu-mobile-point": # Adding 3d coordinates observation @@ -127,6 +130,12 @@ def render(DS,sosConfig): if not (field["name_uom"]=="" or field["name_uom"]==None or field["name_uom"]=="NULL"): uom = et.SubElement(quantity,"{%s}uom" % ns["swe"]) uom.attrib["code"] = field["name_uom"] + + interval = et.SubElement(quantity, "{%s}interval" % ns['swe']) + begin = et.SubElement(interval, "{%s}begin" % ns['swe']) + begin.text = field['stime_prc'].strftime("%Y-%m-%dT%H:%M:%S.%fZ") + end = et.SubElement(interval, "{%s}end" % ns['swe']) + end.text = field['etime_prc'].strftime("%Y-%m-%dT%H:%M:%S.%fZ") """ if not (field["desc_opr"]=="" or field["desc_opr"]==None or field["desc_opr"]=="NULL"): description = et.SubElement(quantity,"{%s}description" % ns["swe"]) @@ -252,6 +261,88 @@ def render(DS,sosConfig): root = tree.getroot() root.attrib["xmlns"]="http://www.opengis.net/sensorML/1.0.1" root.attrib["version"]="1.0.1" + + # TODO: + # Specifically for IGRAC + #--- Update well information ---# + for index, field in enumerate(DS.sensorProperties): + name = tree.find("{%s}member/{%s}System/{%s}name" % (ns['sml'], ns['sml'], ns['gml'])) + name.text = field['name'] + + photo = et.Element("{%s}photo" % ns["gml"]) + if field["photo"]: + photo.text = field["photo"] + name.addnext(photo) + + org = et.Element("{%s}OriginatingOrganization" % ns["gml"]) + if field["manager"]: + org.text = field["manager"] + photo.addnext(org) + + # Hydrogeology + hydrogeology = et.Element("{%s}Hydrogeology" % ns["gml"]) + aquifer = et.SubElement(hydrogeology, "{%s}Aquifer" % ns["gml"]) + aquifer_name = et.SubElement(aquifer, "{%s}AquiferName" % ns["gml"]) + if field["aquifer_name"]: + aquifer_name.text = field["aquifer_name"] + aquifer_material = et.SubElement(aquifer, "{%s}AquiferMaterial" % ns["gml"]) + if field["aquifer_material"]: + aquifer_material.text = field["aquifer_material"] + aquifer_type = et.SubElement(aquifer, "{%s}AquiferType" % ns["gml"]) + if field["aquifer_type"]: + aquifer_type.text = field["aquifer_type"] + aquifer_thickness = et.SubElement(aquifer, "{%s}AquiferThickness" % ns["gml"]) + if field["aquifer_thickness"]: + aquifer_thickness.text = field["aquifer_thickness"] + confinement = et.SubElement(aquifer, "{%s}Confinement" % ns["gml"]) + if field["confinement"]: + confinement.text = field["confinement"] + org.addnext(hydrogeology) + + id = tree.find("{%s}member/{%s}System/{%s}identification/{%s}IdentifierList/{%s}identifier/{%s}Term/{%s}value" % (ns['sml'], ns['sml'], ns['sml'], ns['sml'], ns['sml'], ns['sml'], ns['sml'])) + id.text = field['ggis_uid'] + point = tree.find("{%s}member/{%s}System/{%s}location/{%s}Point" % (ns['sml'], ns['sml'], ns['sml'], ns['gml'])) + point.attrib["{%s}id" % ns['gml']] = field['original_id'] + location = tree.find("{%s}member/{%s}System/{%s}location" % (ns['sml'], ns['sml'], ns['sml'])) + if field['country']: + country = et.SubElement(location, "{%s}country" % ns["gml"]) + country.text = f"{field['country']}" + + coordinates = tree.find("{%s}member/{%s}System/{%s}location/{%s}Point/{%s}coordinates" % (ns['sml'], ns['sml'], ns['sml'], ns['gml'], ns['gml'])) + # if field['elevation_value']: + # coordinates.text = f"{field['longitude']},{field['latitude']},{field['elevation_value']}" + # else: + # coordinates.text = f"{field['longitude']},{field['latitude']}" + # + # # TODO: WE COMMENT THIS TO ASK FIRST + latitude = et.SubElement(coordinates, "{%s}latitude" % ns["gml"]) + latitude.text = f"{field['latitude']}" + longitude = et.SubElement(coordinates, "{%s}longitude" % ns["gml"]) + longitude.text = f"{field['longitude']}" + elevation = et.SubElement(coordinates, "{%s}elevation" % ns["gml"]) + value = et.SubElement(elevation, "{%s}value" % ns["gml"]) + value.text = f"{field['elevation_value'] if field['elevation_value'] else ''}" + unit = et.SubElement(elevation, "{%s}unit" % ns["gml"]) + unit.text = f"{field['elevation_unit'] if field['elevation_unit'] else ''}" + + license = tree.find("{%s}member/{%s}System/{%s}License" % (ns['sml'], ns['sml'], ns['gml'])) + if 'license' in field and field['license']: + if 'summary' in field and field['summary']: + el = et.SubElement(license, "{%s}summary" % ns["gml"]) + el.text = f"{field['summary']}" + el = et.SubElement(license, "{%s}name" % ns["gml"]) + el.text = f"{field['license']}" + el = et.SubElement(license, "{%s}description" % ns["gml"]) + el.text = f"{field['license_desc']}" + + restriction = tree.find("{%s}member/{%s}System/{%s}Restriction" % (ns['sml'], ns['sml'], ns['gml'])) + if 'restriction_code_type_desc' in field and field['restriction_code_type_desc']: + el = et.SubElement(restriction, "{%s}description" % ns["gml"]) + el.text = f"{field['restriction_code_type_desc']}" + el = et.SubElement(restriction, "{%s}other" % ns["gml"]) + el.text = f"{field['constraints_other']}" + + return b'' + et.tostring(root) diff --git a/istsoslib/renderers/GCresponseRender.py b/istsoslib/renderers/GCresponseRender.py index 335babe0..ffbe66bc 100755 --- a/istsoslib/renderers/GCresponseRender.py +++ b/istsoslib/renderers/GCresponseRender.py @@ -23,6 +23,8 @@ import sys import isodate as iso +from ..utils import escape + def render(GC,sosConfig): r = ''' 0: for a in p.allowedValues: - r += " " + str(a) + "\n" + r += " " + escape(a) + "\n" if len(p.range)>0: r += " \n" r += " " diff --git a/istsoslib/renderers/GFresponseRender.py b/istsoslib/renderers/GFresponseRender.py index 0ad526cb..59502556 100755 --- a/istsoslib/renderers/GFresponseRender.py +++ b/istsoslib/renderers/GFresponseRender.py @@ -21,24 +21,28 @@ # # =============================================================================== import isodate as iso +from lxml import etree as et #import sosConfig def render(GF,sosConfig): - r = "\n" + r = "" if GF.type.lower()=="station" or GF.type.lower()=="point": r += "\n" - r += " " + GF.desc + "\n" + if GF.desc: + r += " " + GF.desc + "\n" r += " " + GF.name + " \n" r += " \n" @@ -98,11 +102,15 @@ def render(GF,sosConfig): r += " \n" r += " \n" - r += " " + GF.geom + "\n" + if GF.geom: + r += " " + GF.geom + "\n" r += " \n" if GF.type.lower()=="station" or GF.type.lower()=="point": r += " \n" elif GF.type=="surface": r += " \n" - - return r + else: + r += " \n" + tree = et.ElementTree(et.fromstring(r)) + root = tree.getroot() + return b'' + et.tostring(root) diff --git a/istsoslib/renderers/GOresponseRender.py b/istsoslib/renderers/GOresponseRender.py index 206d2ebb..c2a02d67 100755 --- a/istsoslib/renderers/GOresponseRender.py +++ b/istsoslib/renderers/GOresponseRender.py @@ -27,6 +27,8 @@ import sys import datetime +from ..utils import escape + date_handler = lambda obj: ( obj.isoformat() if isinstance(obj, (datetime.datetime, datetime.date)) @@ -103,7 +105,7 @@ def XMLformat(GO): r += " \n" #PROCEDURE - r += " \n" + r += " \n" #PROPRIETA OSSERVATA if ob.procedureType == "insitu-mobile-point": @@ -128,7 +130,7 @@ def XMLformat(GO): r += " \n" #FEATURE OF INTEREST - r += " \n" + r += " \n" r += " \n" r += " \n" r += " " + ob.foiGml + "\n" @@ -195,7 +197,13 @@ def XMLformat(GO): r += " \n" if ob.csv: - r += " %s" % ob.csv + values = '' + for row in ob.csv.split('@'): + val = row.split(',') + values += f' {val[1]}\n' + r += " \n" + r += values + r += " \n" elif len(ob.data) > 0: data=[] diff --git a/istsoslib/responders/DSresponse.py b/istsoslib/responders/DSresponse.py index cf26d0ae..877e25c5 100755 --- a/istsoslib/responders/DSresponse.py +++ b/istsoslib/responders/DSresponse.py @@ -25,7 +25,7 @@ import os.path import sys import importlib -from istsoslib import sosException +from istsoslib import sosException, sosDatabase class DescribeSensorResponse: @@ -44,55 +44,46 @@ class DescribeSensorResponse: """ def __init__(self, filter, pgdb): - pgdb.setTimeTZ("UTC") self.version = filter.version - self.smlFile = "" sql = "SELECT id_prc, stime_prc, etime_prc, name_oty from %s.procedures, %s.obs_type" %(filter.sosConfig.schema,filter.sosConfig.schema) - sql += " WHERE id_oty=id_oty_fk AND name_prc = %s" + sql += " WHERE id_oty=id_oty_fk AND name_prc = %s" params = (str(filter.procedure),) try: res=pgdb.select(sql,params) except: raise Exception("Error! sql: %s." %(pgdb.mogrify(sql,params)) ) - + # raise error if the procedure is not found in db if res==None: raise sosException.SOSException("InvalidParameterValue","procedure","Procedure '%s' not exist or can't be found.") - - # look for observation end time try: self.procedureType = res[0]['name_oty'] except: self.procedureType = None - - + if self.procedureType == 'virtual': vpFolder = os.path.join(filter.sosConfig.virtual_processes_folder,filter.procedure) try: sys.path.append(vpFolder) except: raise Exception("Error in loading virtual procedure path") - # check if python file exist if os.path.isfile("%s/%s.py" % (vpFolder,filter.procedure)): - #import procedure process vproc = importlib.import_module(filter.procedure) # exec("import %s as vproc" %(filter.procedure)) - + # Initialization of virtual procedure will load the source data vp = vproc.istvp() vp._configure(filter,pgdb) - + self.stime, self.etime = vp.getSampligTime() - else: self.stime = None self.etime = None - else: # look for observation start time try: @@ -100,40 +91,95 @@ def __init__(self, filter, pgdb): except: self.stime = None #raise sosException.SOSException(1,"Procedure '%s' has no valid stime."%(filter.procedure)) - + + # look for observation end time try: self.etime = res[0]['etime_prc'] except: self.etime = None - - + # check if folder containing SensorML exists if not os.path.isdir(filter.sosConfig.sensorMLpath): raise Exception("istsos configuration error, cannot find sensorMLpath!") - + # clean up the procedure name to produce a valid file name filename = filter.procedure - valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) - for c in filename: - if not c in valid_chars: - raise Exception("procedure name '%s' is not a valid: use only letters or digits!"%(filter.procedure)) filename += '.xml' - - self.smlFile = os.path.join(filter.sosConfig.sensorMLpath, filename) + + self.smlFile = os.path.join(filter.sosConfig.sensorMLpath, 'well.xml') # check if file exist if not os.path.isfile(self.smlFile): raise Exception("SensorML file for procedure '%s' not found!" % (filter.procedure)) - - sqlProc = "SELECT def_opr, name_opr, desc_opr, constr_pro, name_uom, id_pro" - sqlProc += " FROM %s.observed_properties opr, %s.proc_obs po," %(filter.sosConfig.schema,filter.sosConfig.schema) - sqlProc += " %s.procedures pr, %s.uoms um" %(filter.sosConfig.schema,filter.sosConfig.schema) - sqlProc += " WHERE opr.id_opr=po.id_opr_fk AND pr.id_prc=po.id_prc_fk AND um.id_uom = po.id_uom_fk" - sqlProc += " AND name_prc = %s ORDER BY id_pro" + + # TODO: + # IGRAC specified + sqlProc = "SELECT * FROM istsos.observed_properties_sensor WHERE name_prc = %s" params = (str(filter.procedure),) try: self.observedProperties = pgdb.select(sqlProc, params) except Exception as exe: raise Exception("Error! %s\n > sql: %s." % (str(exe), pgdb.mogrify(sqlProc, params))) - - + + # SPECIFICALLY FOR IGRAC SENSOR + fields = [ + 'longitude', 'photo', 'latitude', 'elevation_value', 'elevation_unit', + 'original_id', 'ggis_uid', 'name', 'id', 'country', 'license', + 'restriction_code_type', 'constraints_other', 'organisation', + 'manager', 'aquifer_name', 'aquifer_material', 'aquifer_type', + 'aquifer_thickness', 'confinement' + ] + + sqlProc = f"SELECT {','.join(fields)} from {filter.sosConfig.schema}.vw_istsos_sensor WHERE original_id='{filter.procedure}' LIMIT 1 " + + # Get the data of licenses + gdb = None + if os.environ.get("GEONODE_DATABASE", None): + gdb = sosDatabase.PgDB( + os.environ["GEONODE_DATABASE_USER"], + os.environ["GEONODE_DATABASE_PASSWORD"], + os.environ["GEONODE_DATABASE"], + os.environ["GEONODE_DATABASE_HOST"], + os.environ["GEONODE_DATABASE_PORT"] + ) + + try: + results = pgdb.select(sqlProc, params) + if not results: + raise Exception("Sensor does not exist.") + + self.sensorProperties = [] + for result in results: + row = {} + for idx, field in enumerate(fields): + row[field] = result[idx] + if gdb: + if field == 'license' and row[field]: + licenses = gdb.select( + f'SELECT name, description, abbreviation from base_license where id={row[field]}', + {} + ) + try: + license = licenses[0] + if result['organisation']: + row['summary'] = f'This data was made available by {result["organisation"]} under the {license[2]} license.' + row[field] = license[0] + row['license_desc'] = license[1] + except IndexError: + row[field] = '' + elif field == 'restriction_code_type' and row[field]: + codes = gdb.select( + f'SELECT identifier, description from base_restrictioncodetype where id={row[field]}', + {} + ) + try: + code = codes[0] + row[field] = code[0] + row['restriction_code_type_desc'] = code[1] + except IndexError: + row[field] = '' + print(row) + self.sensorProperties.append(row) + except Exception as exe: + raise Exception("Error! %s\n > sql: %s." % ( + str(exe), pgdb.mogrify(sqlProc, params))) diff --git a/istsoslib/responders/GCresponse.py b/istsoslib/responders/GCresponse.py index 94b53fad..723b5d6e 100755 --- a/istsoslib/responders/GCresponse.py +++ b/istsoslib/responders/GCresponse.py @@ -25,6 +25,7 @@ import sys import importlib from istsoslib.filters import DS_filter +from ..utils import escape class ServiceIdentification: """Service identification of the GetCapabilities responseFormat @@ -271,7 +272,8 @@ def BuildfeatureOfInterestList(pgdb,sosConfig): except: raise Exception("sql: %s" %(pgdb.mogrify(sql))) for row in rows: - list.append(sosConfig.urn["feature"] + row["nfoi"]) + if row["nfoi"]: + list.append(sosConfig.urn["feature"] + row["nfoi"]) return list def BuildOffEnvelope(pgdb,id,sosConfig): @@ -299,6 +301,11 @@ def BuildOffEnvelope(pgdb,id,sosConfig): #---------------- sql += " ) u" params=(id,id) + + # TODO: + # igrac specified + sql = "SELECT ST_asgml(Box2D(location)) as ext FROM mv_well" + params=None try: rows=pgdb.select(sql,params) except: @@ -368,7 +375,7 @@ def BuildOffProcList(pgdb,id,sosConfig): raise Exception("sql: %s" %(pgdb.mogrify(sql,params))) for row in rows: #list.append(sosConfig.urn["procedure"] + row["name_prc"]) - list.append(row["name_prc"]) + list.append(escape(row["name_prc"])) return list def BuildOffObsPrList(pgdb,id,sosConfig): @@ -423,7 +430,8 @@ def BuildOffFoiList(pgdb,id,sosConfig): except: raise Exception("sql: %s" %(pgdb.mogrify(sql,params))) for row in rows: - list.append(sosConfig.urn["feature"] + row["fois"]) + if row["fois"]: + list.append(sosConfig.urn["feature"] + escape(row["fois"])) return list def BuildSensorList(pgdb,sosConfig): @@ -843,10 +851,10 @@ class GetCapabilitiesResponse(): def __init__(self,fil,pgdb): self.version = fil.version - - + + if "all" in fil.sections: - + if self.version == '2.0.0': self.ObservationOfferingList = ObservationOfferingList_2_0_0(pgdb,fil.sosConfig) @@ -864,7 +872,7 @@ def __init__(self,fil,pgdb): self.FilterCapabilities = True else: - + if "contents" in fil.sections: if self.version == '2.0.0': self.ObservationOfferingList = ObservationOfferingList_2_0_0(pgdb,fil.sosConfig) diff --git a/istsoslib/responders/GFresponse.py b/istsoslib/responders/GFresponse.py index 8da5e215..5ba129b7 100755 --- a/istsoslib/responders/GFresponse.py +++ b/istsoslib/responders/GFresponse.py @@ -29,7 +29,8 @@ from istsoslib import sosDatabase #from SOS.config import mimetype from istsoslib import sosException - +from ..utils import escape + class foi: """The Feature of interest object @@ -57,16 +58,16 @@ def __init__(self,filter,pgdb): self.geom="" #select foi - sql = "SELECT id_foi, name_foi, desc_foi, ST_AsGml(ST_Transform(geom_foi,%s)) as geom, name_fty" #%(filter.srsName) + sql = f"SELECT id_foi, name_foi, desc_foi, ST_AsGml(ST_Transform(geom_foi,{filter.srsName})) as geom, name_fty" sql += " FROM %s.foi, %s.feature_type" %(filter.sosConfig.schema,filter.sosConfig.schema) sql += " WHERE id_fty_fk=id_fty AND name_foi=%s" #%(filter.featureOfInterest) - params = (filter.srsName,str(filter.featureOfInterest)) + params = (str(filter.featureOfInterest),) try: foi = pgdb.select(sql,params)[0] except: raise sosException.SOSException("InvalidParameterValue","FeatureOfInterestId","FeatureOfInterestId: Feature of Interest '%s' not found."%(filter.featureOfInterest)) - self.name=foi["name_foi"] + self.name=escape(foi["name_foi"]) self.desc=foi["desc_foi"] self.type=foi["name_fty"] self.geom=foi["geom"] @@ -75,7 +76,7 @@ def __init__(self,filter,pgdb): sql = "SELECT id_prc, name_prc, name_oty " sql += "FROM %s.procedures, %s.foi, %s.obs_type " %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema) sql += "WHERE id_foi_fk=id_foi AND id_oty=id_oty_fk AND name_foi=%s " #%(filter.featureOfInterest) - sql += "ORDER BY name_prc " + sql += "ORDER BY name_prc " params = (str(filter.featureOfInterest),) try: prc = pgdb.select(sql,params) @@ -83,14 +84,14 @@ def __init__(self,filter,pgdb): raise Exception("GFresponse, SQL: %s"%(pgdb.mogrify(sql,params))) for p in prc: - self.procedures.append(p["name_prc"]) + self.procedures.append(escape(p["name_prc"])) self.obsType.append(p["name_oty"]) self.idPrc.append(p["id_prc"]) # select obesrved properties of aa given procedure sql = "SELECT name_opr " sql += " FROM %s.procedures, %s.proc_obs, %s.observed_properties" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema) sql += " WHERE id_prc=id_prc_fk AND id_opr=id_opr_fk AND name_prc=%s" #%(p["name_prc"]) - sql += " ORDER BY name_opr" + sql += " ORDER BY name_opr" params = (p["name_prc"],) try: obs = pgdb.select(sql,params) @@ -112,7 +113,7 @@ def __init__(self,filter,pgdb): for st in samplTime: samplTimeArr.append([st['firstet'],st['lastet']]) self.samplingTime.append(samplTimeArr) - + diff --git a/istsoslib/responders/GOresponse.py b/istsoslib/responders/GOresponse.py index 379fcf0d..6c771dd2 100755 --- a/istsoslib/responders/GOresponse.py +++ b/istsoslib/responders/GOresponse.py @@ -36,7 +36,7 @@ from dateutil.parser import parse from istsoslib import sosException - +from ..utils import escape date_handler = lambda obj: ( obj.isoformat() @@ -215,7 +215,7 @@ def setSampligTime(self): raise Exception("Database error: %s - %s" % (sql, e)) self.samplingTime = (result[0], result[1]) - + def getProceduresFromOffering(self, offering): sql = """ SELECT id_prc, name_prc, st_z(geom_foi), array_agg(def_opr) @@ -252,7 +252,7 @@ def getProceduresFromOffering(self, offering): except Exception as e: raise Exception("Database error: %s - %s" % (sql, e)) - + def getData(self, procedure=None, disableAggregation=True): """Return the observations of associated procedure @@ -413,7 +413,7 @@ def observed_properties(self, offering): sql += """ WHERE id_foi = p3.id_foi_fk AND name_off = \'%s\' AND name_prc = \'%s\' """ % (offering, self.filter.procedure[0]) - + try: result = self.pgdb.select(sql) self.procedures = {} @@ -665,6 +665,10 @@ def BuildobservedPropertyList(pgdb,offering,sosConfig): sql += " %s.observed_properties, %s.off_proc o, %s.offerings" %(sosConfig.schema,sosConfig.schema,sosConfig.schema) sql += " WHERE id_opr_fk=id_opr AND p.id_prc_fk=id_prc AND o.id_prc_fk=id_prc AND id_off=id_off_fk" sql += " AND name_off='%s' ORDER BY p.id_pro" %(offering) + + # TODO: + # IGRAC SPECIALIZED + sql = f"select distinct(def_opr) as nopr from istsos.observed_properties" rows=pgdb.select(sql) for row in rows: list.append(row["nopr"]) @@ -699,13 +703,21 @@ def BuildProcedureList(pgdb,offering,sosConfig): return list + +def BuildProcedureCount(pgdb,offering, procedures, sosConfig): + sql = "SELECT name_prc FROM %s.procedures, %s.off_proc, %s.offerings" %(sosConfig.schema,sosConfig.schema,sosConfig.schema) + sql += f''' WHERE id_prc=id_prc_fk AND id_off=id_off_fk AND name_off='{offering}' AND name_prc IN ({','.join([f"'{procedure}'" for procedure in procedures])})''' + sql += " ORDER BY name_prc" + rows=pgdb.select(sql) + + return len(rows) + def BuildOfferingList(pgdb,sosConfig): list=[] sql = "SELECT distinct(name_off) FROM %s.offerings" %(sosConfig.schema,) rows=pgdb.select(sql) for row in rows: list.append(row["name_off"]) - return list @@ -916,7 +928,7 @@ def setData(self, pgdb, row, filter): sqlObsPro = "SELECT id_pro, id_opr, name_opr, def_opr, name_uom FROM %s.observed_properties, %s.proc_obs, %s.uoms" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema) sqlObsPro += " WHERE id_opr_fk=id_opr AND id_uom_fk=id_uom AND id_prc_fk=%s" %(row["id_prc"]) sqlObsPro += " AND (" - sqlObsPro += " OR ".join(["def_opr SIMILAR TO '%(:|)" + str(i) + "(:|)%'" for i in filter.observedProperty]) + sqlObsPro += " OR ".join(["def_opr='" + str(i) + "'" for i in filter.observedProperty]) sqlObsPro += " ) ORDER BY id_pro ASC" try: obspr_res = pgdb.select(sqlObsPro) @@ -976,21 +988,27 @@ def setData(self, pgdb, row, filter): valeFieldName = [] qi_field_name = [] + + multi_obs = len(obspr_res) > 1 + for idx, obspr_row in enumerate(obspr_res): + key = f'C{idx}' + if not multi_obs: + key = 'et' + if self.qualityIndex==True: cols += [ - "C%s.val_msr as c%s_v" % (idx, idx), - "COALESCE(C%s.id_qi_fk, %s) as c%s_qi" % ( - idx, + "%s.val_msr as %s_v" % (key, key), + "COALESCE(%s.id_qi_fk, %s) as %s_qi" % ( + key, filter.aggregate_nodata_qi, - idx + key ) ] csv_sql_cols += [ - #"C%s.val_msr" % idx, - "COALESCE(C%s.val_msr, %s)" % (idx, filter.aggregate_nodata), - "COALESCE(C%s.id_qi_fk, %s)" % (idx, filter.aggregate_nodata_qi) + "COALESCE(%s.val_msr, %s)" % (key, filter.aggregate_nodata), + "COALESCE(%s.id_qi_fk, %s)" % (key, filter.aggregate_nodata_qi) ] valeFieldName.append("c%s_v" %(idx)) @@ -998,139 +1016,97 @@ def setData(self, pgdb, row, filter): qi_field_name.append("C%s.id_qi_fk" %(idx)) else: - cols.append("C%s.val_msr as c%s_v" %(idx,idx)) - csv_sql_cols.append("C%s.val_msr" %(idx)) - valeFieldName.append("c%s_v" %(idx)) + cols.append("%s.val_msr as c%s_v" %(key,idx)) + csv_sql_cols.append("%s.val_msr" %(key)) + valeFieldName.append("%s_v" %(key)) # If Aggregatation funtion is set if filter.aggregate_interval != None: # This accept only numeric results aggrCols.append( - "COALESCE(%s(nullif(dt.c%s_v, 'NaN')),'%s')" % ( + "COALESCE(%s(nullif(dt.%s_v, 'NaN')),'%s')" % ( filter.aggregate_function, - idx, + key, filter.aggregate_nodata ) ) csv_aggr_cols.append( - "COALESCE(%s(nullif(dt.c%s_v, 'NaN')),'%s')" % ( + "COALESCE(%s(nullif(dt.%s_v, 'NaN')),'%s')" % ( filter.aggregate_function, - idx, + key, filter.aggregate_nodata ) ) if self.qualityIndex==True: - aggrCols.append("COALESCE(MIN(dt.c%s_qi),%s) as c%s_qi\n" % ( - idx, + aggrCols.append("COALESCE(MIN(dt.%s_qi),%s) as %s_qi\n" % ( + key, filter.aggregate_nodata_qi, - idx + key )) - csv_aggr_cols.append("COALESCE(MIN(dt.c%s_qi),%s)" % ( - idx, + csv_aggr_cols.append("COALESCE(MIN(dt.%s_qi),%s)" % ( + key, filter.aggregate_nodata_qi )) - aggrNotNull.append(" c%s_v > -900 " %(idx)) - - # Set SQL JOINS - join_txt = """ - LEFT JOIN ( - SELECT - A%s.id_msr, - A%s.val_msr, - A%s.id_eti_fk - """ % (idx, idx, idx) - - if self.qualityIndex==True: - join_txt += ", A%s.id_qi_fk\n" %(idx) + aggrNotNull.append(" %s_v > -900 " %(key)) - join_txt += """ - FROM - %s.measures A%s - WHERE - A%s.id_pro_fk = %s - """ % ( - filter.sosConfig.schema, idx, - idx, obspr_row["id_pro"] - ) + if multi_obs: + # Set SQL JOINS + join_txt = """ + JOIN ( + SELECT + A%s.id_msr, + A%s.val_msr, + A%s.id_eti_fk + """ % (idx, idx, idx) - # ATTENTION: HERE -999 VALUES ARE EXCLUDED WHEN ASKING AN AGGREAGATE FUNCTION - if filter.aggregate_interval != None: # >> Should be removed because measures data is not inserted if there is a nodata value - join_txt += " AND A%s.val_msr > -900 " % idx + if self.qualityIndex==True: + join_txt += ", A%s.id_qi_fk\n" %(idx) + + join_txt += """ + FROM + %s.measures A%s + WHERE + A%s.id_pro_fk = '%s' + """ % ( + filter.sosConfig.schema, idx, + idx, obspr_row["id_pro"] + ) - # close SQL JOINS - join_txt += " ) as C%s\n" %(idx) - join_txt += " on C%s.id_eti_fk = et.id_eti" %(idx) - joinar.append(join_txt) + # ATTENTION: HERE -999 VALUES ARE EXCLUDED WHEN ASKING AN AGGREAGATE FUNCTION + if filter.aggregate_interval != None: # >> Should be removed because measures data is not inserted if there is a nodata value + join_txt += " AND A%s.val_msr > -900 " % idx - # If MOBILE PROCEDURE - if self.procedureType=="insitu-mobile-point": - join_txt = """ - LEFT JOIN ( - SELECT - Ax.id_pos, - st_X(ST_Transform(Ax.geom_pos,%s)) as x, - st_Y(ST_Transform(Ax.geom_pos,%s)) as y, - st_Z(ST_Transform(Ax.geom_pos,%s)) as z, - Ax.id_eti_fk - """ %( - filter.srsName, - filter.srsName, - filter.srsName - ) - - if self.qualityIndex==True: - join_txt += ", Ax.id_qi_fk as posqi\n" - - join_txt += """ - FROM - %s.positions Ax, - %s.event_time Bx - WHERE - Ax.id_eti_fk = Bx.id_eti - AND - Bx.id_prc_fk = %s - """ %( - filter.sosConfig.schema, - filter.sosConfig.schema, - row["id_prc"] - ) + # close SQL JOINS + join_txt += " ) as C%s\n" %(idx) + join_txt += " on C%s.id_eti_fk = et.id_eti" %(idx) + joinar.append(join_txt) - join_txt += " ) as Cx on Cx.id_eti_fk = et.id_eti\n" - cols.extend([ - "Cx.x as x", - "Cx.y as y", - "Cx.z as z" - ]) - csv_sql_cols.extend([ - "Cx.x", - "Cx.y", - "Cx.z" - ]) - if self.qualityIndex==True: - cols.append("Cx.posqi") - csv_sql_cols.append("Cx.posqi") - - joinar.append(join_txt) # Set FROM CLAUSE - sqlSel += "%s FROM %s.event_time et" % ( - ", ".join(cols), filter.sosConfig.schema + table = f'{filter.sosConfig.schema}.event_time' + if not multi_obs: + table = 'istsos.measures' + + sqlSel += "%s FROM %s et" % ( + ", ".join(cols), table ) # Set FROM CLAUSE - csv_sql_sel += "%s FROM %s.event_time et" % ( + csv_sql_sel += "%s FROM %s et" % ( ( ",".join(cols) if filter.aggregate_interval != None else " || ',' || ".join(csv_sql_cols) ), - filter.sosConfig.schema + table ) # Set WHERE CLAUSES sqlData = " ".join(joinar) sqlData += " WHERE et.id_prc_fk=%s\n" %(row["id_prc"]) + if not multi_obs: + sqlData += f""" AND et.id_pro_fk = '{obspr_row["id_pro"]}'""" # Set FILTER ON RESULT (OGC:COMPARISON) if filter.result: @@ -1155,9 +1131,10 @@ def setData(self, pgdb, row, filter): sqlData += " OR ".join(etf) sqlData += ")\n" - else: - # Get last observed measuement - sqlData += " AND et.time_eti = (SELECT max(time_eti) FROM %s.event_time WHERE id_prc_fk=%s) " %(filter.sosConfig.schema,row["id_prc"]) + # TODO: IGRAC Custom + # else: + # # Get last observed measuement + # sqlData += " AND et.time_eti = (SELECT max(time_eti) FROM %s.event_time WHERE id_prc_fk=%s) " %(filter.sosConfig.schema,row["id_prc"]) # Quality index filtering if ( @@ -1338,9 +1315,9 @@ def setData(self, pgdb, row, filter): try: a = datetime.datetime.now() - self.data = pgdb.select(sql) if 'text/plain' == filter.responseFormat: + self.data = pgdb.select(sql) self.csv = pgdb.to_string(csv_sql) elif filter.responseFormat in [ @@ -1348,6 +1325,9 @@ def setData(self, pgdb, row, filter): "text/xml" ]: self.csv = pgdb.to_string(csv_sql, lineterminator='@') + self.data = [] + else: + self.data = pgdb.select(sql) except Exception as xx: print(traceback.print_exc(), file=sys.stderr) @@ -1416,10 +1396,10 @@ def __init__(self, filter, pgdb): raise sosException.SOSException("InvalidParameterValue","offering","Parameter \"offering\" sent with invalid value: %s - available options for offering are %s" %(filter.offering,off_list)) """ if filter.procedure: - pl = BuildProcedureList(pgdb, filter.offering, filter.sosConfig) - for p in filter.procedure: - if not p in pl: - raise sosException.SOSException("InvalidParameterValue","procedure","Parameter \"procedure\" sent with invalid value: %s - available options for offering \"%s\": %s"%(p,filter.offering,pl)) + # IGRAC SPECIFIED + pl = BuildProcedureCount(pgdb, filter.offering, filter.procedure, filter.sosConfig) + if not pl: + raise sosException.SOSException("InvalidParameterValue","procedure","Parameter \"procedure\" sent with invalid value") if filter.featureOfInterest: fl = BuildfeatureOfInterestList(pgdb,filter.offering, filter.sosConfig) @@ -1431,7 +1411,7 @@ def __init__(self, filter, pgdb): opr_sel = "SELECT def_opr FROM %s.observed_properties WHERE " %(filter.sosConfig.schema,) opr_sel_w = [] for op in filter.observedProperty: - opr_sel_w += ["def_opr SIMILAR TO '%%(:|)%s(:|)%%'" %(op)] + opr_sel_w += ["def_opr = '%s'" %(op)] opr_sel = opr_sel + " OR ".join(opr_sel_w) try: @@ -1439,7 +1419,6 @@ def __init__(self, filter, pgdb): except: raise Exception("SQL: %s"%(opr_sel)) - if not len(opr_filtered)>0: raise sosException.SOSException("InvalidParameterValue","observedProperty","Parameter \"observedProperty\" sent with invalid value: %s - available options: %s"%(filter.observedProperty,opl)) @@ -1472,30 +1451,17 @@ def __init__(self, filter, pgdb): pgdb.setTimeTZ("UTC") - # BUILD PROCEDURES LIST # select part of query - sqlSel = "SELECT DISTINCT" - sqlSel += " id_prc, name_prc, name_oty, stime_prc, etime_prc, time_res_prc" + sqlSel = "SELECT DISTINCT id_prc, name_prc, concat('insitu-fixed-point','') as name_oty, stime_prc, etime_prc, null as time_res_prc" # from part of query - sqlFrom = "FROM %s.procedures, %s.proc_obs p, %s.observed_properties, %s.uoms," %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema) - sqlFrom += " %s.off_proc o, %s.offerings, %s.obs_type" %(filter.sosConfig.schema,filter.sosConfig.schema,filter.sosConfig.schema) - if filter.featureOfInterest or filter.featureOfInterestSpatial: - sqlFrom += " ,%s.foi, %s.feature_type" %(filter.sosConfig.schema,filter.sosConfig.schema) - - sqlWhere = "WHERE id_prc=p.id_prc_fk AND id_opr_fk=id_opr AND o.id_prc_fk=id_prc AND id_off_fk=id_off AND id_uom=id_uom_fk AND id_oty=id_oty_fk" - sqlWhere += " AND name_off='%s'" %(filter.offering) - - # where condition based on featureOfInterest - if filter.featureOfInterest: - sqlWhere += " AND id_foi=id_foi_fk AND id_fty=id_fty_fk AND (name_foi IN (%s))" %(",".join( [ "'"+f+"'" for f in filter.featureOfInterest.split(",")])) - if filter.featureOfInterestSpatial: - sqlWhere += " AND id_foi_fk=id_foi AND %s" %(filter.featureOfInterestSpatial) + sqlFrom = "FROM istsos.observed_properties_sensor" + sqlWhere = "WHERE " # where condition based on procedures if filter.procedure: - sqlWhere += " AND (" + sqlWhere += " (" procWhere = [] for proc in filter.procedure: procWhere.append("name_prc='%s'" %(proc)) @@ -1503,7 +1469,9 @@ def __init__(self, filter, pgdb): sqlWhere += ")" # where condition based on observed properties - sqlWhere += " AND (" + if sqlWhere != 'WHERE': + sqlWhere += " AND " + sqlWhere += " (" obsprWhere = [] for obs in opr_filtered: obsprWhere.append("def_opr='%s'" %(obs["def_opr"])) diff --git a/istsoslib/responders/factory_response.py b/istsoslib/responders/factory_response.py index 2df62958..4d1a29ad 100755 --- a/istsoslib/responders/factory_response.py +++ b/istsoslib/responders/factory_response.py @@ -24,7 +24,12 @@ from istsoslib import sosException def sosFactoryResponse(sosFilter, pgdb): - + if sosFilter.version != '1.0.0': + return sosException.SOSException( + "InvalidRequest", "request", + "\"version\": %s not supported" % (sosFilter.version) + ) + if sosFilter.request == "getcapabilities": from istsoslib.responders import GCresponse return GCresponse.GetCapabilitiesResponse(sosFilter, pgdb) @@ -35,28 +40,25 @@ def sosFactoryResponse(sosFilter, pgdb): elif sosFilter.request == "getobservation": from istsoslib.responders import GOresponse - if sosFilter.version == '2.0.0': - return GOresponse.GetObservationResponse_2_0_0(sosFilter, pgdb) - - else: - return GOresponse.GetObservationResponse(sosFilter, pgdb) - + return GOresponse.GetObservationResponse(sosFilter, pgdb) + elif sosFilter.request == "getfeatureofinterest": from istsoslib.responders import GFresponse return GFresponse.foi(sosFilter, pgdb) - - elif sosFilter.request == "insertobservation": - from istsoslib.responders import IOresponse - return IOresponse.InsertObservationResponse(sosFilter, pgdb) - - elif sosFilter.request == "registersensor": - from istsoslib.responders import RSresponse - return RSresponse.RegisterSensorResponse(sosFilter, pgdb) - - elif sosFilter.request == "updatesensordescription": - from istsoslib.responders import USDresponse - return USDresponse.UpdateSensorDescription(sosFilter, pgdb) - + + # TODO: IGRAC DISABLES THIS + # elif sosFilter.request == "insertobservation": + # from istsoslib.responders import IOresponse + # return IOresponse.InsertObservationResponse(sosFilter, pgdb) + # + # elif sosFilter.request == "registersensor": + # from istsoslib.responders import RSresponse + # return RSresponse.RegisterSensorResponse(sosFilter, pgdb) + # + # elif sosFilter.request == "updatesensordescription": + # from istsoslib.responders import USDresponse + # return USDresponse.UpdateSensorDescription(sosFilter, pgdb) + else: raise sosException.SOSException("InvalidRequest", "request", "\"request\": %s not supported" % (sosFilter.request)) diff --git a/istsoslib/utils.py b/istsoslib/utils.py new file mode 100644 index 00000000..e0f433c2 --- /dev/null +++ b/istsoslib/utils.py @@ -0,0 +1,14 @@ +def escape(s, quote=True): + """ + Replace special characters "&", "<" and ">" to HTML-safe sequences. + If the optional flag quote is true (the default), the quotation mark + characters, both double quote (") and single quote (') characters are also + translated. + """ + s = s.replace("&", "&") # Must be done first! + s = s.replace("<", "<") + s = s.replace(">", ">") + if quote: + s = s.replace('"', """) + s = s.replace('\'', "'") + return s