From 98c4631d3dec0f86d4c2bbecce06a71b5a8d2da3 Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 15 Nov 2018 18:14:48 +0200 Subject: [PATCH 01/22] show logged user in home template --- app/modules/home.html | 1 + app/modules/home.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/modules/home.html b/app/modules/home.html index b8554a0..7f6d4c7 100644 --- a/app/modules/home.html +++ b/app/modules/home.html @@ -9,5 +9,6 @@

Home

  • Datastore example (load)
  • Yaml example
    • +

    Loged in as : {{user_name}}

    diff --git a/app/modules/home.py b/app/modules/home.py index 8850ae3..9521d49 100644 --- a/app/modules/home.py +++ b/app/modules/home.py @@ -2,9 +2,11 @@ import logging import os from google.appengine.ext.webapp import template +from google.appengine.api import users class HomeHandler(webapp2.RequestHandler): def get(self): logging.info("HomeHandler get()") + user = users.get_current_user() template_path = os.path.join(os.path.dirname(__file__), 'home.html') - self.response.write(template.render(template_path, {})) + self.response.write(template.render(template_path, {'user_name' : user.nickname()})) From 7087c1b7255f37514d5fdf4d6ca4926183bd425e Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 15 Nov 2018 18:42:36 +0200 Subject: [PATCH 02/22] miss spelling corrected --- app/modules/home.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/modules/home.html b/app/modules/home.html index 7f6d4c7..da8027f 100644 --- a/app/modules/home.html +++ b/app/modules/home.html @@ -9,6 +9,6 @@

    Home

  • Datastore example (load)
  • Yaml example
    • -

    Loged in as : {{user_name}}

    +

    Logged in as : {{user_name}}

    From bb196187da339a73311ad5b6d24eb9fd15345206 Mon Sep 17 00:00:00 2001 From: Najipian Date: Sun, 18 Nov 2018 14:25:54 +0200 Subject: [PATCH 03/22] twice square ta=raining task --- .datastore/.gitkeep | 0 .env.dist | 0 .gitignore | 1 + README.md | 0 TASKS.md | 0 app-deploy.yaml | 0 app.yaml | 0 app/__init__.py | 0 app/handlers.py | 2 ++ app/modules/__init__.py | 0 app/modules/common/__init__.py | 1 + app/modules/common/squares.py | 23 ++++++++++++++++++ app/modules/examples/__init__.py | 0 app/modules/examples/datastore_example.html | 0 app/modules/examples/datastore_example.py | 0 app/modules/examples/http_example.html | 0 app/modules/examples/http_example.py | 0 app/modules/examples/pipeline_example.html | 0 app/modules/examples/pipeline_example.py | 0 app/modules/examples/yaml_example.html | 0 app/modules/examples/yaml_example.py | 0 app/modules/examples/yaml_example.yaml | 0 app/modules/home.html | 1 + app/modules/home.py | 0 app/modules/training/__init__.py | 1 + app/modules/training/pipeline_training.py | 27 +++++++++++++++++++++ app/modules/training/pipeline_traning.html | 8 ++++++ docker-compose.yml | 0 docker/Dockerfile | 0 env.yaml.dist | 0 index.yaml | 0 libs/.gitkeep | 0 main.py | 0 requirements.txt | 0 34 files changed, 64 insertions(+) mode change 100644 => 100755 .datastore/.gitkeep mode change 100644 => 100755 .env.dist mode change 100644 => 100755 .gitignore mode change 100644 => 100755 README.md mode change 100644 => 100755 TASKS.md mode change 100644 => 100755 app-deploy.yaml mode change 100644 => 100755 app.yaml mode change 100644 => 100755 app/__init__.py mode change 100644 => 100755 app/handlers.py mode change 100644 => 100755 app/modules/__init__.py create mode 100755 app/modules/common/__init__.py create mode 100755 app/modules/common/squares.py mode change 100644 => 100755 app/modules/examples/__init__.py mode change 100644 => 100755 app/modules/examples/datastore_example.html mode change 100644 => 100755 app/modules/examples/datastore_example.py mode change 100644 => 100755 app/modules/examples/http_example.html mode change 100644 => 100755 app/modules/examples/http_example.py mode change 100644 => 100755 app/modules/examples/pipeline_example.html mode change 100644 => 100755 app/modules/examples/pipeline_example.py mode change 100644 => 100755 app/modules/examples/yaml_example.html mode change 100644 => 100755 app/modules/examples/yaml_example.py mode change 100644 => 100755 app/modules/examples/yaml_example.yaml mode change 100644 => 100755 app/modules/home.html mode change 100644 => 100755 app/modules/home.py create mode 100755 app/modules/training/__init__.py create mode 100755 app/modules/training/pipeline_training.py create mode 100755 app/modules/training/pipeline_traning.html mode change 100644 => 100755 docker-compose.yml mode change 100644 => 100755 docker/Dockerfile mode change 100644 => 100755 env.yaml.dist mode change 100644 => 100755 index.yaml mode change 100644 => 100755 libs/.gitkeep mode change 100644 => 100755 main.py mode change 100644 => 100755 requirements.txt diff --git a/.datastore/.gitkeep b/.datastore/.gitkeep old mode 100644 new mode 100755 diff --git a/.env.dist b/.env.dist old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 6cc9a81..dda0959 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Environment vars .env env.yaml +.vscode # Google App Engine files in Docker .datastore/* diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/TASKS.md b/TASKS.md old mode 100644 new mode 100755 diff --git a/app-deploy.yaml b/app-deploy.yaml old mode 100644 new mode 100755 diff --git a/app.yaml b/app.yaml old mode 100644 new mode 100755 diff --git a/app/__init__.py b/app/__init__.py old mode 100644 new mode 100755 diff --git a/app/handlers.py b/app/handlers.py old mode 100644 new mode 100755 index 6c15b7e..82f33ed --- a/app/handlers.py +++ b/app/handlers.py @@ -3,6 +3,7 @@ from app.modules.examples.http_example import HttpExampleHandler from app.modules.examples.datastore_example import DatastoreExampleHandler from app.modules.examples.yaml_example import YamlExampleHandler +from app.modules.training.pipeline_training import PipelineTrainigHandler def handlers(): return [ @@ -15,4 +16,5 @@ def handlers(): (r'/examples/datastore/([^\/]+)', DatastoreExampleHandler), (r'/examples/datastore/([^\/]+)/([^\/]+)', DatastoreExampleHandler), (r'/examples/yaml', YamlExampleHandler), + (r'/training/pipeline/(\d+)', PipelineTrainigHandler), ] diff --git a/app/modules/__init__.py b/app/modules/__init__.py old mode 100644 new mode 100755 diff --git a/app/modules/common/__init__.py b/app/modules/common/__init__.py new file mode 100755 index 0000000..4957e95 --- /dev/null +++ b/app/modules/common/__init__.py @@ -0,0 +1 @@ +# __init__.py needed to create "package" for this directory diff --git a/app/modules/common/squares.py b/app/modules/common/squares.py new file mode 100755 index 0000000..6f8e441 --- /dev/null +++ b/app/modules/common/squares.py @@ -0,0 +1,23 @@ +import webapp2 +import logging +import pipeline + +class SquarePipeline(pipeline.Pipeline): + + def run(self, number): + return number * number + + +class TwiceSquaredPipeline(pipeline.Pipeline): + + def run(self, number): + + first_square = yield SquarePipeline(number) + second_square = yield SquarePipeline(first_square) + yield LogResult(second_square) + + +class LogResult(pipeline.Pipeline): + + def run(self, number): + logging.info('All done! Value is %s', number) \ No newline at end of file diff --git a/app/modules/examples/__init__.py b/app/modules/examples/__init__.py old mode 100644 new mode 100755 diff --git a/app/modules/examples/datastore_example.html b/app/modules/examples/datastore_example.html old mode 100644 new mode 100755 diff --git a/app/modules/examples/datastore_example.py b/app/modules/examples/datastore_example.py old mode 100644 new mode 100755 diff --git a/app/modules/examples/http_example.html b/app/modules/examples/http_example.html old mode 100644 new mode 100755 diff --git a/app/modules/examples/http_example.py b/app/modules/examples/http_example.py old mode 100644 new mode 100755 diff --git a/app/modules/examples/pipeline_example.html b/app/modules/examples/pipeline_example.html old mode 100644 new mode 100755 diff --git a/app/modules/examples/pipeline_example.py b/app/modules/examples/pipeline_example.py old mode 100644 new mode 100755 diff --git a/app/modules/examples/yaml_example.html b/app/modules/examples/yaml_example.html old mode 100644 new mode 100755 diff --git a/app/modules/examples/yaml_example.py b/app/modules/examples/yaml_example.py old mode 100644 new mode 100755 diff --git a/app/modules/examples/yaml_example.yaml b/app/modules/examples/yaml_example.yaml old mode 100644 new mode 100755 diff --git a/app/modules/home.html b/app/modules/home.html old mode 100644 new mode 100755 index da8027f..320464a --- a/app/modules/home.html +++ b/app/modules/home.html @@ -8,6 +8,7 @@

    Home

  • Datastore example (save)
  • Datastore example (load)
  • Yaml example
    • +
  • Pipeline training

    • Logged in as : {{user_name}}

    diff --git a/app/modules/home.py b/app/modules/home.py old mode 100644 new mode 100755 diff --git a/app/modules/training/__init__.py b/app/modules/training/__init__.py new file mode 100755 index 0000000..4957e95 --- /dev/null +++ b/app/modules/training/__init__.py @@ -0,0 +1 @@ +# __init__.py needed to create "package" for this directory diff --git a/app/modules/training/pipeline_training.py b/app/modules/training/pipeline_training.py new file mode 100755 index 0000000..154b244 --- /dev/null +++ b/app/modules/training/pipeline_training.py @@ -0,0 +1,27 @@ +import webapp2 +import logging +import os + +from google.appengine.ext.webapp import template +from app.modules.common.squares import TwiceSquaredPipeline + +class PipelineTrainigHandler(webapp2.RequestHandler): + def get(self, number=10): + logging.info("PipelineTrainigHandler get(%s)" % number) + + if number == "": + number = 10 + else: + number = int(number) + + stage = TwiceSquaredPipeline(number) + stage.start() + + #square_stage = SquarePipeline(number) + #square_stage.start() + + template_path = os.path.join(os.path.dirname(__file__), 'pipeline_traning.html') + self.response.write(template.render(template_path, { + 'number': number, + })) + diff --git a/app/modules/training/pipeline_traning.html b/app/modules/training/pipeline_traning.html new file mode 100755 index 0000000..2e5aa31 --- /dev/null +++ b/app/modules/training/pipeline_traning.html @@ -0,0 +1,8 @@ + + +

    Pipeline Training

    +

    Square a number twice

    +

    Input: {{ number }}

    +

    Check application logs for square output

    + + diff --git a/docker-compose.yml b/docker-compose.yml old mode 100644 new mode 100755 diff --git a/docker/Dockerfile b/docker/Dockerfile old mode 100644 new mode 100755 diff --git a/env.yaml.dist b/env.yaml.dist old mode 100644 new mode 100755 diff --git a/index.yaml b/index.yaml old mode 100644 new mode 100755 diff --git a/libs/.gitkeep b/libs/.gitkeep old mode 100644 new mode 100755 diff --git a/main.py b/main.py old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 From c968b876b258df1a4010e197bce2d431023cc58c Mon Sep 17 00:00:00 2001 From: Najipian Date: Sun, 18 Nov 2018 19:08:07 +0200 Subject: [PATCH 04/22] http training task --- app.yaml | 4 ++ app/handlers.py | 2 + app/modules/home.html | 1 + app/modules/home.py | 7 +++- app/modules/training/http_training.html | 14 +++++++ app/modules/training/http_training.py | 55 +++++++++++++++++++++++++ requirements.txt | 2 + 7 files changed, 84 insertions(+), 1 deletion(-) create mode 100755 app/modules/training/http_training.html create mode 100644 app/modules/training/http_training.py diff --git a/app.yaml b/app.yaml index 60535e7..e44ed98 100755 --- a/app.yaml +++ b/app.yaml @@ -19,3 +19,7 @@ handlers: includes: - ./env.yaml + +libraries: + - name: ssl + version: 2.7 diff --git a/app/handlers.py b/app/handlers.py index 82f33ed..1747caf 100755 --- a/app/handlers.py +++ b/app/handlers.py @@ -4,6 +4,7 @@ from app.modules.examples.datastore_example import DatastoreExampleHandler from app.modules.examples.yaml_example import YamlExampleHandler from app.modules.training.pipeline_training import PipelineTrainigHandler +from app.modules.training.http_training import HttpTrainingHandler def handlers(): return [ @@ -17,4 +18,5 @@ def handlers(): (r'/examples/datastore/([^\/]+)/([^\/]+)', DatastoreExampleHandler), (r'/examples/yaml', YamlExampleHandler), (r'/training/pipeline/(\d+)', PipelineTrainigHandler), + (r'/training/http/([\d\.]+)/([\d\.]+)', HttpTrainingHandler), ] diff --git a/app/modules/home.html b/app/modules/home.html index 320464a..a341e1f 100755 --- a/app/modules/home.html +++ b/app/modules/home.html @@ -9,6 +9,7 @@

    Home

  • Datastore example (load)
  • Yaml example
  • Pipeline training
    • +
  • Http training

    • Logged in as : {{user_name}}

    diff --git a/app/modules/home.py b/app/modules/home.py index 9521d49..d7f5ab3 100755 --- a/app/modules/home.py +++ b/app/modules/home.py @@ -1,12 +1,17 @@ import webapp2 import logging import os +import geocoder from google.appengine.ext.webapp import template from google.appengine.api import users class HomeHandler(webapp2.RequestHandler): def get(self): logging.info("HomeHandler get()") + + # get logind user user = users.get_current_user() + template_path = os.path.join(os.path.dirname(__file__), 'home.html') - self.response.write(template.render(template_path, {'user_name' : user.nickname()})) + + self.response.write(template.render(template_path, {'user_name' : user.nickname() })) diff --git a/app/modules/training/http_training.html b/app/modules/training/http_training.html new file mode 100755 index 0000000..ba916c6 --- /dev/null +++ b/app/modules/training/http_training.html @@ -0,0 +1,14 @@ + + +

    HTTP Training

    + {% if title %} +

    Latitude: {{ lat }}

    +

    Longitude: {{ lon }}

    +

    Title: {{ title }}

    +

    Extract : {{ extract }}

    + + {% else %} +

    {{ error }}

    + {% endif %} + + diff --git a/app/modules/training/http_training.py b/app/modules/training/http_training.py new file mode 100644 index 0000000..a441f2f --- /dev/null +++ b/app/modules/training/http_training.py @@ -0,0 +1,55 @@ +import webapp2 +import logging +import os +import requests +from google.appengine.ext.webapp import template +from requests_toolbelt.adapters import appengine + + +appengine.monkeypatch(validate_certificate=False) + +class HttpTrainingHandler(webapp2.RequestHandler): + def get(self, lat = None, lon = None): + logging.info("HttpTrainingHandler get(%s,%s)" % (lat, lon)) + + # wikipedia base api url + API_HOST = "http://en.wikipedia.org/w/api.php?" + + # add given lat long to api url + apiUrl = "%sformat=json&action=query&prop=extracts&exintro=1&explaintext=1&exlimit=20&generator=geosearch&ggsradius=10000&ggslimit=100&ggscoord=%s%s%s" % (API_HOST, lat,"|", lon) + + logging.info(apiUrl) + + template_vars = {} + + try: + # make the api call + request = requests.get(apiUrl) + data = request.json() + + # check if there is data returned + if 'query' in data : + + for idx , page in data['query']['pages'].items(): + # get the 0 indexd page (first page data) + if page['index'] == 0: + template_vars = { + 'title': page['title'], + 'extract': page['extract'], + } + + break + + else: + # if there is no data returned show user + template_vars = { + 'error' : 'No pages found' + } + except requests.exceptions.RequestException as e: + logging.info(e) + template_vars = { + 'error' : e + } + + template_path = os.path.join(os.path.dirname(__file__), 'http_training.html') + self.response.write(template.render(template_path, template_vars )) diff --git a/requirements.txt b/requirements.txt index 67c8b3a..a3e8c2a 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ GoogleAppEnginePipeline GoogleAppEngineCloudStorageClient requests +requests_toolbelt +geocoder \ No newline at end of file From 86eecde676bd2708b206544a10ae695e6b447b25 Mon Sep 17 00:00:00 2001 From: Najipian Date: Mon, 19 Nov 2018 13:27:02 +0200 Subject: [PATCH 05/22] correct permissions --- .datastore/.gitkeep | 0 .env.dist | 0 README.md | 0 TASKS.md | 0 app-deploy.yaml | 0 app.yaml | 0 app/__init__.py | 0 app/modules/__init__.py | 0 app/modules/examples/__init__.py | 0 app/modules/examples/datastore_example.html | 0 app/modules/examples/datastore_example.py | 0 app/modules/examples/http_example.html | 0 app/modules/examples/http_example.py | 0 app/modules/examples/pipeline_example.html | 0 app/modules/examples/pipeline_example.py | 0 app/modules/examples/yaml_example.html | 0 app/modules/examples/yaml_example.py | 0 app/modules/examples/yaml_example.yaml | 0 app/modules/home.py | 0 docker-compose.yml | 0 docker/Dockerfile | 0 env.yaml.dist | 0 index.yaml | 0 libs/.gitkeep | 0 main.py | 0 requirements.txt | 0 26 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 .datastore/.gitkeep mode change 100755 => 100644 .env.dist mode change 100755 => 100644 README.md mode change 100755 => 100644 TASKS.md mode change 100755 => 100644 app-deploy.yaml mode change 100755 => 100644 app.yaml mode change 100755 => 100644 app/__init__.py mode change 100755 => 100644 app/modules/__init__.py mode change 100755 => 100644 app/modules/examples/__init__.py mode change 100755 => 100644 app/modules/examples/datastore_example.html mode change 100755 => 100644 app/modules/examples/datastore_example.py mode change 100755 => 100644 app/modules/examples/http_example.html mode change 100755 => 100644 app/modules/examples/http_example.py mode change 100755 => 100644 app/modules/examples/pipeline_example.html mode change 100755 => 100644 app/modules/examples/pipeline_example.py mode change 100755 => 100644 app/modules/examples/yaml_example.html mode change 100755 => 100644 app/modules/examples/yaml_example.py mode change 100755 => 100644 app/modules/examples/yaml_example.yaml mode change 100755 => 100644 app/modules/home.py mode change 100755 => 100644 docker-compose.yml mode change 100755 => 100644 docker/Dockerfile mode change 100755 => 100644 env.yaml.dist mode change 100755 => 100644 index.yaml mode change 100755 => 100644 libs/.gitkeep mode change 100755 => 100644 main.py mode change 100755 => 100644 requirements.txt diff --git a/.datastore/.gitkeep b/.datastore/.gitkeep old mode 100755 new mode 100644 diff --git a/.env.dist b/.env.dist old mode 100755 new mode 100644 diff --git a/README.md b/README.md old mode 100755 new mode 100644 diff --git a/TASKS.md b/TASKS.md old mode 100755 new mode 100644 diff --git a/app-deploy.yaml b/app-deploy.yaml old mode 100755 new mode 100644 diff --git a/app.yaml b/app.yaml old mode 100755 new mode 100644 diff --git a/app/__init__.py b/app/__init__.py old mode 100755 new mode 100644 diff --git a/app/modules/__init__.py b/app/modules/__init__.py old mode 100755 new mode 100644 diff --git a/app/modules/examples/__init__.py b/app/modules/examples/__init__.py old mode 100755 new mode 100644 diff --git a/app/modules/examples/datastore_example.html b/app/modules/examples/datastore_example.html old mode 100755 new mode 100644 diff --git a/app/modules/examples/datastore_example.py b/app/modules/examples/datastore_example.py old mode 100755 new mode 100644 diff --git a/app/modules/examples/http_example.html b/app/modules/examples/http_example.html old mode 100755 new mode 100644 diff --git a/app/modules/examples/http_example.py b/app/modules/examples/http_example.py old mode 100755 new mode 100644 diff --git a/app/modules/examples/pipeline_example.html b/app/modules/examples/pipeline_example.html old mode 100755 new mode 100644 diff --git a/app/modules/examples/pipeline_example.py b/app/modules/examples/pipeline_example.py old mode 100755 new mode 100644 diff --git a/app/modules/examples/yaml_example.html b/app/modules/examples/yaml_example.html old mode 100755 new mode 100644 diff --git a/app/modules/examples/yaml_example.py b/app/modules/examples/yaml_example.py old mode 100755 new mode 100644 diff --git a/app/modules/examples/yaml_example.yaml b/app/modules/examples/yaml_example.yaml old mode 100755 new mode 100644 diff --git a/app/modules/home.py b/app/modules/home.py old mode 100755 new mode 100644 diff --git a/docker-compose.yml b/docker-compose.yml old mode 100755 new mode 100644 diff --git a/docker/Dockerfile b/docker/Dockerfile old mode 100755 new mode 100644 diff --git a/env.yaml.dist b/env.yaml.dist old mode 100755 new mode 100644 diff --git a/index.yaml b/index.yaml old mode 100755 new mode 100644 diff --git a/libs/.gitkeep b/libs/.gitkeep old mode 100755 new mode 100644 diff --git a/main.py b/main.py old mode 100755 new mode 100644 diff --git a/requirements.txt b/requirements.txt old mode 100755 new mode 100644 From f6f4bb24de77bc1afa174ae234d5e0853d9655a5 Mon Sep 17 00:00:00 2001 From: Najipian Date: Mon, 19 Nov 2018 13:49:03 +0200 Subject: [PATCH 06/22] fix reviews --- .gitignore | 4 +++- app.yaml | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index dda0959..5c14171 100755 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ # Environment vars .env env.yaml -.vscode # Google App Engine files in Docker .datastore/* @@ -13,3 +12,6 @@ libs/* # Byte-compiled / optimized / Python files *.pyc + +# vscode folder +.vscode diff --git a/app.yaml b/app.yaml index e44ed98..a2b4548 100644 --- a/app.yaml +++ b/app.yaml @@ -20,6 +20,4 @@ handlers: includes: - ./env.yaml -libraries: - - name: ssl - version: 2.7 + From 32d49ae60948fbc61dc5a705898ee7c785aeaa1a Mon Sep 17 00:00:00 2001 From: Najipian Date: Mon, 19 Nov 2018 13:50:14 +0200 Subject: [PATCH 07/22] fix reviews --- app/modules/home.py | 2 +- requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/modules/home.py b/app/modules/home.py index d7f5ab3..ac0d5d0 100644 --- a/app/modules/home.py +++ b/app/modules/home.py @@ -1,7 +1,7 @@ import webapp2 import logging import os -import geocoder + from google.appengine.ext.webapp import template from google.appengine.api import users diff --git a/requirements.txt b/requirements.txt index a3e8c2a..d7b2f4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ GoogleAppEnginePipeline GoogleAppEngineCloudStorageClient requests requests_toolbelt -geocoder \ No newline at end of file From f844a4b82479491676f255c09dff04e1a0ce3d68 Mon Sep 17 00:00:00 2001 From: Najipian Date: Mon, 19 Nov 2018 14:00:16 +0200 Subject: [PATCH 08/22] fix reviews --- app/modules/common/squares.py | 13 +++++++++++++ app/modules/examples/pipeline_example.py | 14 ++------------ app/modules/home.py | 2 +- app/modules/training/http_training.html | 3 +-- app/modules/training/http_training.py | 2 +- app/modules/training/pipeline_training.py | 3 --- 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/app/modules/common/squares.py b/app/modules/common/squares.py index 6f8e441..96de852 100755 --- a/app/modules/common/squares.py +++ b/app/modules/common/squares.py @@ -2,6 +2,19 @@ import logging import pipeline +class SquarePipelineExample(pipeline.Pipeline): + + output_names = ['square'] + + def run(self, number): + logging.info("SquarePipelineExample run()") + self.fill(self.outputs.square, number * number) + + def finalized(self): + logging.info("SquarePipelineExample finalized()") + logging.info('All done! Square is %s', self.outputs.square.value) + + class SquarePipeline(pipeline.Pipeline): def run(self, number): diff --git a/app/modules/examples/pipeline_example.py b/app/modules/examples/pipeline_example.py index 7d619e6..2ac3eaf 100644 --- a/app/modules/examples/pipeline_example.py +++ b/app/modules/examples/pipeline_example.py @@ -3,6 +3,7 @@ import os import pipeline from google.appengine.ext.webapp import template +from app.modules.common.squares import SquarePipelineExample class PipelineExampleHandler(webapp2.RequestHandler): def get(self, number=10): @@ -13,7 +14,7 @@ def get(self, number=10): else: number = int(number) - square_stage = SquarePipeline(number) + square_stage = SquarePipelineExample(number) square_stage.start() template_path = os.path.join(os.path.dirname(__file__), 'pipeline_example.html') @@ -21,14 +22,3 @@ def get(self, number=10): 'number': number, })) -class SquarePipeline(pipeline.Pipeline): - - output_names = ['square'] - - def run(self, number): - logging.info("SquarePipeline run()") - self.fill(self.outputs.square, number * number) - - def finalized(self): - logging.info("SquarePipeline finalized()") - logging.info('All done! Square is %s', self.outputs.square.value) diff --git a/app/modules/home.py b/app/modules/home.py index ac0d5d0..99d99cc 100644 --- a/app/modules/home.py +++ b/app/modules/home.py @@ -9,7 +9,7 @@ class HomeHandler(webapp2.RequestHandler): def get(self): logging.info("HomeHandler get()") - # get logind user + # get logiged in user user = users.get_current_user() template_path = os.path.join(os.path.dirname(__file__), 'home.html') diff --git a/app/modules/training/http_training.html b/app/modules/training/http_training.html index ba916c6..0212eb0 100755 --- a/app/modules/training/http_training.html +++ b/app/modules/training/http_training.html @@ -2,8 +2,7 @@

    HTTP Training

    {% if title %} -

    Latitude: {{ lat }}

    -

    Longitude: {{ lon }}

    +

    Title: {{ title }}

    Extract : {{ extract }}

    diff --git a/app/modules/training/http_training.py b/app/modules/training/http_training.py index a441f2f..330b153 100644 --- a/app/modules/training/http_training.py +++ b/app/modules/training/http_training.py @@ -46,7 +46,7 @@ def get(self, lat = None, lon = None): 'error' : 'No pages found' } except requests.exceptions.RequestException as e: - logging.info(e) + logging.error(e) template_vars = { 'error' : e } diff --git a/app/modules/training/pipeline_training.py b/app/modules/training/pipeline_training.py index 154b244..5faef21 100755 --- a/app/modules/training/pipeline_training.py +++ b/app/modules/training/pipeline_training.py @@ -17,9 +17,6 @@ def get(self, number=10): stage = TwiceSquaredPipeline(number) stage.start() - #square_stage = SquarePipeline(number) - #square_stage.start() - template_path = os.path.join(os.path.dirname(__file__), 'pipeline_traning.html') self.response.write(template.render(template_path, { 'number': number, From 37eb5cfb47cdc89142e70e373a3365e446bafcb4 Mon Sep 17 00:00:00 2001 From: Najipian Date: Mon, 19 Nov 2018 15:46:30 +0200 Subject: [PATCH 09/22] Datastore training --- app/handlers.py | 3 +++ app/modules/common/kinds.py | 7 ++++++ app/modules/home.html | 1 + app/modules/training/datastore_training.html | 11 +++++++++ app/modules/training/datastore_training.py | 24 ++++++++++++++++++++ 5 files changed, 46 insertions(+) create mode 100644 app/modules/common/kinds.py create mode 100644 app/modules/training/datastore_training.html create mode 100644 app/modules/training/datastore_training.py diff --git a/app/handlers.py b/app/handlers.py index 1747caf..2730278 100755 --- a/app/handlers.py +++ b/app/handlers.py @@ -5,6 +5,7 @@ from app.modules.examples.yaml_example import YamlExampleHandler from app.modules.training.pipeline_training import PipelineTrainigHandler from app.modules.training.http_training import HttpTrainingHandler +from app.modules.training.datastore_training import DatastoreTrainingHandler def handlers(): return [ @@ -19,4 +20,6 @@ def handlers(): (r'/examples/yaml', YamlExampleHandler), (r'/training/pipeline/(\d+)', PipelineTrainigHandler), (r'/training/http/([\d\.]+)/([\d\.]+)', HttpTrainingHandler), + (r'/training/http/([\d\.]+)/([\d\.]+)', HttpTrainingHandler), + (r'/training/datastore', DatastoreTrainingHandler), ] diff --git a/app/modules/common/kinds.py b/app/modules/common/kinds.py new file mode 100644 index 0000000..00c8a0e --- /dev/null +++ b/app/modules/common/kinds.py @@ -0,0 +1,7 @@ +import webapp2 +import logging +from google.appengine.ext import ndb + + +class Example(ndb.Model): + value = ndb.StringProperty() \ No newline at end of file diff --git a/app/modules/home.html b/app/modules/home.html index a341e1f..f363919 100755 --- a/app/modules/home.html +++ b/app/modules/home.html @@ -10,6 +10,7 @@

    Home

  • Yaml example
  • Pipeline training
  • Http training
    • +
  • Datastore training

    • Logged in as : {{user_name}}

    diff --git a/app/modules/training/datastore_training.html b/app/modules/training/datastore_training.html new file mode 100644 index 0000000..597cf8f --- /dev/null +++ b/app/modules/training/datastore_training.html @@ -0,0 +1,11 @@ + + +

    Datastore Training

    +
    + {% for entity in entities %} +

    {{ entity.key }}

    +

    {{ entity.value }}

    + {% endfor %} +
    + + diff --git a/app/modules/training/datastore_training.py b/app/modules/training/datastore_training.py new file mode 100644 index 0000000..1d2001e --- /dev/null +++ b/app/modules/training/datastore_training.py @@ -0,0 +1,24 @@ +import webapp2 +import logging +import os +from google.appengine.ext import ndb +from google.appengine.ext.webapp import template +from app.modules.common.kinds import Example + +class DatastoreTrainingHandler(webapp2.RequestHandler): + def get(self): + logging.info("DatastoreTrainingHandler") + + # get entities + entities_for_example_kind = Example.query().fetch(20) + + # make a list of key value pairs + entities = [] + for entity in entities_for_example_kind: + entities.append({'key':entity.key.string_id() , 'value':entity.value}) + + template_path = os.path.join(os.path.dirname(__file__), 'datastore_training.html') + self.response.write(template.render(template_path, { + 'entities': entities, + })) + From aafb4307f852adc9ff7fd4930c4068c28b1e6fe4 Mon Sep 17 00:00:00 2001 From: Najipian Date: Mon, 19 Nov 2018 15:51:50 +0200 Subject: [PATCH 10/22] fix reviews --- app.yaml | 2 -- app/modules/home.py | 2 +- app/modules/training/http_training.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app.yaml b/app.yaml index a2b4548..60535e7 100644 --- a/app.yaml +++ b/app.yaml @@ -19,5 +19,3 @@ handlers: includes: - ./env.yaml - - diff --git a/app/modules/home.py b/app/modules/home.py index 99d99cc..3c1e726 100644 --- a/app/modules/home.py +++ b/app/modules/home.py @@ -9,7 +9,7 @@ class HomeHandler(webapp2.RequestHandler): def get(self): logging.info("HomeHandler get()") - # get logiged in user + # get logged in user user = users.get_current_user() template_path = os.path.join(os.path.dirname(__file__), 'home.html') diff --git a/app/modules/training/http_training.py b/app/modules/training/http_training.py index 330b153..e6bf0d2 100644 --- a/app/modules/training/http_training.py +++ b/app/modules/training/http_training.py @@ -5,7 +5,7 @@ from google.appengine.ext.webapp import template from requests_toolbelt.adapters import appengine - +# https://toolbelt.readthedocs.io/en/latest/adapters.html#appengineadapter appengine.monkeypatch(validate_certificate=False) class HttpTrainingHandler(webapp2.RequestHandler): From 6c76bbfd4e5b71421dd2763c1758342d0fb6d863 Mon Sep 17 00:00:00 2001 From: Najipian Date: Wed, 21 Nov 2018 17:22:26 +0200 Subject: [PATCH 11/22] create files and modules --- app/handlers.py | 11 +++-------- app/modules/common/kinds.py | 7 ++++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/handlers.py b/app/handlers.py index 5e415eb..8661a9c 100755 --- a/app/handlers.py +++ b/app/handlers.py @@ -5,10 +5,8 @@ from app.modules.examples.yaml_example import YamlExampleHandler from app.modules.training.pipeline_training import PipelineTrainigHandler from app.modules.training.http_training import HttpTrainingHandler -<<<<<<< HEAD -======= from app.modules.training.datastore_training import DatastoreTrainingHandler ->>>>>>> datastore-training +from app.modules.cityinfo.cityinfo_build import CityInfoBuildHandler def handlers(): return [ @@ -22,10 +20,7 @@ def handlers(): (r'/examples/datastore/([^\/]+)/([^\/]+)', DatastoreExampleHandler), (r'/examples/yaml', YamlExampleHandler), (r'/training/pipeline/(\d+)', PipelineTrainigHandler), - (r'/training/http/([\d\.]+)/([\d\.]+)', HttpTrainingHandler), -<<<<<<< HEAD -======= - (r'/training/http/([\d\.]+)/([\d\.]+)', HttpTrainingHandler), + (r'/training/http/([-?\d\.]+)/([-?\d\.]+)', HttpTrainingHandler), (r'/training/datastore', DatastoreTrainingHandler), ->>>>>>> datastore-training + (r'/cityinfo/build', CityInfoBuildHandler), ] diff --git a/app/modules/common/kinds.py b/app/modules/common/kinds.py index 00c8a0e..d4789f7 100644 --- a/app/modules/common/kinds.py +++ b/app/modules/common/kinds.py @@ -4,4 +4,9 @@ class Example(ndb.Model): - value = ndb.StringProperty() \ No newline at end of file + value = ndb.StringProperty() + +class CityInfo(ndb.Model): + Location = ndb.StringProperty() + Info = ndb.TextProperty() + LastUpdated = ndb.DateTimeProperty(auto_now_add=True) \ No newline at end of file From da396b43d4bd1282b7a0c869d790edd13813332f Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 22 Nov 2018 13:30:27 +0200 Subject: [PATCH 12/22] create pipelines --- app/modules/common/squares.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/modules/common/squares.py b/app/modules/common/squares.py index 96de852..fe63051 100755 --- a/app/modules/common/squares.py +++ b/app/modules/common/squares.py @@ -33,4 +33,5 @@ def run(self, number): class LogResult(pipeline.Pipeline): def run(self, number): - logging.info('All done! Value is %s', number) \ No newline at end of file + logging.info('All done! Value is %s', number) + From b42d97b6ee71d493fb9b9152fbd9b89ffa50ef9a Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 22 Nov 2018 15:03:51 +0200 Subject: [PATCH 13/22] get city info --- app/modules/cityinfo/__init__.py | 1 + app/modules/cityinfo/cityinfo.yaml | 13 +++ app/modules/cityinfo/cityinfo_build.py | 11 +++ app/modules/cityinfo/cityinfo_view.py | 0 app/modules/common/city_info.py | 118 +++++++++++++++++++++++++ 5 files changed, 143 insertions(+) create mode 100644 app/modules/cityinfo/__init__.py create mode 100644 app/modules/cityinfo/cityinfo.yaml create mode 100644 app/modules/cityinfo/cityinfo_build.py create mode 100644 app/modules/cityinfo/cityinfo_view.py create mode 100644 app/modules/common/city_info.py diff --git a/app/modules/cityinfo/__init__.py b/app/modules/cityinfo/__init__.py new file mode 100644 index 0000000..4957e95 --- /dev/null +++ b/app/modules/cityinfo/__init__.py @@ -0,0 +1 @@ +# __init__.py needed to create "package" for this directory diff --git a/app/modules/cityinfo/cityinfo.yaml b/app/modules/cityinfo/cityinfo.yaml new file mode 100644 index 0000000..bd3011d --- /dev/null +++ b/app/modules/cityinfo/cityinfo.yaml @@ -0,0 +1,13 @@ +cities: + - + name: London + lat: 51.507222 + lon: -0.1275 + - + name: Cairo + lat: 30.044444 + lon: 31.235833 + - + name: Beijing + lat: 39.916667 + lon: 116.383333 diff --git a/app/modules/cityinfo/cityinfo_build.py b/app/modules/cityinfo/cityinfo_build.py new file mode 100644 index 0000000..21c52a8 --- /dev/null +++ b/app/modules/cityinfo/cityinfo_build.py @@ -0,0 +1,11 @@ +import webapp2 +import logging +import os +from google.appengine.ext import ndb +from google.appengine.ext.webapp import template +from app.modules.common.city_info import CityInfoRootPipeline +class CityInfoBuildHandler(webapp2.RequestHandler): + def get(self): + CitiesUpdate = CityInfoRootPipeline() + CitiesUpdate.start() + diff --git a/app/modules/cityinfo/cityinfo_view.py b/app/modules/cityinfo/cityinfo_view.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/common/city_info.py b/app/modules/common/city_info.py new file mode 100644 index 0000000..650284a --- /dev/null +++ b/app/modules/common/city_info.py @@ -0,0 +1,118 @@ +import webapp2 +import logging +import pipeline +import os +import yaml +from pipeline import common +import requests +from requests_toolbelt.adapters import appengine + +# https://toolbelt.readthedocs.io/en/latest/adapters.html#appengineadapter +appengine.monkeypatch(validate_certificate=False) + +class CityInfoRootPipeline(pipeline.Pipeline): + + def run(self): + logging.info("CityInfoRootPipeline") + + # Read cityinfo.yaml file + yaml_path = os.path.join(os.path.dirname(__file__), '../cityinfo/cityinfo.yaml') + with open(yaml_path, 'r') as stream: + data = yaml.load(stream) + + logging.info(data) + + cities = [] + + for city in data['cities'] : + cityInfo = yield CityInfoFetchPipeline(city) + #yield common.Log.info('SplitCount result = %s', cityData) + + cities.append(cityInfo) + # yield common.List(cities) + + + #logging.info(*cities) + + yield CityInfoPersistPipeline(*cities) + +class CityInfoFetchPipeline(pipeline.Pipeline): + + def run(self, city): + logging.info("CityInfoFetchPipeline") + + cityinfo = yield CityInfoInfoPipeline(city['lat'] , city['lon'] ) + citytemp = yield CityInfoWeatherPipeline(city['lat'] , city['lon']) + + yield TestReturn(city['name'], cityinfo,citytemp) + + + +class CityInfoInfoPipeline(pipeline.Pipeline): + + def run(self, lat , lon): + logging.info("CityInfoInfoPipeline") + + # wikipedia base api url + API_HOST = "http://en.wikipedia.org/w/api.php?" + + + # add given lat long to api url + apiUrl = "%sformat=json&action=query&prop=extracts&exintro=1&explaintext=1&exlimit=20&generator=geosearch&ggsradius=10000&ggslimit=100&ggscoord=%s%s%s" % (API_HOST, lat,"|", lon) + + logging.info(apiUrl) + + template_vars = {} + + try: + # make the api call + request = requests.get(apiUrl) + data = request.json() + + # check if there is data returned + if 'query' in data : + + for idx , page in data['query']['pages'].items(): + # get the 0 indexd page (first page data) + if page['index'] == 0: + # return page['extract'] + return "good place" + + else: + # if there is no data returned show user + logging.error(e) + return None + except requests.exceptions.RequestException as e: + logging.error(e) + return None + +class CityInfoWeatherPipeline(pipeline.Pipeline): + + def run(self , lat , lon): + logging.info("CityInfoWeatherPipeline") + + API_HOST = "http://api.openweathermap.org" + API_KEY = os.environ.get("HTTP_EXAMPLE_API_KEY") + + if API_KEY: + apiUrl = "%s/data/2.5/forecast?appid=%s&mode=json&units=metric&lat=%s&lon=%s" % (API_HOST, API_KEY, lat, lon) + request = requests.get(apiUrl) + data = request.json() + return data['list'][0]['main']['temp'] + else: + return None + + +class CityInfoPersistPipeline(pipeline.Pipeline): + + def run(self, *args): + logging.info("CityInfoPersistPipeline") + logging.info(args) + +class TestReturn(pipeline.Pipeline): + + def run(self, cityname,cityInfo , cityTemp): + logging.info("TestReturn") + + return {'Location' : cityname ,'Info' : cityInfo , 'Temp' : cityTemp } + From 19f1118126e7c313f30a0b3fe8c07a440f6ada0e Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 22 Nov 2018 15:21:56 +0200 Subject: [PATCH 14/22] get city temp and store data --- app/modules/common/city_info.py | 21 +++++++++++++++++++++ app/modules/common/kinds.py | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/app/modules/common/city_info.py b/app/modules/common/city_info.py index 650284a..888da8d 100644 --- a/app/modules/common/city_info.py +++ b/app/modules/common/city_info.py @@ -6,6 +6,8 @@ from pipeline import common import requests from requests_toolbelt.adapters import appengine +from app.modules.common.kinds import CityInfo +from google.appengine.ext import ndb # https://toolbelt.readthedocs.io/en/latest/adapters.html#appengineadapter appengine.monkeypatch(validate_certificate=False) @@ -109,6 +111,25 @@ def run(self, *args): logging.info("CityInfoPersistPipeline") logging.info(args) + for city in args: + logging.info(city) + entity_key = ndb.Key('CityInfo', city['Location']) + entity = entity_key.get() + + if entity is None: + entity = CityInfo( + Location=city['Location'], + Info=city['Info'], + Temp=city['Temp'] + ) + entity.key = ndb.Key('CityInfo', city['Location']) + entity.put() + + else: + entity.Info = city['Info'] + entity.Temp = city['Temp'] + entity.put() + class TestReturn(pipeline.Pipeline): def run(self, cityname,cityInfo , cityTemp): diff --git a/app/modules/common/kinds.py b/app/modules/common/kinds.py index d4789f7..7d78c4f 100644 --- a/app/modules/common/kinds.py +++ b/app/modules/common/kinds.py @@ -9,4 +9,5 @@ class Example(ndb.Model): class CityInfo(ndb.Model): Location = ndb.StringProperty() Info = ndb.TextProperty() - LastUpdated = ndb.DateTimeProperty(auto_now_add=True) \ No newline at end of file + Temp = ndb.FloatProperty() + LastUpdated = ndb.DateTimeProperty(auto_now=True) \ No newline at end of file From 79c4051abc4080d643ee921ecb3e769f1db15815 Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 22 Nov 2018 16:05:31 +0200 Subject: [PATCH 15/22] seperate functions --- app/modules/cityinfo/city_extras.py | 77 +++++++++++++++++++++++++++ app/modules/common/city_info.py | 81 +++-------------------------- 2 files changed, 85 insertions(+), 73 deletions(-) create mode 100644 app/modules/cityinfo/city_extras.py diff --git a/app/modules/cityinfo/city_extras.py b/app/modules/cityinfo/city_extras.py new file mode 100644 index 0000000..83554c6 --- /dev/null +++ b/app/modules/cityinfo/city_extras.py @@ -0,0 +1,77 @@ +import requests +import logging +import os +from google.appengine.ext import ndb +from requests_toolbelt.adapters import appengine + +# https://toolbelt.readthedocs.io/en/latest/adapters.html#appengineadapter +appengine.monkeypatch(validate_certificate=False) + +def CityWikiInfo(lat , lon): + # wikipedia base api url + API_HOST = "http://en.wikipedia.org/w/api.php?" + + + # add given lat long to api url + apiUrl = "%sformat=json&action=query&prop=extracts&exintro=1&explaintext=1&exlimit=20&generator=geosearch&ggsradius=10000&ggslimit=100&ggscoord=%s%s%s" % (API_HOST, lat,"|", lon) + + logging.info(apiUrl) + + template_vars = {} + + try: + # make the api call + request = requests.get(apiUrl) + data = request.json() + + # check if there is data returned + if 'query' in data : + + for idx , page in data['query']['pages'].items(): + # get the 0 indexd page (first page data) + if page['index'] == 0: + return page['extract'] + + else: + # if there is no data returned show user + logging.error(e) + return None + except requests.exceptions.RequestException as e: + logging.error(e) + return None + +def CityWeatherTemp(lat , lon): + API_HOST = "http://api.openweathermap.org" + API_KEY = os.environ.get("HTTP_EXAMPLE_API_KEY") + + if API_KEY: + try: + apiUrl = "%s/data/2.5/forecast?appid=%s&mode=json&units=metric&lat=%s&lon=%s" % (API_HOST, API_KEY, lat, lon) + request = requests.get(apiUrl) + data = request.json() + return data['list'][0]['main']['temp'] + except requests.exceptions.RequestException as e: + logging.error(e) + + return None + +def StoreCitiesInto(cities): + for city in cities: + logging.info(city) + entity_key = ndb.Key('CityInfo', city['Location']) + entity = entity_key.get() + + if entity is None: + entity = CityInfo( + Location=city['Location'], + Info=city['Info'], + Temp=city['Temp'] + ) + entity.key = ndb.Key('CityInfo', city['Location']) + entity.put() + + else: + entity.Info = city['Info'] + entity.Temp = city['Temp'] + entity.put() + diff --git a/app/modules/common/city_info.py b/app/modules/common/city_info.py index 888da8d..580088f 100644 --- a/app/modules/common/city_info.py +++ b/app/modules/common/city_info.py @@ -3,14 +3,9 @@ import pipeline import os import yaml -from pipeline import common -import requests -from requests_toolbelt.adapters import appengine from app.modules.common.kinds import CityInfo from google.appengine.ext import ndb - -# https://toolbelt.readthedocs.io/en/latest/adapters.html#appengineadapter -appengine.monkeypatch(validate_certificate=False) +import app.modules.cityinfo.city_extras as CityExtras class CityInfoRootPipeline(pipeline.Pipeline): @@ -46,7 +41,7 @@ def run(self, city): cityinfo = yield CityInfoInfoPipeline(city['lat'] , city['lon'] ) citytemp = yield CityInfoWeatherPipeline(city['lat'] , city['lon']) - yield TestReturn(city['name'], cityinfo,citytemp) + yield CityInfoReturn(city['name'], cityinfo,citytemp) @@ -54,86 +49,26 @@ class CityInfoInfoPipeline(pipeline.Pipeline): def run(self, lat , lon): logging.info("CityInfoInfoPipeline") - - # wikipedia base api url - API_HOST = "http://en.wikipedia.org/w/api.php?" - - - # add given lat long to api url - apiUrl = "%sformat=json&action=query&prop=extracts&exintro=1&explaintext=1&exlimit=20&generator=geosearch&ggsradius=10000&ggslimit=100&ggscoord=%s%s%s" % (API_HOST, lat,"|", lon) - - logging.info(apiUrl) - - template_vars = {} - - try: - # make the api call - request = requests.get(apiUrl) - data = request.json() - - # check if there is data returned - if 'query' in data : - - for idx , page in data['query']['pages'].items(): - # get the 0 indexd page (first page data) - if page['index'] == 0: - # return page['extract'] - return "good place" - - else: - # if there is no data returned show user - logging.error(e) - return None - except requests.exceptions.RequestException as e: - logging.error(e) - return None + return CityExtras.CityWikiInfo(lat , lon) class CityInfoWeatherPipeline(pipeline.Pipeline): def run(self , lat , lon): logging.info("CityInfoWeatherPipeline") - - API_HOST = "http://api.openweathermap.org" - API_KEY = os.environ.get("HTTP_EXAMPLE_API_KEY") - - if API_KEY: - apiUrl = "%s/data/2.5/forecast?appid=%s&mode=json&units=metric&lat=%s&lon=%s" % (API_HOST, API_KEY, lat, lon) - request = requests.get(apiUrl) - data = request.json() - return data['list'][0]['main']['temp'] - else: - return None - + return CityExtras.CityWeatherTemp(lat , lon) class CityInfoPersistPipeline(pipeline.Pipeline): def run(self, *args): logging.info("CityInfoPersistPipeline") - logging.info(args) - - for city in args: - logging.info(city) - entity_key = ndb.Key('CityInfo', city['Location']) - entity = entity_key.get() - - if entity is None: - entity = CityInfo( - Location=city['Location'], - Info=city['Info'], - Temp=city['Temp'] - ) - entity.key = ndb.Key('CityInfo', city['Location']) - entity.put() + # logging.info(args) - else: - entity.Info = city['Info'] - entity.Temp = city['Temp'] - entity.put() + CityExtras.StoreCitiesInto(args) -class TestReturn(pipeline.Pipeline): +class CityInfoReturn(pipeline.Pipeline): def run(self, cityname,cityInfo , cityTemp): - logging.info("TestReturn") + logging.info("CityInfoReturn") return {'Location' : cityname ,'Info' : cityInfo , 'Temp' : cityTemp } From 38dd837f737f650bc96ee8f61c75dcf21302d5d2 Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 22 Nov 2018 16:44:10 +0200 Subject: [PATCH 16/22] cityinfo view end point --- app/handlers.py | 4 +++- app/modules/cityinfo/city_extras.py | 6 +++--- app/modules/cityinfo/cityinfo_build.py | 11 ----------- app/modules/cityinfo/cityinfo_view.py | 0 app/modules/common/city_info.py | 15 +-------------- 5 files changed, 7 insertions(+), 29 deletions(-) delete mode 100644 app/modules/cityinfo/cityinfo_build.py delete mode 100644 app/modules/cityinfo/cityinfo_view.py diff --git a/app/handlers.py b/app/handlers.py index 8661a9c..4e3472c 100755 --- a/app/handlers.py +++ b/app/handlers.py @@ -6,7 +6,8 @@ from app.modules.training.pipeline_training import PipelineTrainigHandler from app.modules.training.http_training import HttpTrainingHandler from app.modules.training.datastore_training import DatastoreTrainingHandler -from app.modules.cityinfo.cityinfo_build import CityInfoBuildHandler +from app.modules.cityinfo.cityinfo import CityInfoBuildHandler +from app.modules.cityinfo.cityinfo import CityInfoViewHandler def handlers(): return [ @@ -23,4 +24,5 @@ def handlers(): (r'/training/http/([-?\d\.]+)/([-?\d\.]+)', HttpTrainingHandler), (r'/training/datastore', DatastoreTrainingHandler), (r'/cityinfo/build', CityInfoBuildHandler), + (r'/cityinfo/view', CityInfoViewHandler), ] diff --git a/app/modules/cityinfo/city_extras.py b/app/modules/cityinfo/city_extras.py index 83554c6..b6aced1 100644 --- a/app/modules/cityinfo/city_extras.py +++ b/app/modules/cityinfo/city_extras.py @@ -3,6 +3,7 @@ import os from google.appengine.ext import ndb from requests_toolbelt.adapters import appengine +from app.modules.common.kinds import CityInfo # https://toolbelt.readthedocs.io/en/latest/adapters.html#appengineadapter appengine.monkeypatch(validate_certificate=False) @@ -35,10 +36,10 @@ def CityWikiInfo(lat , lon): else: # if there is no data returned show user logging.error(e) - return None except requests.exceptions.RequestException as e: logging.error(e) - return None + + return None def CityWeatherTemp(lat , lon): API_HOST = "http://api.openweathermap.org" @@ -57,7 +58,6 @@ def CityWeatherTemp(lat , lon): def StoreCitiesInto(cities): for city in cities: - logging.info(city) entity_key = ndb.Key('CityInfo', city['Location']) entity = entity_key.get() diff --git a/app/modules/cityinfo/cityinfo_build.py b/app/modules/cityinfo/cityinfo_build.py deleted file mode 100644 index 21c52a8..0000000 --- a/app/modules/cityinfo/cityinfo_build.py +++ /dev/null @@ -1,11 +0,0 @@ -import webapp2 -import logging -import os -from google.appengine.ext import ndb -from google.appengine.ext.webapp import template -from app.modules.common.city_info import CityInfoRootPipeline -class CityInfoBuildHandler(webapp2.RequestHandler): - def get(self): - CitiesUpdate = CityInfoRootPipeline() - CitiesUpdate.start() - diff --git a/app/modules/cityinfo/cityinfo_view.py b/app/modules/cityinfo/cityinfo_view.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/modules/common/city_info.py b/app/modules/common/city_info.py index 580088f..4c45b69 100644 --- a/app/modules/common/city_info.py +++ b/app/modules/common/city_info.py @@ -3,8 +3,6 @@ import pipeline import os import yaml -from app.modules.common.kinds import CityInfo -from google.appengine.ext import ndb import app.modules.cityinfo.city_extras as CityExtras class CityInfoRootPipeline(pipeline.Pipeline): @@ -17,19 +15,11 @@ def run(self): with open(yaml_path, 'r') as stream: data = yaml.load(stream) - logging.info(data) - cities = [] for city in data['cities'] : cityInfo = yield CityInfoFetchPipeline(city) - #yield common.Log.info('SplitCount result = %s', cityData) - cities.append(cityInfo) - # yield common.List(cities) - - - #logging.info(*cities) yield CityInfoPersistPipeline(*cities) @@ -42,8 +32,6 @@ def run(self, city): citytemp = yield CityInfoWeatherPipeline(city['lat'] , city['lon']) yield CityInfoReturn(city['name'], cityinfo,citytemp) - - class CityInfoInfoPipeline(pipeline.Pipeline): @@ -61,8 +49,7 @@ class CityInfoPersistPipeline(pipeline.Pipeline): def run(self, *args): logging.info("CityInfoPersistPipeline") - # logging.info(args) - + CityExtras.StoreCitiesInto(args) class CityInfoReturn(pipeline.Pipeline): From e77e4d49bb0670156f48fb8683e8879e1162acf8 Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 22 Nov 2018 16:45:05 +0200 Subject: [PATCH 17/22] cityinfo view end point --- app/modules/cityinfo/city_info.html | 14 ++++++++++++++ app/modules/cityinfo/cityinfo.py | 27 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 app/modules/cityinfo/city_info.html create mode 100644 app/modules/cityinfo/cityinfo.py diff --git a/app/modules/cityinfo/city_info.html b/app/modules/cityinfo/city_info.html new file mode 100644 index 0000000..c7b24fd --- /dev/null +++ b/app/modules/cityinfo/city_info.html @@ -0,0 +1,14 @@ + + + +

    Cities Information

    + +
    + {% for city in cities %} +

    {{ city.Location }}

    +

    Temp : {{ city.Temp }}

    +

    Info : {{ city.Info }}

    + {% endfor %} +
    + + diff --git a/app/modules/cityinfo/cityinfo.py b/app/modules/cityinfo/cityinfo.py new file mode 100644 index 0000000..8177163 --- /dev/null +++ b/app/modules/cityinfo/cityinfo.py @@ -0,0 +1,27 @@ +import webapp2 +import logging +import os +from google.appengine.ext.webapp import template +from app.modules.common.city_info import CityInfoRootPipeline +from app.modules.common.kinds import CityInfo + +class CityInfoBuildHandler(webapp2.RequestHandler): + def get(self): + logging.info("CityInfoBuildHandler") + CitiesUpdate = CityInfoRootPipeline() + CitiesUpdate.start() + +class CityInfoViewHandler(webapp2.RequestHandler): + def get(self): + logging.info("CityInfoViewHandler") + + # get cities + cities = CityInfo.query().fetch(20) + + logging.info(cities) + template_path = os.path.join(os.path.dirname(__file__), 'city_info.html') + self.response.write(template.render(template_path, { + 'cities': cities, + })) + + From f1fa563d0059fa84180c39f20622ff482d14e371 Mon Sep 17 00:00:00 2001 From: Najipian Date: Thu, 22 Nov 2018 18:33:38 +0200 Subject: [PATCH 18/22] cron job --- app/modules/cityinfo/city_extras.py | 1 - app/modules/cityinfo/city_info.html | 1 + app/modules/cityinfo/cityinfo.py | 2 -- app/modules/common/city_info.py | 1 - app/modules/common/kinds.py | 3 ++- cron.yaml | 4 ++++ 6 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 cron.yaml diff --git a/app/modules/cityinfo/city_extras.py b/app/modules/cityinfo/city_extras.py index b6aced1..bb7bd5b 100644 --- a/app/modules/cityinfo/city_extras.py +++ b/app/modules/cityinfo/city_extras.py @@ -74,4 +74,3 @@ def StoreCitiesInto(cities): entity.Info = city['Info'] entity.Temp = city['Temp'] entity.put() - diff --git a/app/modules/cityinfo/city_info.html b/app/modules/cityinfo/city_info.html index c7b24fd..1931fb0 100644 --- a/app/modules/cityinfo/city_info.html +++ b/app/modules/cityinfo/city_info.html @@ -7,6 +7,7 @@

    Cities Information

    {% for city in cities %}

    {{ city.Location }}

    Temp : {{ city.Temp }}

    +

    Last Updated : {{ city.LastUpdated }}

    Info : {{ city.Info }}

    {% endfor %} diff --git a/app/modules/cityinfo/cityinfo.py b/app/modules/cityinfo/cityinfo.py index 8177163..79d4542 100644 --- a/app/modules/cityinfo/cityinfo.py +++ b/app/modules/cityinfo/cityinfo.py @@ -23,5 +23,3 @@ def get(self): self.response.write(template.render(template_path, { 'cities': cities, })) - - diff --git a/app/modules/common/city_info.py b/app/modules/common/city_info.py index 4c45b69..79407a3 100644 --- a/app/modules/common/city_info.py +++ b/app/modules/common/city_info.py @@ -58,4 +58,3 @@ def run(self, cityname,cityInfo , cityTemp): logging.info("CityInfoReturn") return {'Location' : cityname ,'Info' : cityInfo , 'Temp' : cityTemp } - diff --git a/app/modules/common/kinds.py b/app/modules/common/kinds.py index 7d78c4f..db348c6 100644 --- a/app/modules/common/kinds.py +++ b/app/modules/common/kinds.py @@ -10,4 +10,5 @@ class CityInfo(ndb.Model): Location = ndb.StringProperty() Info = ndb.TextProperty() Temp = ndb.FloatProperty() - LastUpdated = ndb.DateTimeProperty(auto_now=True) \ No newline at end of file + LastUpdated = ndb.DateTimeProperty(auto_now=True) + \ No newline at end of file diff --git a/cron.yaml b/cron.yaml new file mode 100644 index 0000000..1ca68f5 --- /dev/null +++ b/cron.yaml @@ -0,0 +1,4 @@ +cron: +- description: "City info build hourly" + url: /cityinfo/build + schedule: every 1 hours \ No newline at end of file From e72b42ed1d650ab6110f5536027f6543af955085 Mon Sep 17 00:00:00 2001 From: Najipian Date: Sun, 25 Nov 2018 11:38:13 +0200 Subject: [PATCH 19/22] city view link , deploy cron file --- app/modules/home.html | 5 +++-- deploy-cron.sh | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 deploy-cron.sh diff --git a/app/modules/home.html b/app/modules/home.html index f363919..47b8f02 100755 --- a/app/modules/home.html +++ b/app/modules/home.html @@ -9,8 +9,9 @@

    Home

  • Datastore example (load)
  • Yaml example
  • Pipeline training
    • -
  • Http training
    • -
  • Datastore training
    • +
  • Http training
    • +
  • Datastore training
    • +
  • City info view

    • Logged in as : {{user_name}}

    diff --git a/deploy-cron.sh b/deploy-cron.sh new file mode 100644 index 0000000..69a0985 --- /dev/null +++ b/deploy-cron.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +source /google-cloud-sdk/path.bash.inc + +echo "Google auth..." +GCLOUD_USER=`gcloud auth list --filter=status:ACTIVE --format="value(account)"` +printf "GCLOUD_USER: %s\n" $GCLOUD_USER +if [ -z "$GCLOUD_USER" ] +then + gcloud auth login +fi + +echo "Starting AppEngine deploy..." +printf "APPENGINE_PROJECT_ID: %s\n" $APPENGINE_PROJECT_ID +if [ -z "$APPENGINE_PROJECT_ID" ] +then + echo "You must obtain an AppEngine PROJECT_ID [https://console.cloud.google.com/appengine] and place it in .env before running this!" +else + gcloud app deploy cron.yaml --project $APPENGINE_PROJECT_ID +fi From 7c4b827dc7e0a0c7939756b0949ffbdc11d1f71c Mon Sep 17 00:00:00 2001 From: Najipian Date: Sun, 2 Dec 2018 12:41:19 +0200 Subject: [PATCH 20/22] review files --- app/handlers.py | 3 +-- app/modules/cityinfo/city_extras.py | 1 - app/modules/cityinfo/city_info.html | 2 -- app/modules/common/kinds.py | 4 +--- app/modules/training/http_training.py | 5 ++--- 5 files changed, 4 insertions(+), 11 deletions(-) diff --git a/app/handlers.py b/app/handlers.py index 4e3472c..f124aeb 100755 --- a/app/handlers.py +++ b/app/handlers.py @@ -6,8 +6,7 @@ from app.modules.training.pipeline_training import PipelineTrainigHandler from app.modules.training.http_training import HttpTrainingHandler from app.modules.training.datastore_training import DatastoreTrainingHandler -from app.modules.cityinfo.cityinfo import CityInfoBuildHandler -from app.modules.cityinfo.cityinfo import CityInfoViewHandler +from app.modules.cityinfo.cityinfo import CityInfoBuildHandler , CityInfoViewHandler def handlers(): return [ diff --git a/app/modules/cityinfo/city_extras.py b/app/modules/cityinfo/city_extras.py index bb7bd5b..b4809a4 100644 --- a/app/modules/cityinfo/city_extras.py +++ b/app/modules/cityinfo/city_extras.py @@ -12,7 +12,6 @@ def CityWikiInfo(lat , lon): # wikipedia base api url API_HOST = "http://en.wikipedia.org/w/api.php?" - # add given lat long to api url apiUrl = "%sformat=json&action=query&prop=extracts&exintro=1&explaintext=1&exlimit=20&generator=geosearch&ggsradius=10000&ggslimit=100&ggscoord=%s%s%s" % (API_HOST, lat,"|", lon) diff --git a/app/modules/cityinfo/city_info.html b/app/modules/cityinfo/city_info.html index 1931fb0..6c9addc 100644 --- a/app/modules/cityinfo/city_info.html +++ b/app/modules/cityinfo/city_info.html @@ -1,8 +1,6 @@ -

    Cities Information

    -
    {% for city in cities %}

    {{ city.Location }}

    diff --git a/app/modules/common/kinds.py b/app/modules/common/kinds.py index db348c6..6b6e3eb 100644 --- a/app/modules/common/kinds.py +++ b/app/modules/common/kinds.py @@ -2,7 +2,6 @@ import logging from google.appengine.ext import ndb - class Example(ndb.Model): value = ndb.StringProperty() @@ -10,5 +9,4 @@ class CityInfo(ndb.Model): Location = ndb.StringProperty() Info = ndb.TextProperty() Temp = ndb.FloatProperty() - LastUpdated = ndb.DateTimeProperty(auto_now=True) - \ No newline at end of file + LastUpdated = ndb.DateTimeProperty(auto_now=True) \ No newline at end of file diff --git a/app/modules/training/http_training.py b/app/modules/training/http_training.py index 0131348..d2eeec8 100644 --- a/app/modules/training/http_training.py +++ b/app/modules/training/http_training.py @@ -13,9 +13,8 @@ def get(self, lat = None, lon = None): logging.info("HttpTrainingHandler get(%s,%s)" % (lat, lon)) # wikipedia base api url - API_HOST = "http://en.wikipedia.org/w/api.php?" - - + API_HOST = "https://en.wikipedia.org/w/api.php?" + # add given lat long to api url apiUrl = "%sformat=json&action=query&prop=extracts&exintro=1&explaintext=1&exlimit=20&generator=geosearch&ggsradius=10000&ggslimit=100&ggscoord=%s%s%s" % (API_HOST, lat,"|", lon) From b18791f89c9cd7cb3efb12855363a4edc3b91ce1 Mon Sep 17 00:00:00 2001 From: Najipian Date: Sun, 2 Dec 2018 12:43:58 +0200 Subject: [PATCH 21/22] review files --- app/modules/cityinfo/city_extras.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/modules/cityinfo/city_extras.py b/app/modules/cityinfo/city_extras.py index b4809a4..ca43450 100644 --- a/app/modules/cityinfo/city_extras.py +++ b/app/modules/cityinfo/city_extras.py @@ -10,7 +10,7 @@ def CityWikiInfo(lat , lon): # wikipedia base api url - API_HOST = "http://en.wikipedia.org/w/api.php?" + API_HOST = "https://en.wikipedia.org/w/api.php?" # add given lat long to api url apiUrl = "%sformat=json&action=query&prop=extracts&exintro=1&explaintext=1&exlimit=20&generator=geosearch&ggsradius=10000&ggslimit=100&ggscoord=%s%s%s" % (API_HOST, lat,"|", lon) From 78a04f400d86a0d1dc9fb7d7fc1e3d81cc319d63 Mon Sep 17 00:00:00 2001 From: Najipian Date: Mon, 3 Dec 2018 12:40:18 +0200 Subject: [PATCH 22/22] cityinfo reviews --- app/modules/cityinfo/city_extras.py | 30 ++++++++++++++--------------- app/modules/cityinfo/city_info.html | 3 +++ app/modules/cityinfo/cityinfo.py | 1 - app/modules/cityinfo/cityinfo.yaml | 5 +++++ app/modules/common/city_info.py | 10 +++------- app/modules/common/util.py | 10 ++++++++++ deploy-cron.sh | 20 ------------------- deploy.sh | 2 +- 8 files changed, 37 insertions(+), 44 deletions(-) create mode 100644 app/modules/common/util.py delete mode 100644 deploy-cron.sh diff --git a/app/modules/cityinfo/city_extras.py b/app/modules/cityinfo/city_extras.py index ca43450..1627cbe 100644 --- a/app/modules/cityinfo/city_extras.py +++ b/app/modules/cityinfo/city_extras.py @@ -55,21 +55,21 @@ def CityWeatherTemp(lat , lon): return None -def StoreCitiesInto(cities): +def StoreCitiesInfo(cities): for city in cities: - entity_key = ndb.Key('CityInfo', city['Location']) - entity = entity_key.get() + entity_key = ndb.Key('CityInfo', city['Location']) + entity = entity_key.get() - if entity is None: - entity = CityInfo( - Location=city['Location'], - Info=city['Info'], - Temp=city['Temp'] - ) - entity.key = ndb.Key('CityInfo', city['Location']) - entity.put() + if entity is None: + entity = CityInfo( + Location=city['Location'], + Info=city['Info'], + Temp=city['Temp'] + ) + entity.key = ndb.Key('CityInfo', city['Location']) + entity.put() - else: - entity.Info = city['Info'] - entity.Temp = city['Temp'] - entity.put() + else: + entity.Info = city['Info'] + entity.Temp = city['Temp'] + entity.put() diff --git a/app/modules/cityinfo/city_info.html b/app/modules/cityinfo/city_info.html index 6c9addc..982e730 100644 --- a/app/modules/cityinfo/city_info.html +++ b/app/modules/cityinfo/city_info.html @@ -8,6 +8,9 @@

    Cities Information

    Last Updated : {{ city.LastUpdated }}

    Info : {{ city.Info }}

    {% endfor %} + {% if not cities %} + No cities found + {% endif %}
    diff --git a/app/modules/cityinfo/cityinfo.py b/app/modules/cityinfo/cityinfo.py index 79d4542..266e7cf 100644 --- a/app/modules/cityinfo/cityinfo.py +++ b/app/modules/cityinfo/cityinfo.py @@ -18,7 +18,6 @@ def get(self): # get cities cities = CityInfo.query().fetch(20) - logging.info(cities) template_path = os.path.join(os.path.dirname(__file__), 'city_info.html') self.response.write(template.render(template_path, { 'cities': cities, diff --git a/app/modules/cityinfo/cityinfo.yaml b/app/modules/cityinfo/cityinfo.yaml index bd3011d..5ce7847 100644 --- a/app/modules/cityinfo/cityinfo.yaml +++ b/app/modules/cityinfo/cityinfo.yaml @@ -11,3 +11,8 @@ cities: name: Beijing lat: 39.916667 lon: 116.383333 + + - + name: Paris + lat: 48.8588376 + lon: 2.2768489 diff --git a/app/modules/common/city_info.py b/app/modules/common/city_info.py index 79407a3..1809aee 100644 --- a/app/modules/common/city_info.py +++ b/app/modules/common/city_info.py @@ -1,9 +1,8 @@ import webapp2 import logging import pipeline -import os -import yaml import app.modules.cityinfo.city_extras as CityExtras +import app.modules.common.util as Utils class CityInfoRootPipeline(pipeline.Pipeline): @@ -11,10 +10,7 @@ def run(self): logging.info("CityInfoRootPipeline") # Read cityinfo.yaml file - yaml_path = os.path.join(os.path.dirname(__file__), '../cityinfo/cityinfo.yaml') - with open(yaml_path, 'r') as stream: - data = yaml.load(stream) - + data = Utils.ReadYamlFile('../cityinfo/cityinfo.yaml') cities = [] for city in data['cities'] : @@ -50,7 +46,7 @@ class CityInfoPersistPipeline(pipeline.Pipeline): def run(self, *args): logging.info("CityInfoPersistPipeline") - CityExtras.StoreCitiesInto(args) + CityExtras.StoreCitiesInfo(args) class CityInfoReturn(pipeline.Pipeline): diff --git a/app/modules/common/util.py b/app/modules/common/util.py new file mode 100644 index 0000000..c2d6fa9 --- /dev/null +++ b/app/modules/common/util.py @@ -0,0 +1,10 @@ +import os +import yaml + +def ReadYamlFile(fileLocation): + data = {} + yaml_path = os.path.join(os.path.dirname(__file__), fileLocation) + with open(yaml_path, 'r') as stream: + data = yaml.load(stream) + + return data \ No newline at end of file diff --git a/deploy-cron.sh b/deploy-cron.sh deleted file mode 100644 index 69a0985..0000000 --- a/deploy-cron.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -source /google-cloud-sdk/path.bash.inc - -echo "Google auth..." -GCLOUD_USER=`gcloud auth list --filter=status:ACTIVE --format="value(account)"` -printf "GCLOUD_USER: %s\n" $GCLOUD_USER -if [ -z "$GCLOUD_USER" ] -then - gcloud auth login -fi - -echo "Starting AppEngine deploy..." -printf "APPENGINE_PROJECT_ID: %s\n" $APPENGINE_PROJECT_ID -if [ -z "$APPENGINE_PROJECT_ID" ] -then - echo "You must obtain an AppEngine PROJECT_ID [https://console.cloud.google.com/appengine] and place it in .env before running this!" -else - gcloud app deploy cron.yaml --project $APPENGINE_PROJECT_ID -fi diff --git a/deploy.sh b/deploy.sh index fe1347a..371f95a 100755 --- a/deploy.sh +++ b/deploy.sh @@ -16,5 +16,5 @@ if [ -z "$APPENGINE_PROJECT_ID" ] then echo "You must obtain an AppEngine PROJECT_ID [https://console.cloud.google.com/appengine] and place it in .env before running this!" else - gcloud app deploy app-deploy.yaml --project $APPENGINE_PROJECT_ID --version 1 --promote + gcloud app deploy app-deploy.yaml cron.yaml --project $APPENGINE_PROJECT_ID --version 1 --promote fi