settings.py

import sys
from os import getenv
from flask_talisman import Talisman
import requests
import logging
from random import randint
from datetime import datetime
from google.auth.transport.requests import Request
from requests.exceptions import ConnectionError

IS_APP_ENGINE = getenv("IS_APP_ENGINE", "false").lower() == "true"
TIER = getenv('TIER', 'dev')

if IS_APP_ENGINE:
    import google.cloud.logging
    client = google.cloud.logging_v2.Client()
    client.setup_logging()
else:
    from logging.config import dictConfig
    dictConfig({
        'version': 1,
        'formatters': {'default': {
            'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
        }},
        'handlers': {'wsgi': {
            'class': 'logging.StreamHandler',
            'stream': 'ext://flask.logging.wsgi_errors_stream',
            'formatter': 'default'
        }},
        'root': {
            'level': 'DEBUG',
            'handlers': ['wsgi']
        }
    })

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG if TIER == 'dev' else logging.INFO)

hsts_max_age = int(getenv('HSTS_MAX_AGE') or 3600)
IS_LOCAL = bool(getenv('IS_LOCAL','False').lower() == 'true')
BQ_METADATA_PROJ = getenv('BQ_METADATA_PROJ', 'isb-cgc-dev-1')
BQ_ECOSYS_BUCKET = getenv('BQ_ECOSYS_BUCKET',
                                  'https://storage.googleapis.com/isb-cgc-dev-bqs-metadata/bq_ecosys/')
BQ_FILTER_FILE_NAME = 'bq_meta_filters.json'
BQ_FILTER_FILE_PATH = BQ_ECOSYS_BUCKET + BQ_FILTER_FILE_NAME
BQ_METADATA_FILE_NAME = 'bq_meta_data.json'
BQ_METADATA_FILE_PATH = BQ_ECOSYS_BUCKET + BQ_METADATA_FILE_NAME

#
# We use the metadata server to get Bearer Tokens to access private buckets, except for local dev:
#
METADATA_URL = 'http://metadata.google.internal/computeMetadata/v1/'
METADATA_HEADERS = {'Metadata-Flavor': 'Google'}
SERVICE_ACCOUNT = getenv('SERVICE_ACCOUNT', 'default')
CREDENTIAL_SCOPES = ["https://www.googleapis.com/auth/devstorage.read_only"]

bq_table_files = {
    'bq_filters': {'last_modified': None, 'file_path': BQ_FILTER_FILE_PATH,
                   'file_data': None},
    'bq_metadata': {'last_modified': None, 'file_path': BQ_METADATA_FILE_PATH,
                    'file_data': None}
}
bq_total_entries = 0


# setup application
def setup_app(app):
    app.config['TESTING'] = (TIER.lower() != 'prod')
    app.config['ENV'] = 'production' if TIER.lower() == 'prod' else 'development'
    if not IS_LOCAL:
        Talisman(app, strict_transport_security_max_age=hsts_max_age, content_security_policy={
            'default-src': [
                '\'self\'',
                '*.googletagmanager.com',
                '*.google-analytics.com',
                '*.googleapis.com',
                "*.fontawesome.com",
                '*.jsdelivr.net',
                '\'unsafe-inline\'',
                'data:',
                'blob:'
            ],
            'font-src': ['\'self\'', '*.gstatic.com', '*.fontawesome.com']
        })


def get_access_token():
    access_token = None
    if not IS_LOCAL:
        try:
            url = '{}instance/service-accounts/{}/token'.format(METADATA_URL, SERVICE_ACCOUNT)
            # Request an access token from the metadata server.
            r = requests.get(url, headers=METADATA_HEADERS)
            r.raise_for_status()
            # Extract the access token from the response.
            access_token = r.json()['access_token']
        except ConnectionError as e:
            sys.exit(1)
    else:
        import google.auth
        # When developing locally, developer must use "gcloud auth application-default login" to get
        # the application default credentials working locally:
        credentials, _ = google.auth.default(scopes=CREDENTIAL_SCOPES)
        credentials.refresh(google.auth.transport.requests.Request())  # refresh token
        access_token = credentials.token

    return access_token


# checks the last modified dates of bq filter and bq metadata files from the bucket
# and fetches the file data if the cached file data is outdated
def pull_metadata():
    global bq_table_files, bq_total_entries
    status_code = 200
    token = get_access_token()
    try:
        is_bq_metadata_updated = False
        headers = {"Authorization": "Bearer {}".format(token)}
        for f in bq_table_files:
            r = requests.head(bq_table_files[f]['file_path'] + '?t=' + str(randint(1000, 9999)), headers=headers)
            r.raise_for_status()
            file_last_modified = datetime.strptime(r.headers['Last-Modified'], '%a, %d %b %Y %H:%M:%S GMT')
            if not bq_table_files[f]['file_data'] or \
                    not bq_table_files[f]['last_modified'] or (
                    bq_table_files[f]['last_modified'] and (bq_table_files[f]['last_modified'] < file_last_modified)):
                bq_table_files[f]['last_modified'] = file_last_modified
                bq_table_files[f]['file_data'] = requests.get(bq_table_files[f]['file_path'], headers=headers).json()
                if f == 'bq_metadata':
                    is_bq_metadata_updated = (not is_bq_metadata_updated)
        bq_total_entries = len(bq_table_files['bq_metadata']['file_data']) if bq_table_files['bq_metadata'][
            'file_data'] else 0
        error_message = None
    except Exception as e:
        logger.error("[ERROR] While trying to pull metadata:")
        logger.exception(e)
        status_code = 500
        error_message = "Error"
        if isinstance(e, (requests.exceptions.HTTPError, requests.exceptions.Timeout, requests.exceptions.ConnectionError)):
            status_code = e.response.status_code
            error_message = type(e).__name__
    message = None
    if status_code != 200:
        bq_table_files['bq_filters']['file_data'] = None
        bq_table_files['bq_metadata']['file_data'] = None
        bq_total_entries = 0
        message = f'ERROR While attempting to retrieve BQ metadata file: [{status_code}] {error_message}'
    return message