Commit b6d3857b authored by Robin Knapp's avatar Robin Knapp

blueprints now cached

parent 53defcd2
......@@ -16,50 +16,8 @@ ENV READ_SPECS_FROM_DISK=False
ENV READ_BLUEPRINTS_FROM_DISK=False
ENV PLATFORM_ENDPOINT=172.18.1.9:1026
ENV PLATFORM_ENDPOINT_HISTORIC=172.18.1.9:1026
# URL resources
ENV SPECS_BASE_PATH=https://gitlab.distantaccess.com/naiades/dmv_public/-/raw/master/specs
#________________________________________
ENV ALERT_SPECS_URL=$SPECS_BASE_PATH/Specs_Alert.json
ENV V2_ALERT_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_Alert.json
ENV LD_ALERT_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_Alert.json
ENV DEVICE_SPECS_URL=$SPECS_BASE_PATH/Specs_Device.json
ENV V2_DEVICE_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_Device.json
ENV LD_DEVICE_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_Device.json
ENV FLOWERBED_SPECS_URL=$SPECS_BASE_PATH/Specs_FlowerBed.json
ENV V2_FLOWERBED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_FlowerBed.json
ENV LD_FLOWERBED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_FlowerBed.json
ENV FOUNTAINUSAGEOBSERVED_SPECS_URL=$SPECS_BASE_PATH/Specs_FountainUsageObserved.json
ENV V2_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_FountainUsageObserved.json
ENV LD_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_FountainUsageObserved.json
ENV NOISE_SPECS_URL=$SPECS_BASE_PATH/Specs_Noise.json
ENV V2_NOISE_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_Noise.json
ENV LD_NOISE_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_Noise.json
ENV WATERCONSUMPTIONOBSERVED_SPECS_URL=$SPECS_BASE_PATH/Specs_WaterConsumptionObserved.json
ENV V2_WATERCONSUMPTIONOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WaterConsumptionObserved.json
ENV LD_WATERCONSUMPTIONOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WaterConsumptionObserved.json
ENV WATERQUALITYFORECAST_SPECS_URL=$SPECS_BASE_PATH/Specs_WaterQualityForecast.json
ENV V2_WATERQUALITYFORECAST_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WaterQualityForecast.json
ENV LD_WATERQUALITYFORECAST_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WaterQualityForecast.json
ENV WATERQUALITYOBSERVED_SPECS_URL=$SPECS_BASE_PATH/Specs_WaterQualityObserved.json
ENV V2_WATERQUALITYOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WaterQualityObserved.json
ENV LD_WATERQUALITYOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WaterQualityObserved.json
ENV WEATHERFORECAST_SPECS_URL=$SPECS_BASE_PATH/Specs_WeatherForecast.json
ENV V2_WEATHERFORECAST_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WeatherForecast.json
ENV LD_WEATHERFORECAST_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WeatherForecast.json
ENV WEATHEROBSERVED_SPECS_URL=$SPECS_BASE_PATH/Specs_WeatherObserved.json
ENV V2_WEATHEROBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WeatherObserved.json
ENV LD_WEATHEROBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WeatherObserved.json
#________________________________________
RUN useradd -ms /bin/bash naiades
......
......@@ -32,14 +32,12 @@ def create_app():
from src.api import develop_config
log_file_handler = TimedRotatingFileHandler(filename="dmv_log", when='M', interval=1, backupCount=5, utc=True)
app.config.from_mapping(develop_config.APP_CONFIG)
app.config['URL_RESOURCES'] = develop_config.URL_RESOURCES
@app.route("/dev/<path:url>", methods=["GET"])
def read_dev(url):
return jsonify(code=200, dev=True)
else:
app.config.from_mapping(config.APP_CONFIG)
app.config['URL_RESOURCES'] = config.URL_RESOURCES
log_file_handler = TimedRotatingFileHandler(filename=config.LOG_CONFIG['LOG_FILE_NAME'],
when=config.LOG_CONFIG['LOG_WHEN'],
backupCount=config.LOG_CONFIG['LOG_BACKUP_COUNT'],
......
......@@ -2,7 +2,39 @@
import os
DEPLOYMENT_ENV = 'PROD' # PROD or DEV
SPECS_BASE_PATH = "https://gitlab.distantaccess.com/naiades/dmv_public/-/raw/master/specs"
def get_resource_urls() -> dict:
"""
Dynamically builds and returns a dict from `config.USED_MODELS` that contains the urls which are needed for
resource retrieval of the specs and blueprint files from remote host.
Returns:
urls: dict with the URL mapping. Example.: {'SPECS_ALERT_URL': '<uri>/Specs_Alert.json',
"""
urls = {}
# Specs_FlowerBed.json
specs_file_convention = RESOURCE_CONFIG["NAME_CONVENTIONS"]["FILE_NAME_CONVENTIONS"]["SPECS"]
blueprint_file_convention = RESOURCE_CONFIG["NAME_CONVENTIONS"]["FILE_NAME_CONVENTIONS"]["BLUEPRINTS"]
# SPECS_FLOWERBED_URL
specs_url_convention = RESOURCE_CONFIG["NAME_CONVENTIONS"]["URL_CONFIGNAME_CONVENTIONS"]["SPECS"]
blueprint_url_convention = RESOURCE_CONFIG["NAME_CONVENTIONS"]["URL_CONFIGNAME_CONVENTIONS"]["BLUEPRINTS"]
for modelname in USED_MODELS:
specs_url = specs_url_convention.format(modelname=modelname.upper())
blueprint_ld_url = blueprint_url_convention.format(modeltype='LD', modelname=modelname.upper())
blueprint_v2_url = blueprint_url_convention.format(modeltype='V2', modelname=modelname.upper())
urls[specs_url] = f'{SPECS_BASE_PATH}/{specs_file_convention.format(modelname=modelname)}'
urls[blueprint_ld_url] = f'{SPECS_BASE_PATH}/{blueprint_file_convention.format(modeltype="LD",modelname=modelname)}'
urls[blueprint_v2_url] = f'{SPECS_BASE_PATH}/{blueprint_file_convention.format(modeltype="V2",modelname=modelname)}'
return urls
DEPLOYMENT_ENV = 'DEV' # PROD or DEV
APP_CONFIG = {
'SPECS_TIMEOUT_SEC': os.environ.get('SPECS_TIMEOUT_SEC', 10),
......@@ -16,43 +48,33 @@ APP_CONFIG = {
}
URL_RESOURCES = {
"URL_NAME_CONVENTIONS": {
"SPECS": '{modelname}_SPECS_URL',
"BLUEPRINTS": '{modeltype}_{modelname}_BLUEPRINT_URL'
},
"ALERT_SPECS_URL": os.environ.get("ALERT_SPECS_URL"),
"V2_ALERT_BLUEPRINT_URL": os.environ.get("V2_ALERT_BLUEPRINT_URL"),
"LD_ALERT_BLUEPRINT_URL": os.environ.get("LD_ALERT_BLUEPRINT_URL"),
"DEVICE_SPECS_URL": os.environ.get("DEVICE_SPECS_URL"),
"V2_DEVICE_BLUEPRINT_URL": os.environ.get("V2_DEVICE_BLUEPRINT_URL"),
"LD_DEVICE_BLUEPRINT_URL": os.environ.get("LD_DEVICE_BLUEPRINT_URL"),
"FLOWERBED_SPECS_URL": os.environ.get("FLOWERBED_SPECS_URL"),
"V2_FLOWERBED_BLUEPRINT_URL": os.environ.get("V2_FLOWERBED_BLUEPRINT_URL"),
"LD_FLOWERBED_BLUEPRINT_URL": os.environ.get("LD_FLOWERBED_BLUEPRINT_URL"),
"FOUNTAINUSAGEOBSERVED_SPECS_URL": os.environ.get("FOUNTAINUSAGEOBSERVED_SPECS_URL"),
"V2_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL": os.environ.get("V2_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL"),
"LD_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL": os.environ.get("LD_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL"),
"NOISE_SPECS_URL": os.environ.get("NOISE_SPECS_URL"),
"V2_NOISE_BLUEPRINT_URL": os.environ.get("V2_NOISE_BLUEPRINT_URL"),
"LD_NOISE_BLUEPRINT_URL": os.environ.get("LD_NOISE_BLUEPRINT_URL"),
"WATERCONSUMPTIONOBSERVED_SPECS_URL": os.environ.get("WATERCONSUMPTIONOBSERVED_SPECS_URL"),
"V2_WATERCONSUMPTIONOBSERVED_BLUEPRINT_URL": os.environ.get("V2_FLOWERBED_BLUEPRINT_URL"),
"LD_WATERCONSUMPTIONOBSERVED_BLUEPRINT_URL": os.environ.get("LD_FLOWERBED_BLUEPRINT_URL"),
"WATERQUALITYFORECAST_SPECS_URL": os.environ.get("WATERQUALITYFORECAST_SPECS_URL"),
"V2_WATERQUALITYFORECAST_BLUEPRINT_URL": os.environ.get("V2_WATERQUALITYFORECAST_BLUEPRINT_URL"),
"LD_WATERQUALITYFORECAST_BLUEPRINT_URL": os.environ.get("LD_WATERQUALITYFORECAST_BLUEPRINT_URL"),
"WATERQUALITYOBSERVED_SPECS_URL": os.environ.get("WATERQUALITYOBSERVED_SPECS_URL"),
"V2_WATERQUALITYOBSERVED_BLUEPRINT_URL": os.environ.get("V2_WATERQUALITYOBSERVED_BLUEPRINT_URL"),
"LD_WATERQUALITYOBSERVED_BLUEPRINT_URL": os.environ.get("LD_WATERQUALITYOBSERVED_BLUEPRINT_URL"),
"WEATHERFORECAST_SPECS_URL": os.environ.get("WEATHERFORECAST_SPECS_URL"),
"V2_WEATHERFORECAST_BLUEPRINT_URL": os.environ.get("V2_WEATHERFORECAST_BLUEPRINT_URL"),
"LD_WEATHERFORECAST_BLUEPRINT_URL": os.environ.get("LD_WEATHERFORECAST_BLUEPRINT_URL"),
"WEATHEROBSERVED_SPECS_URL": os.environ.get("WEATHEROBSERVED_SPECS_URL"),
"V2_WEATHEROBSERVED_BLUEPRINT_URL": os.environ.get("V2_WEATHEROBSERVED_BLUEPRINT_URL"),
"LD_WEATHEROBSERVED_BLUEPRINT_URL": os.environ.get("LD_WEATHEROBSERVED_BLUEPRINT_URL")
RESOURCE_CONFIG = {
# example: modelname = FlowerBed
# example: modeltype = LD / V2
"NAME_CONVENTIONS": {
# Case sensitive
"FILE_NAME_CONVENTIONS": {
"SPECS": 'Specs_{modelname}.json',
"BLUEPRINTS": "{modeltype}_Blueprint_{modelname}.json",
},
# Upper Case
"URL_CONFIGNAME_CONVENTIONS": {
"SPECS": 'SPECS_{modelname}_URL',
"BLUEPRINTS": '{modeltype}_BLUEPRINT_{modelname}_URL'
},
# Upper Case
"CACHE_NAME_CONVENTIONS": {
"SPECS": 'SPECS_{modelname}',
"BLUEPRINTS": "{modeltype}_BLUEPRINT_{modelname}",
}
}
}
USED_MODELS = ["Alert", "Device", "FlowerBed", "FountainUsageObserved", "Noise", "WaterConsumptionObserved",
"WaterQualityForecast", "WaterQualityObserved", "WeatherForecast", "WeatherObserved"]
URL_RESOURCES = get_resource_urls()
LOG_CONFIG = {
'LOG_FILE_NAME': os.environ.get('LOG_FILE_NAME', 'dmv_log'),
'LOG_WHEN': os.environ.get('LOG_FILE_NAME', 'midnight'),
......
......@@ -8,12 +8,16 @@ from typing import Union
from src.api.server_utils import ValidationErrorResponse, extract_modelname_from_url, Connector, Converter, \
send_to_platform
from src.logic import ValidationException, Creator, Updater
from src.api.config import RESOURCE_CONFIG
CURRENT_V2_ROUTER = 'CURRENT_V2_ROUTER'
HISTORIC_V2_ROUTER = 'HISTORIC_V2_ROUTER'
CURRENT_LD_ROUTER = 'CURRENT_LD_ROUTER'
HISTORIC_LD_ROUTER = 'HISTORIC_LD_ROUTER'
CACHE_SPECS_NAME_CONVENTION = RESOURCE_CONFIG["NAME_CONVENTIONS"]["CACHE_NAME_CONVENTIONS"]["SPECS"]
CACHE_BLUEPRINTS_NAME_CONVENTION = RESOURCE_CONFIG["NAME_CONVENTIONS"]["CACHE_NAME_CONVENTIONS"]["BLUEPRINTS"]
validation_v2_entities = Blueprint(CURRENT_V2_ROUTER, __name__, url_prefix='/validation/v2/entities')
historic_validation_v2_entities = Blueprint(HISTORIC_V2_ROUTER, __name__,
url_prefix='/historic/validation/v2/entities')
......@@ -23,6 +27,23 @@ historic_validation_ld_entities = Blueprint(HISTORIC_LD_ROUTER, __name__,
url_prefix='/historic/validation/ld/entities')
def load_resource(cache_key: str, reads_from_disk: bool, load_specs: bool, model_type: str, model_version=None):
cache = current_app.config.get('APP_CACHE')
if cache.get(cache_key):
return cache.get(cache_key)
else: # load specs from resource
conn = Connector(reads_from_disk=reads_from_disk)
if load_specs:
resource = conn.load_specs(model_type=model_type)
cache.set(cache_key, resource)
current_app.logger.warning(f'<<{__name__}>> write specs to cache')
else:
resource = conn.load_blueprints(model_type=model_type, model_version=model_version)
cache.set(cache_key, resource)
current_app.logger.warning(f'<<{__name__}>> write Blueprint to cache')
return resource
def validate_post_v2(model: dict, cache: flask_caching.Cache):
"""
Loads the Blueprints and Specs consecutively and evaluates the `model` for a POST request.
......@@ -39,20 +60,33 @@ def validate_post_v2(model: dict, cache: flask_caching.Cache):
if not model_type:
raise ValidationException(extern=f'Model is missing the "type" attribute', log=False)
if cache.get(model_type.lower()):
specs = cache.get(model_type.lower())
# load specs from cache or url
specs_cache_key = CACHE_SPECS_NAME_CONVENTION.format(modelname=model_type.upper())
if cache.get(specs_cache_key):
specs = cache.get(specs_cache_key)
else: # load specs from resource
conn = Connector(reads_from_disk=current_app.config['READ_SPECS_FROM_DISK'])
specs = conn.load_specs(model_type=model_type)
cache.set(model_type.lower(), specs)
current_app.logger.warning(f'<<{__name__}>> write specs to cache')
cache.set(specs_cache_key, specs)
current_app.logger.warning(f'<<{__name__}>> write Specs to cache')
current_app.logger.debug(f'<<{__name__}>> Specs retrieved')
# v2 models need to be converted to kv models for validation
if request.args.get('options', None) is None:
conn = Connector(reads_from_disk=current_app.config['READ_BLUEPRINTS_FROM_DISK'])
blueprint = conn.load_blueprints(model_type=model_type, model_version='v2')
# load blueprints from cache or url
blueprint_cache_key = CACHE_BLUEPRINTS_NAME_CONVENTION.format(modeltype="V2", modelname=model_type.upper())
if cache.get(blueprint_cache_key):
blueprint = cache.get(blueprint_cache_key)
else:
conn = Connector(reads_from_disk=current_app.config['READ_BLUEPRINTS_FROM_DISK'])
blueprint = conn.load_blueprints(model_type=model_type, model_version='v2')
cache.set(blueprint_cache_key, blueprint)
current_app.logger.warning(f'<<{__name__}>> write Blueprint to cache')
current_app.logger.debug(f'<<{__name__}>> Blueprints retrieved')
model = Converter.convert_v2_to_kv(v2_model=model, blueprint=blueprint, updating=False)
logic = Creator(specs=specs, model=model, request=request)
logic.workflow()
......@@ -122,20 +156,40 @@ def validate_patch_v2(model: dict, model_type: str, cache: flask_caching.Cache):
"""
try:
if cache.get(model_type.lower()):
specs = cache.get(model_type.lower())
# load specs from cache or url
specs_cache_key = CACHE_SPECS_NAME_CONVENTION.format(modelname=model_type.upper())
# TODO load_resource() verwenden
# specs = load_resource(cache_key=CACHE_SPECS_NAME_CONVENTION.format(modelname=model_type.upper()),
# reads_from_disk=current_app.config['READ_SPECS_FROM_DISK'],
# model_type=model_type,
# load_specs=True)
if cache.get(specs_cache_key):
specs = cache.get(specs_cache_key)
else: # load specs from resource
conn = Connector(reads_from_disk=current_app.config['READ_SPECS_FROM_DISK'])
specs = conn.load_specs(model_type=model_type)
cache.set(model_type.lower(), specs)
cache.set(specs_cache_key, specs)
current_app.logger.warning(f'<<{__name__}>> write specs to cache')
current_app.logger.debug(f'<<{__name__}>> Specs retrieved')
# v2 models need to be converted to kv models for validation
if not request.args.get('options', None):
conn = Connector(reads_from_disk=current_app.config['READ_BLUEPRINTS_FROM_DISK'])
blueprint = conn.load_blueprints(model_type=model_type, model_version='v2')
if not request.args.get('options'):
# load blueprints from cache or url
blueprint_cache_key = CACHE_BLUEPRINTS_NAME_CONVENTION.format(modeltype="V2", modelname=model_type.upper())
if cache.get(blueprint_cache_key):
blueprint = cache.get(blueprint_cache_key)
else:
conn = Connector(reads_from_disk=current_app.config['READ_BLUEPRINTS_FROM_DISK'])
blueprint = conn.load_blueprints(model_type=model_type, model_version='v2')
cache.set(blueprint_cache_key, blueprint)
current_app.logger.warning(f'<<{__name__}>> write Blueprint to cache')
current_app.logger.debug(f'<<{__name__}>> Blueprints retrieved')
model = Converter.convert_v2_to_kv(v2_model=model, blueprint=blueprint, updating=True)
# use kv-version for evaluating the update
......
......@@ -12,6 +12,7 @@ from flask import current_app, redirect, request
from werkzeug.exceptions import HTTPException
from src.logic import ValidationException, ValidationLogic, FiwareElement
from src.api.config import RESOURCE_CONFIG, URL_RESOURCES
class Connector(object):
......@@ -202,9 +203,9 @@ class Connector(object):
"""
try:
specs_key = current_app.config["URL_RESOURCES"]["URL_NAME_CONVENTIONS"]["SPECS"]
specs_key = specs_key.format(modelname=model_type.upper()) # {modelname}_SPECS_URL
url = current_app.config["URL_RESOURCES"][specs_key]
specs_url = RESOURCE_CONFIG["NAME_CONVENTIONS"]["URL_CONFIGNAME_CONVENTIONS"]["SPECS"]
specs_url = specs_url.format(modelname=model_type.upper()) # 'SPECS_{modelname}_URL'
url = URL_RESOURCES[specs_url]
current_app.logger.warning(f'<<{__name__}>> load specs from {url}')
return self.load_json_from_url(url=url)
......@@ -232,10 +233,10 @@ class Connector(object):
"""
try:
bp_key = current_app.config["URL_RESOURCES"]["URL_NAME_CONVENTIONS"]["BLUEPRINTS"]
bp_key = bp_key.format(modeltype=model_version.upper(),
modelname=model_type.upper()) # '{modeltype}_{modelname}_BLUEPRINT_URL'
url = current_app.config["URL_RESOURCES"][bp_key]
blueprint_url = current_app.config["URL_RESOURCES"]["URL_NAME_CONVENTIONS"]["BLUEPRINTS"]
blueprint_url = blueprint_url.format(modeltype=model_version.upper(),
modelname=model_type.upper()) # '{modeltype}_BLUEPRINT_{modelname}_URL'
url = URL_RESOURCES[blueprint_url]
current_app.logger.warning(f'<<{__name__}>> load blueprint from {url}')
return self.load_json_from_url(url=url)
except KeyError as kerr:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment