Commit c2146826 authored by Julian Bruns's avatar Julian Bruns

Merge branch 'feature-caching' into 'master'

Feature caching

See merge request !6
parents 5f31645d abdd80d4
......@@ -16,50 +16,8 @@ ENV READ_SPECS_FROM_DISK=False
ENV READ_BLUEPRINTS_FROM_DISK=False
ENV PLATFORM_ENDPOINT=172.18.1.9:1026
ENV PLATFORM_ENDPOINT_HISTORIC=172.18.1.9:1026
# URL resources
ENV SPECS_BASE_PATH=https://gitlab.distantaccess.com/naiades/dmv_public/-/raw/master/specs
#________________________________________
ENV ALERT_SPECS_URL=$SPECS_BASE_PATH/Specs_Alert.json
ENV V2_ALERT_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_Alert.json
ENV LD_ALERT_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_Alert.json
ENV DEVICE_SPECS_URL=$SPECS_BASE_PATH/Specs_Device.json
ENV V2_DEVICE_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_Device.json
ENV LD_DEVICE_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_Device.json
ENV FLOWERBED_SPECS_URL=$SPECS_BASE_PATH/Specs_FlowerBed.json
ENV V2_FLOWERBED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_FlowerBed.json
ENV LD_FLOWERBED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_FlowerBed.json
ENV FOUNTAINUSAGEOBSERVED_SPECS_URL=$SPECS_BASE_PATH/Specs_FountainUsageObserved.json
ENV V2_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_FountainUsageObserved.json
ENV LD_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_FountainUsageObserved.json
ENV NOISE_SPECS_URL=$SPECS_BASE_PATH/Specs_Noise.json
ENV V2_NOISE_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_Noise.json
ENV LD_NOISE_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_Noise.json
ENV WATERCONSUMPTIONOBSERVED_SPECS_URL=$SPECS_BASE_PATH/Specs_WaterConsumptionObserved.json
ENV V2_WATERCONSUMPTIONOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WaterConsumptionObserved.json
ENV LD_WATERCONSUMPTIONOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WaterConsumptionObserved.json
ENV WATERQUALITYFORECAST_SPECS_URL=$SPECS_BASE_PATH/Specs_WaterQualityForecast.json
ENV V2_WATERQUALITYFORECAST_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WaterQualityForecast.json
ENV LD_WATERQUALITYFORECAST_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WaterQualityForecast.json
ENV WATERQUALITYOBSERVED_SPECS_URL=$SPECS_BASE_PATH/Specs_WaterQualityObserved.json
ENV V2_WATERQUALITYOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WaterQualityObserved.json
ENV LD_WATERQUALITYOBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WaterQualityObserved.json
ENV WEATHERFORECAST_SPECS_URL=$SPECS_BASE_PATH/Specs_WeatherForecast.json
ENV V2_WEATHERFORECAST_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WeatherForecast.json
ENV LD_WEATHERFORECAST_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WeatherForecast.json
ENV WEATHEROBSERVED_SPECS_URL=$SPECS_BASE_PATH/Specs_WeatherObserved.json
ENV V2_WEATHEROBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/V2_Blueprint_WeatherObserved.json
ENV LD_WEATHEROBSERVED_BLUEPRINT_URL=$SPECS_BASE_PATH/LD_Blueprint_WeatherObserved.json
ENV REMOTE_SPECS_BASE_PATH=https://gitlab.distantaccess.com/naiades/dmv_public/-/raw/master/specs
# ENV SIGNATURE_SERVICE_ENDPOINT=TBD
#________________________________________
RUN useradd -ms /bin/bash naiades
......
......@@ -32,14 +32,12 @@ def create_app():
from src.api import develop_config
log_file_handler = TimedRotatingFileHandler(filename="dmv_log", when='M', interval=1, backupCount=5, utc=True)
app.config.from_mapping(develop_config.APP_CONFIG)
app.config['URL_RESOURCES'] = develop_config.URL_RESOURCES
@app.route("/dev/<path:url>", methods=["GET"])
def read_dev(url):
return jsonify(code=200, dev=True)
else:
app.config.from_mapping(config.APP_CONFIG)
app.config['URL_RESOURCES'] = config.URL_RESOURCES
log_file_handler = TimedRotatingFileHandler(filename=config.LOG_CONFIG['LOG_FILE_NAME'],
when=config.LOG_CONFIG['LOG_WHEN'],
backupCount=config.LOG_CONFIG['LOG_BACKUP_COUNT'],
......
......@@ -11,48 +11,73 @@ APP_CONFIG = {
'READ_BLUEPRINTS_FROM_DISK': bool(os.environ.get('READ_BLUEPRINTS_FROM_DISK', False)),
'PLATFORM_ENDPOINT': os.environ.get('PLATFORM_ENDPOINT'),
'PLATFORM_ENDPOINT_HISTORIC': os.environ.get('PLATFORM_ENDPOINT_HISTORIC'),
'SIGNATURE_SERVICE_ENDPOINT': '',
'BLUEPRINTS_ABS_PATH': os.environ.get('BLUEPRINTS_ABS_PATH', '/home/naiades/specs/'),
'SPECS_ABS_PATH': os.environ.get('SPECS_ABS_PATH', '/home/naiades/specs/')
'SPECS_ABS_PATH': os.environ.get('SPECS_ABS_PATH', '/home/naiades/specs/'),
'REMOTE_SPECS_BASE_PATH': os.environ.get('REMOTE_SPECS_BASE_PATH'),
}
URL_RESOURCES = {
"URL_NAME_CONVENTIONS": {
"SPECS": '{modelname}_SPECS_URL',
"BLUEPRINTS": '{modeltype}_{modelname}_BLUEPRINT_URL'
},
"ALERT_SPECS_URL": os.environ.get("ALERT_SPECS_URL"),
"V2_ALERT_BLUEPRINT_URL": os.environ.get("V2_ALERT_BLUEPRINT_URL"),
"LD_ALERT_BLUEPRINT_URL": os.environ.get("LD_ALERT_BLUEPRINT_URL"),
"DEVICE_SPECS_URL": os.environ.get("DEVICE_SPECS_URL"),
"V2_DEVICE_BLUEPRINT_URL": os.environ.get("V2_DEVICE_BLUEPRINT_URL"),
"LD_DEVICE_BLUEPRINT_URL": os.environ.get("LD_DEVICE_BLUEPRINT_URL"),
"FLOWERBED_SPECS_URL": os.environ.get("FLOWERBED_SPECS_URL"),
"V2_FLOWERBED_BLUEPRINT_URL": os.environ.get("V2_FLOWERBED_BLUEPRINT_URL"),
"LD_FLOWERBED_BLUEPRINT_URL": os.environ.get("LD_FLOWERBED_BLUEPRINT_URL"),
"FOUNTAINUSAGEOBSERVED_SPECS_URL": os.environ.get("FOUNTAINUSAGEOBSERVED_SPECS_URL"),
"V2_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL": os.environ.get("V2_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL"),
"LD_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL": os.environ.get("LD_FOUNTAINUSAGEOBSERVED_BLUEPRINT_URL"),
"NOISE_SPECS_URL": os.environ.get("NOISE_SPECS_URL"),
"V2_NOISE_BLUEPRINT_URL": os.environ.get("V2_NOISE_BLUEPRINT_URL"),
"LD_NOISE_BLUEPRINT_URL": os.environ.get("LD_NOISE_BLUEPRINT_URL"),
"WATERCONSUMPTIONOBSERVED_SPECS_URL": os.environ.get("WATERCONSUMPTIONOBSERVED_SPECS_URL"),
"V2_WATERCONSUMPTIONOBSERVED_BLUEPRINT_URL": os.environ.get("V2_FLOWERBED_BLUEPRINT_URL"),
"LD_WATERCONSUMPTIONOBSERVED_BLUEPRINT_URL": os.environ.get("LD_FLOWERBED_BLUEPRINT_URL"),
"WATERQUALITYFORECAST_SPECS_URL": os.environ.get("WATERQUALITYFORECAST_SPECS_URL"),
"V2_WATERQUALITYFORECAST_BLUEPRINT_URL": os.environ.get("V2_WATERQUALITYFORECAST_BLUEPRINT_URL"),
"LD_WATERQUALITYFORECAST_BLUEPRINT_URL": os.environ.get("LD_WATERQUALITYFORECAST_BLUEPRINT_URL"),
"WATERQUALITYOBSERVED_SPECS_URL": os.environ.get("WATERQUALITYOBSERVED_SPECS_URL"),
"V2_WATERQUALITYOBSERVED_BLUEPRINT_URL": os.environ.get("V2_WATERQUALITYOBSERVED_BLUEPRINT_URL"),
"LD_WATERQUALITYOBSERVED_BLUEPRINT_URL": os.environ.get("LD_WATERQUALITYOBSERVED_BLUEPRINT_URL"),
"WEATHERFORECAST_SPECS_URL": os.environ.get("WEATHERFORECAST_SPECS_URL"),
"V2_WEATHERFORECAST_BLUEPRINT_URL": os.environ.get("V2_WEATHERFORECAST_BLUEPRINT_URL"),
"LD_WEATHERFORECAST_BLUEPRINT_URL": os.environ.get("LD_WEATHERFORECAST_BLUEPRINT_URL"),
"WEATHEROBSERVED_SPECS_URL": os.environ.get("WEATHEROBSERVED_SPECS_URL"),
"V2_WEATHEROBSERVED_BLUEPRINT_URL": os.environ.get("V2_WEATHEROBSERVED_BLUEPRINT_URL"),
"LD_WEATHEROBSERVED_BLUEPRINT_URL": os.environ.get("LD_WEATHEROBSERVED_BLUEPRINT_URL")
def get_resource_urls() -> dict:
"""
Dynamically builds and returns a dict from `config.USED_MODELS` that contains the urls which are needed for
resource retrieval of the specs and blueprint files from remote host.
Returns:
urls: dict with the URL mapping. Example.: {'SPECS_ALERT_URL': '<uri>/Specs_Alert.json',
"""
urls = {}
# Specs_FlowerBed.json
specs_file_convention = RESOURCE_CONFIG["NAME_CONVENTIONS"]["FILE_NAME_CONVENTIONS"]["SPECS"]
blueprint_file_convention = RESOURCE_CONFIG["NAME_CONVENTIONS"]["FILE_NAME_CONVENTIONS"]["BLUEPRINTS"]
# SPECS_FLOWERBED_URL
specs_url_convention = RESOURCE_CONFIG["NAME_CONVENTIONS"]["URL_CONFIGNAME_CONVENTIONS"]["SPECS"]
blueprint_url_convention = RESOURCE_CONFIG["NAME_CONVENTIONS"]["URL_CONFIGNAME_CONVENTIONS"]["BLUEPRINTS"]
base_path = APP_CONFIG['REMOTE_SPECS_BASE_PATH']
for modelname in USED_MODELS:
specs_url = specs_url_convention.format(modelname=modelname.upper())
blueprint_ld_url = blueprint_url_convention.format(modeltype='LD', modelname=modelname.upper())
blueprint_v2_url = blueprint_url_convention.format(modeltype='V2', modelname=modelname.upper())
urls[specs_url] = f'{base_path}/{specs_file_convention.format(modelname=modelname)}'
urls[blueprint_ld_url] = f'{base_path}/{blueprint_file_convention.format(modeltype="LD",modelname=modelname)}'
urls[blueprint_v2_url] = f'{base_path}/{blueprint_file_convention.format(modeltype="V2",modelname=modelname)}'
return urls
RESOURCE_CONFIG = {
# example: modelname = FlowerBed
# example: modeltype = LD / V2
"NAME_CONVENTIONS": {
# modelname Camelcase e.g. FlowerBed,
# modeltype Uppercase e.g. LD or V2
"FILE_NAME_CONVENTIONS": {
"SPECS": 'Specs_{modelname}.json',
"BLUEPRINTS": "{modeltype}_Blueprint_{modelname}.json",
# ""
},
# modelname && modeltype Upper Case
"URL_CONFIGNAME_CONVENTIONS": {
"SPECS": 'SPECS_{modelname}_URL',
"BLUEPRINTS": '{modeltype}_BLUEPRINT_{modelname}_URL'
},
# modelname && modeltype Upper Case
"CACHE_NAME_CONVENTIONS": {
"SPECS": 'SPECS_{modelname}',
"BLUEPRINTS": "{modeltype}_BLUEPRINT_{modelname}",
}
}
}
USED_MODELS = ["Alert", "Device", "FlowerBed", "FountainUsageObserved", "Noise", "WaterConsumptionObserved",
"WaterQualityForecast", "WaterQualityObserved", "WeatherForecast", "WeatherObserved"]
URL_RESOURCES = get_resource_urls()
LOG_CONFIG = {
'LOG_FILE_NAME': os.environ.get('LOG_FILE_NAME', 'dmv_log'),
'LOG_WHEN': os.environ.get('LOG_FILE_NAME', 'midnight'),
......
This diff is collapsed.
......@@ -12,6 +12,7 @@ from flask import current_app, redirect, request
from werkzeug.exceptions import HTTPException
from src.logic import ValidationException, ValidationLogic, FiwareElement
from src.api.config import RESOURCE_CONFIG, URL_RESOURCES
class Connector(object):
......@@ -111,6 +112,9 @@ class Connector(object):
# target_dir = Connector.add_backslashes_unix_(target_dir)
target_file = f'{model_version.upper()}_Blueprint_{model_type}.json'
target_path = f'{target_dir}{target_file}'
current_app.logger.info(f'<<{__name__}.Connector.load_blueprint_from_disk>> load from disk. Target: {target_path}')
try:
return self.load_json_from_disk(abs_path=target_path)
except OSError:
......@@ -136,7 +140,7 @@ class Connector(object):
target_file = f'Specs_{model_type}.json'
target_path = f'{target_dir}{target_file}'
current_app.logger.debug(f'<<{__name__}.Connector.load_specs_from_disk>> load from disk. Target: {target_path}')
current_app.logger.info(f'<<{__name__}.Connector.load_specs_from_disk>> load from disk. Target: {target_path}')
try:
return self.load_json_from_disk(abs_path=target_path)
......@@ -202,9 +206,9 @@ class Connector(object):
"""
try:
specs_key = current_app.config["URL_RESOURCES"]["URL_NAME_CONVENTIONS"]["SPECS"]
specs_key = specs_key.format(modelname=model_type.upper()) # {modelname}_SPECS_URL
url = current_app.config["URL_RESOURCES"][specs_key]
specs_url = RESOURCE_CONFIG["NAME_CONVENTIONS"]["URL_CONFIGNAME_CONVENTIONS"]["SPECS"]
specs_url = specs_url.format(modelname=model_type.upper()) # 'SPECS_{modelname}_URL'
url = URL_RESOURCES[specs_url]
current_app.logger.warning(f'<<{__name__}>> load specs from {url}')
return self.load_json_from_url(url=url)
......@@ -232,10 +236,11 @@ class Connector(object):
"""
try:
bp_key = current_app.config["URL_RESOURCES"]["URL_NAME_CONVENTIONS"]["BLUEPRINTS"]
bp_key = bp_key.format(modeltype=model_version.upper(),
modelname=model_type.upper()) # '{modeltype}_{modelname}_BLUEPRINT_URL'
url = current_app.config["URL_RESOURCES"][bp_key]
i = 0
blueprint_url = RESOURCE_CONFIG["NAME_CONVENTIONS"]["URL_CONFIGNAME_CONVENTIONS"]["BLUEPRINTS"]
blueprint_url = blueprint_url.format(modeltype=model_version.upper(),
modelname=model_type.upper()) # '{modeltype}_BLUEPRINT_{modelname}_URL'
url = URL_RESOURCES[blueprint_url]
current_app.logger.warning(f'<<{__name__}>> load blueprint from {url}')
return self.load_json_from_url(url=url)
except KeyError as kerr:
......@@ -414,7 +419,7 @@ def load_url_resources(model_name: str = '*'):
ValidationException: in error case during loading
"""
current_app.logger.warning(f'<<{__name__}>> Start loading url resources for `{model_name}` to cache.')
current_app.logger.info(f'<<{__name__}>> Start loading url resources for `{model_name}` to cache.')
conn = Connector(reads_from_disk=current_app.config['READ_SPECS_FROM_DISK'])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment