Commit ae0579cc authored by Robin Knapp's avatar Robin Knapp

WIP: ld integration and optimized resource loading

parent d92989ed
{
"location": {
"nav": "type==type&value=value",
"type": "GeoProperty"
},
"refDevice" : {
"nav": "type==type&value=object",
"ldType": "Relationship"
},
"flowerType": {
"nav": "type==type&value=value",
"type": "Property"
},
"category": {
"nav": "type==type&value=value",
"type": "Property"
},
"width" : {
"nav": "type==type&value=value",
"type": "Property"
},
"dateLastWatering": {
"nav": "type==type&value=value",
"type": "Property"
},
"@context": {
"type" : "StructuredValue"
}
}
\ No newline at end of file
......@@ -110,4 +110,9 @@ def create_app():
""" For developing purposes: Executed after every request, regardless of any occurring error. """
app.logger.debug('Teardown request')
# @app.before_first_request
# def load_validation_context
# load specs und blueprints at server start
# TODO create central function to load context files
return app
......@@ -10,8 +10,14 @@ APP_CONFIG = {
'PLATFORM_ENDPOINT': os.environ.get('PLATFORM_ENDPOINT'),
'PLATFORM_ENDPOINT_HISTORIC': os.environ.get('PLATFORM_ENDPOINT_HISTORIC'),
'BLUEPRINTS_ABS_PATH': os.environ.get('BLUEPRINTS_ABS_PATH', '/home/naiades/specs/'),
'SPECS_ABS_PATH': os.environ.get('SPECS_ABS_PATH', '/home/naiades/specs/'),
# URL resources
'SPECS_ABS_PATH': os.environ.get('SPECS_ABS_PATH', '/home/naiades/specs/')
}
URL_RESOURCES = {
"URL_NAME_CONVENTIONS": {
"SPECS": 'MODELNAME_SPECS_URL',
"BLUEPRINTS": 'MODELTYPE_MODELNAME_BLUEPRINTS_URL'
},
"ALERT_SPECS_URL": os.environ.get("ALERT_SPECS_URL"),
"V2_ALERT_BLUEPRINT_URL": os.environ.get("V2_ALERT_BLUEPRINT_URL"),
"DEVICE_SPECS_URL": os.environ.get("DEVICE_SPECS_URL"),
......
This diff is collapsed.
import ast
import json
import flask_caching
import ast
from flask import Blueprint, request, current_app, redirect
from werkzeug.exceptions import HTTPException
from flask import Blueprint, request, current_app
from src.api.server_utils import ValidationErrorResponse, extract_modelname_from_url, Connector, Converter
from src.api.server_utils import ValidationErrorResponse, extract_modelname_from_url, Connector, Converter, \
send_to_platform
from src.logic import ValidationException, Creator, Updater
validation_v2_entities = Blueprint('val_v2_ent', __name__, url_prefix='/validation/v2/entities')
......@@ -93,25 +94,6 @@ def validate_patch_v2(model: dict, model_type: str, cache: flask_caching.Cache):
log=False)
def send_to_platform(forward_url: str):
"""
Redirects the request context (body, header) to the IOT platform.
Args:
forward_url: Full uri of the platform (including host, port, path, params, ...)
Raises:
ValidationErrorResponse: Response for any occurring HTTPException
"""
current_app.logger.info(f'<<{__name__}>> Trying to forward request to {forward_url}')
try:
# redirect to preserve context of request (header, body, ...)
return redirect(f'http://{forward_url}', code=307)
except HTTPException:
raise ValidationErrorResponse(msg=f'Redirect for {current_app.config["REDIRECT_ENDPOINT"]} failed.',
module=__name__)
@historic_validation_v2_entities.route('/<path:url>', methods=["GET"])
@validation_v2_entities.route('/<path:url>', methods=["GET"])
def get_validation_v2_entities(url):
......@@ -160,7 +142,7 @@ def put_validation_v2_entities(url):
validate_patch_v2(model=model, model_type="", cache=current_app.config.get('APP_CACHE'))
except ValidationException as vex:
raise ValidationErrorResponse(msg=f'Json-model failed the evaluation. Error message: {vex.extern_logging}'
,log=False)
, log=False)
return send_to_platform(forward_url=forward_url)
......@@ -203,7 +185,6 @@ def patch_validation_v2_entities(url):
@historic_validation_v2_entities.route('/<path:url>', methods=["POST"])
@validation_v2_entities.route('/<path:url>', methods=["POST"])
def post_validation_v2_entities(url):
model = ast.literal_eval(request.data.decode('UTF-8')) # dict or list
if request.blueprint == 'val_v2_ent':
......
......@@ -4,13 +4,14 @@ import random
import re
import string
from json import load, JSONDecodeError
from typing import Any
from typing import Any, NewType
from enum import Enum
import requests
from flask import current_app, request
from flask import current_app, request, redirect
from werkzeug.exceptions import HTTPException
from src.logic import ValidationException
from src.logic import ValidationException, ValidationLogic
from src.api import config
class Connector(object):
......@@ -206,7 +207,8 @@ class Connector(object):
except KeyError as kerr:
current_app.logger.error(f'{model_type.upper()}_SPECS_URL is undefined')
raise ValidationException(extern=f"'{model_type}' is undefined.", module=__name__, func='load_specs_from_url')
raise ValidationException(extern=f"'{model_type}' is undefined.", module=__name__,
func='load_specs_from_url')
except ValidationException as v_exc:
raise v_exc
......@@ -314,6 +316,137 @@ class Converter(object):
f' "{v2_model_key}"', func='Converter.convert_v2_to_kv', module=__name__)
return kv_model
@staticmethod
def convert_ld_to_kv(ld_model: dict, blueprint: dict, updating=False) -> dict:
"""
Converts a Fiware-LD into Fiware-KeyValue model.
Args:
ld_model: Case sensitive type of fiware model e.g. FlowerBed
blueprint: Blueprint that contains the conversion metainfo
updating: Indicates if PATCH or POST http method was used for request
Returns:
KeyValue model as python dict
Raises:
ValidationException: in error case with according description
"""
kv_model = {}
# updating model can miss `id` and `type`
if not updating:
if not ld_model.get('id'):
raise ValidationException(extern=f'Entity id is missing in model', module=__name__,
func='Converter.convert_ld_to_kv')
if not ld_model.get('type'):
raise ValidationException(extern=f'Entity type is missing in model', module=__name__,
func='Converter.convert_ld_to_kv')
kv_model = {'id': ld_model.pop('id'),
"type": ld_model.pop('type'),
}
# check and delete @context from ld-model
if not ld_model.get('@context'):
raise ValidationException(extern=f'LD-Model must contain `@context attribute`', module=__name__,
func='Converter.convert_ld_to_kv')
val = ValidationLogic(None, None, None)
val.parse_expression(expression=f'FiwareElement({blueprint["@context"]["type"].lower()}={ld_model["@context"]}')
ld_model.pop('@context')
# iterate model and check attributes with blueprints & add attributes to kv-model
try:
for ld_model_key, ld_model_value in ld_model.items():
nav = blueprint[ld_model_key]['nav']
cmds = nav.split('&')
for cmd in cmds:
if '==' in cmd:
blueprint_attr, ld_model_attr = cmd.split('==')
if not blueprint[ld_model_key][blueprint_attr] == ld_model_value[ld_model_attr]:
raise ValidationException(extern=f'{ld_model[ld_model_attr]} is not compliant to model.'
, func='Converter.convert_ld_to_kv', module=__name__)
elif '=' in cmd:
kv_model_attr, ld_model_attr = cmd.split('=') # value, value/object
kv_model[ld_model_key] = ld_model_value[ld_model_attr]
else:
current_app.logger.debug(f"Error in {cmd}. Does not contain a comparison or assignment")
raise ValidationException(extern=f'Internal error. Please contact admin.'
, func='Converter.convert_v2_to_kv', module=__name__)
except KeyError:
raise ValidationException(extern=f'Attributes are not compliant with model. Caused by attribute'
f' "{ld_model_key}"', func='Converter.convert_v2_to_kv', module=__name__)
return kv_model
def load_url_resources(model_name: str = '*'):
"""
Function loads `URL_RESOURCES` from src.api.config onto the flask-app cache. Default behaviour is to load all
resources. Can also be used to load the LD- AND V2-Blueprint AND Specs for the given model.
Args:
model_name: Case sensitive type of fiware model e.g. FlowerBed
Raises:
ValidationException: in error case during loading
"""
current_app.logger.warning(f'<<{__name__}>> Start loading url resources for `{model_name}` to cache.')
conn = Connector(reads_from_disk=current_app.config['READ_SPECS_FROM_DISK'])
if model_name == '*': # load all models
for resource_name, url in config.URL_RESOURCES.items():
resource = conn.load_json_from_url(url=url)
current_app.config['APP_CACHE'].set(resource_name.lower(), resource)
current_app.logger.debug(f'<<{__name__}>> All Url resources were loaded to cache.')
else: # load resources for one model
specs = conn.load_specs(model_type=model_name)
blueprints_ld = conn.load_blueprints(model_type=model_name, model_version='ld')
blueprints_v2 = conn.load_blueprints(model_type=model_name, model_version='v2')
url_name_conventions = config.URL_RESOURCES['URL_NAME_CONVENTIONS']
specs_key = url_name_conventions['SPECS'].replace('MODELNAME', model_name)
ld_blueprint_key = url_name_conventions['BLUEPRINTS']\
.replace('MODELNAME', model_name)\
.replace('MODELTYPE', 'LD')
v2_blueprint_key = url_name_conventions['BLUEPRINTS'] \
.replace('MODELNAME', model_name) \
.replace('MODELTYPE', 'V2')
current_app.config['APP_CACHE'].set(specs_key.lower(), specs)
current_app.config['APP_CACHE'].set(ld_blueprint_key.lower(), blueprints_ld)
current_app.config['APP_CACHE'].set(v2_blueprint_key.lower(), blueprints_v2)
current_app.logger.debug(f'<<{__name__}>> Url resources for `{model_name}` loaded to cache.')
def send_to_platform(forward_url: str):
"""
Redirects the request context (body, header) to the IOT platform.
Args:
forward_url: Full uri of the platform (including host, port, path, params, ...)
Raises:
ValidationErrorResponse: Response for any occurring HTTPException
"""
current_app.logger.info(f'<<{__name__}>> Trying to forward request to {forward_url}')
try:
# redirect to preserve context of request (header, body, ...)
return redirect(f'http://{forward_url}', code=307)
except HTTPException:
raise ValidationErrorResponse(msg=f'Redirect for `http://{forward_url}` failed.',
module=__name__)
def extract_modelname_from_url(url: str) -> Any:
# url looks like /v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-8/attrs
......
......@@ -503,7 +503,7 @@ class ValidationLogic(ABC):
return f'FiwareElement({fiware_type}="{model_value_}")' # e.g ex = FiwareElement(text="urn") / (number="110")
def _parse_expression(self, expression: str) -> None:
def parse_expression(self, expression: str) -> None:
""" Accepts an expression that will be evaluated.The goal is to create an FiwareElement that features an
attribute that was extracted from a fiware model. Any occurring errors during the creation will be registered
and a ValidationException will be raised. A regular expression will ensure the given expression does not
......@@ -557,7 +557,7 @@ class Updater(ValidationLogic):
# Iterate model and parse every key-value pair according to specs-dict
for model_key, model_value in self.model.items():
ex = self._prepare_expression(model_key_=model_key, model_value_=model_value)
self._parse_expression(ex)
self.parse_expression(ex)
except ValidationException as v_err:
raise ValidationException(extern=f'Parsing error occurred when evaluating "{model_key}": "{model_value}". '
......@@ -594,7 +594,7 @@ class Creator(ValidationLogic):
# Iterate model and parse every key-value pair according to specs-dict
for model_key, model_value in self.model.items():
ex = self._prepare_expression(model_key_=model_key, model_value_=model_value)
self._parse_expression(ex)
self.parse_expression(ex)
except ValidationException as v_err:
raise ValidationException(extern=f'Parsing error when evaluating "{model_key}": "{model_value}". Caused by:'
......
......@@ -358,14 +358,14 @@ class ParseExpressionTest(TestCase):
vl = ValidationLogic(specs='', model='', logger='')
self.text = "urn:"
expr = f'FiwareElement(text="{self.text}")'
vl._parse_expression(expression=expr)
vl.parse_expression(expression=expr)
def test_expression_wrong_fiware_type_should_raise_validation_exception(self):
vl = ValidationLogic(specs='', model='', logger='')
expr = 'FiwareElement(hello="world")'
expected = "__init__() got an unexpected keyword argument 'hello'"
with self.assertRaises(ValidationException) as err:
vl._parse_expression(expression=expr)
vl.parse_expression(expression=expr)
exception = err.exception
self.assertEqual(exception.intern_logging, f"Expression {expr} is unsuitable for parsing. Caused by: "
f"{expected}")
......@@ -374,7 +374,7 @@ class ParseExpressionTest(TestCase):
vl = ValidationLogic(specs='', model='', logger='')
expr = 'Element(text="world")'
with self.assertRaises(ValidationException) as err:
vl._parse_expression(expression=expr)
vl.parse_expression(expression=expr)
exception = err.exception
self.assertEqual(exception.intern_logging, f'Expression {expr} does not match pattern')
......@@ -383,7 +383,7 @@ class ParseExpressionTest(TestCase):
expr = "os.mkdir(path=sys.path[0])"
expected_error_message = f'Expression {expr} does not match pattern'
with self.assertRaises(ValidationException) as err:
vl._parse_expression(expression=expr)
vl.parse_expression(expression=expr)
exception = err.exception
self.assertEqual(exception.intern_logging, expected_error_message)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment