Commit bb0b44e5 authored by Robin Knapp's avatar Robin Knapp

create first draft of ld-blueprints

parent 86521f04
{
"source" : {
"nav": "type==type&value=value",
"type": "Property"
},
"dataProvider" : {
"nav": "type==type&value=value",
"type": "Property"
},
"category": {
"nav": "type==type&value=value",
"type": "Property"
},
"subCategory": {
"nav": "type==type&value=value",
"type": "Property"
},
"pHAlarm" : {
"nav": "type==type&value=value",
"type": "Property"
},
"pHPredictionAlarm" : {
"nav": "type==type&value=value",
"type": "Property"
},
"chlorinePredictionAlarm" : {
"nav": "type==type&value=value",
"type": "Property"
},
"chlorineAlarm" : {
"nav": "type==type&value=value",
"type": "Property"
},
"salineDetection" : {
"nav": "type==type&value=value",
"type": "Property"
},
"leakagesDetection" : {
"nav": "type==type&value=value",
"type": "Property"
},
"location" : {
"nav": "type==type&value=value",
"type": "GeoProperty"
},
"address" : {
"nav": "type==type&value=value",
"type": "Property"
},
"dateIssued": {
"nav": "type==type&value=value",
"type": "Property"
},
"validFrom": {
"nav": "type==type&value=value",
"type": "Property"
},
"validTo": {
"nav": "type==type&value=value",
"type": "Property"
},
"description" : {
"nav": "type==type&value=value",
"type": "Property"
},
"alertSource" : {
"nav": "type==type&value=value",
"type": "Property"
},
"data" : {
"nav": "type==type&value=value",
"type": "Property"
},
"serverity" : {
"nav": "type==type&value=value",
"type": "Property"
}
}
{
"dateObserved" : {
"nav": "type==type&value=value",
"type": "Property"
},
"location" : {
"nav": "type==type&value=value",
"type": "GeoProperty"
},
"observations" : {
"nav": "type==type&value=value",
"type": "Property"
},
"temperature" : {
"nav": "type==type&value=value",
"type": "Property"
},
"temperatureWater" : {
"nav": "type==type&value=value",
"type": "Property"
},
"chlorineContent" : {
"nav": "type==type&value=value",
"type": "Property"
},
"phWater" : {
"nav": "type==type&value=value",
"type": "Property"
},
"waterRecycle" : {
"nav": "type==type&value=value",
"type": "Property"
},
"swimmers" : {
"nav": "type==type&value=value",
"type": "Property"
},
"animals" : {
"nav": "type==type&value=value",
"type": "Property"
},
"controlHours" : {
"nav": "type==type&value=value",
"type": "Property"
}
}
{
"value": {
"nav": "type==type&value=value",
"type": "Property"
},
"serialNumber": {
"nav": "type==type&value=value",
"type": "Property"
},
"name": {
"nav": "type==type&value=value",
"type": "Property"
},
"controlledProperty": {
"nav": "type==type&value=value",
"type": "Property"
},
"location" : {
"nav": "type==type&value=value",
"type": "GeoProperty"
}
}
\ No newline at end of file
{
"waterConsumption" : {
"nav": "type==type&value=value",
"type": "Property"
},
"name" : {
"nav": "type==type&value=value",
"type": "Property"
},
"location": {
"nav": "type==type&value=value",
"type": "GeoProperty"
},
"domainActivity": {
"nav": "type==type&value=value",
"type": "Property"
},
"numberOfUsers" : {
"nav": "type==type&value=value",
"type": "Property"
}
}
{
"treatmentChlorinationDosagePrediction" : {
"nav": "type==type&value=value",
"type": "Property"
},
"treatmentFiltrationTimePrediction" : {
"nav": "type==type&value=value",
"type": "Property"
},
"treatmentCoagulantDosagePrediction": {
"nav": "type==type&value=value",
"type": "Property"
},
"chlorinationDosageRecommendation" : {
"nav": "type==type&value=value",
"type": "Property"
},
"coagulantDosageRecommendation" : {
"nav": "type==type&value=value",
"type": "Property"
},
"filtrationTimeRecommendation": {
"nav": "type==type&value=value",
"type": "Property"
},
"pHPrediction" : {
"nav": "type==type&value=value",
"type": "Property"
},
"totalChlorinePrediction" : {
"nav": "type==type&value=value",
"type": "Property"
},
"freeChlorinePrediction": {
"nav": "type==type&value=value",
"type": "Property"
},
"turbidityPrediction" : {
"nav": "type==type&value=value",
"type": "Property"
},
"conductivityPrediction" : {
"nav": "type==type&value=value",
"type": "Property"
},
"chloratePrediction": {
"nav": "type==type&value=value",
"type": "Property"
},
"validFrom" : {
"nav": "type==type&value=value",
"type": "Property"
},
"validTo" : {
"nav": "type==type&value=value",
"type": "Property"
},
"location": {
"nav": "type==type&value=value",
"type": "GeoProperty"
}
}
{
"dateObserved" : {
"nav": "type==type&value=value",
"type": "Property"
},
"pH" : {
"nav": "type==type&value=value",
"type": "Property"
},
"temperature" : {
"nav": "type==type&value=value",
"type": "Property"
},
"turbidity" : {
"nav": "type==type&value=value",
"type": "Property"
},
"freeChlorine" : {
"nav": "type==type&value=value",
"type": "Property"
},
"waterFlow" : {
"nav": "type==type&value=value",
"type": "Property"
},
"totalChlorine" : {
"nav": "type==type&value=value",
"type": "Property"
},
"conductivity" : {
"nav": "type==type&value=value",
"type": "Property"
},
"treatmentChlorinationDosage" : {
"nav": "type==type&value=value",
"type": "Property"
},
"treatmentFiltrationTime" : {
"nav": "type==type&value=value",
"type": "Property"
},
"treatmentCoagulantDosage" : {
"nav": "type==type&value=value",
"type": "Property"
},
"redox" : {
"nav": "type==type&value=value",
"type": "Property"
},
"chlorateEstimation" : {
"nav": "type==type&value=value",
"type": "Property"
},
"phThresholdHigh" : {
"nav": "type==type&value=value",
"type": "Property"
},
"phThresholdLow" : {
"nav": "type==type&value=value",
"type": "Property"
},
"chlorineThresholdHigh" : {
"nav": "type==type&value=value",
"type": "Property"
},
"chlorineThresholdLow" : {
"nav": "type==type&value=value",
"type": "Property"
},
"location" : {
"nav": "type==type&value=value",
"type": "GeoProperty"
},
"refDevice" : {
"nav": "type==type&value=value",
"type": "Property"
}
}
......@@ -9,9 +9,9 @@ from flask_caching import Cache
from werkzeug.exceptions import HTTPException
from src.api import config
from src.api.routes import validation_v2_entities, historic_validation_v2_entities, validation_ld_entities,\
from src.api.routes import validation_v2_entities, historic_validation_v2_entities, validation_ld_entities, \
historic_validation_ld_entities
from src.api.server_utils import ValidationErrorResponse, random_id, load_url_resources, parse_model
from src.api.server_utils import ValidationErrorResponse, random_id, parse_model
from src.logger import FORMATTER
......@@ -96,10 +96,13 @@ def create_app():
@app.before_first_request
def load_validation_context():
""" Initially loads all URL resources to cache `APP_CACHE`. """
request.custom_id = 'no request context'
# """ Initially loads all URL resources to cache `APP_CACHE`. """
# request.custom_id = 'before_first_request'
# TODO TBD maybe remove this condition
if not app.config["READ_SPECS_FROM_DISK"] and not app.config["READ_BLUEPRINTS_FROM_DISK"]:
load_url_resources(model_name='*')
# if not app.config["READ_SPECS_FROM_DISK"] and not app.config["READ_BLUEPRINTS_FROM_DISK"]:
# try:
# load_url_resources(model_name='*')
# except ValidationException:
pass
return app
......@@ -86,14 +86,17 @@ def validate_post_ld(model: dict, cache: flask_caching.Cache):
conn = Connector(reads_from_disk=current_app.config['READ_SPECS_FROM_DISK'])
specs = conn.load_specs(model_type=model_type)
cache.set(model_type.lower(), specs)
current_app.logger.warning(f'<<{__name__}>> write specs to cache')
current_app.logger.warning(f'<<{__name__}>> specs was cached')
current_app.logger.debug(f'<<{__name__}>> Specs retrieved')
# v2 models need to be converted to kv models for validation
if request.args.get('options', None) is None:
# lookup cache for suitable blueprints
conn = Connector(reads_from_disk=current_app.config['READ_BLUEPRINTS_FROM_DISK'])
blueprint = conn.load_blueprints(model_type=model_type, model_version='ld')
# cache.set(model_type.lower(), specs)
# current_app.logger.warning(f'<<{__name__}>> blueprint was cached')
model = Converter.convert_ld_to_kv(ld_model=model, blueprint=blueprint, updating=False)
logic = Creator(specs=specs, model=model, request=request)
logic.workflow()
......@@ -146,7 +149,7 @@ def validate_patch_v2(model: dict, model_type: str, cache: flask_caching.Cache):
log=False)
def validate_patch_ld(model: dict, model_type: str, cache: flask_caching.Cache):
def validate_patch_ld(model: dict, model_type: str, cache: flask_caching.Cache, upsert: bool = False):
"""
Loads the Blueprints and Specs consecutively and evaluates the `model` for a PATCH request.
......@@ -154,12 +157,12 @@ def validate_patch_ld(model: dict, model_type: str, cache: flask_caching.Cache):
model: Json Fiware model
model_type: Name of the Fiware model
cache: Flask Cache that is searched for cached entities
upsert: Indicates POST request. @Context attribute can then be missing
Raises:
ValidationErrorResponse: Any occurring error will be caught and responded with ErrorResponse
"""
try:
if cache.get(model_type.lower()):
specs = cache.get(model_type.lower())
else: # load specs from resource
......@@ -174,7 +177,7 @@ def validate_patch_ld(model: dict, model_type: str, cache: flask_caching.Cache):
if not request.args.get('options', None):
conn = Connector(reads_from_disk=current_app.config['READ_BLUEPRINTS_FROM_DISK'])
blueprint = conn.load_blueprints(model_type=model_type, model_version='ld')
model = Converter.convert_ld_to_kv(ld_model=model, blueprint=blueprint, updating=True)
model = Converter.convert_ld_to_kv(ld_model=model, blueprint=blueprint, updating=True, upsert=upsert)
# use kv-version for evaluating the update
logic = Updater(specs=specs, model=model, request=request)
......@@ -348,7 +351,6 @@ def patch_validation_all_entities(url):
@validation_v2_entities.route('/<path:url>', methods=["POST"])
def post_validation_current_entities(url):
model = ast.literal_eval(request.data.decode('UTF-8')) # must be dict
if isinstance(model, list):
raise ValidationErrorResponse(msg=f'None-historic api does not support json-array.', module=__name__)
......@@ -363,7 +365,7 @@ def post_validation_current_entities(url):
cache=current_app.config.get('APP_CACHE'))
else:
validate_patch_ld(model=model, model_type=extract_modelname_from_url(url),
cache=current_app.config.get('APP_CACHE'))
cache=current_app.config.get('APP_CACHE'), upsert=True)
except ValidationException as vex:
raise ValidationErrorResponse(msg=f'Json-model failed the evaluation for upsert-request. Error message:'
f' {vex.extern_logging}', log=False)
......@@ -406,7 +408,7 @@ def post_validation_historic_entities(url):
cache=current_app.config.get('APP_CACHE'))
else:
validate_patch_ld(model=m, model_type=extract_modelname_from_url(url),
cache=current_app.config.get('APP_CACHE'))
cache=current_app.config.get('APP_CACHE'), upsert=upsert)
except ValidationException as vex:
raise ValidationErrorResponse(
msg=f'Json-model failed the evaluation for upsert-request. Error message:'
......@@ -419,7 +421,7 @@ def post_validation_historic_entities(url):
cache=current_app.config.get('APP_CACHE'))
else:
validate_patch_ld(model=model, model_type=extract_modelname_from_url(url),
cache=current_app.config.get('APP_CACHE'))
cache=current_app.config.get('APP_CACHE'), upsert=upsert)
except ValidationException as vex:
raise ValidationErrorResponse(msg=f'Json-model failed the evaluation for upsert-request. Error message:'
f' {vex.extern_logging}', log=False)
......
......@@ -182,7 +182,7 @@ class Connector(object):
try:
r = requests.get(url=url, timeout=current_app.config.get('SPECS_TIMEOUT_SEC'))
return r.json()
except requests.exceptions.RequestException as conn_err:
except (requests.exceptions.RequestException, JSONDecodeError) as err:
raise ValidationException(extern=f'Service currently unable to retrieve model specifications.'
, module=__name__, func='load_json_from_url')
......@@ -233,8 +233,8 @@ class Connector(object):
try:
bp_key = current_app.config["URL_RESOURCES"]["URL_NAME_CONVENTIONS"]["BLUEPRINTS"]
bp_key = bp_key.format(modelname=model_type,
modeltype=model_version) # '{modeltype}_{modelname}_BLUEPRINTS_URL'
bp_key = bp_key.format(modeltype=model_version.upper(),
modelname=model_type.upper()) # '{modeltype}_{modelname}_BLUEPRINT_URL'
url = current_app.config["URL_RESOURCES"][bp_key]
current_app.logger.warning(f'<<{__name__}>> load blueprint from {url}')
return self.load_json_from_url(url=url)
......@@ -323,7 +323,7 @@ class Converter(object):
return kv_model
@staticmethod
def convert_ld_to_kv(ld_model: dict, blueprint: dict, updating=False) -> dict:
def convert_ld_to_kv(ld_model: dict, blueprint: dict, updating: bool = False, upsert: bool = False) -> dict:
"""
Converts a Fiware-LD into Fiware-KeyValue model.
......@@ -331,6 +331,7 @@ class Converter(object):
ld_model: Case sensitive type of fiware model e.g. FlowerBed
blueprint: Blueprint that contains the conversion metainfo
updating: Indicates if PATCH or POST http method was used for request
upsert: Indictes Upsert POST request. @Context attribute can be missing then.
Returns:
KeyValue model as python dict
......@@ -354,14 +355,16 @@ class Converter(object):
"type": ld_model.pop('type'),
}
# check and delete @context from ld-model
if not ld_model.get('@context'):
raise ValidationException(extern=f'LD-Model must contain `@context attribute`', module=__name__,
func='Converter.convert_ld_to_kv')
# upsert request can miss @context
if not upsert:
# check and delete @context from ld-model
if not ld_model.get('@context'):
raise ValidationException(extern=f'LD-Model must contain `@context attribute`', module=__name__,
func='Converter.convert_ld_to_kv')
# single validation for @context attribute, delete afterwards if successful
FiwareElement(structuredvalue=str(ld_model["@context"]))
ld_model.pop('@context')
# single validation for @context attribute, delete afterwards if successful
FiwareElement(structuredvalue=str(ld_model["@context"]))
ld_model.pop('@context')
# iterate model and check attributes with blueprints & add attributes to kv-model
try:
......@@ -416,12 +419,19 @@ def load_url_resources(model_name: str = '*'):
conn = Connector(reads_from_disk=current_app.config['READ_SPECS_FROM_DISK'])
if model_name == '*': # load all models
for resource_name, url in current_app.config["URL_RESOURCES"].items():
if resource_name == 'URL_NAME_CONVENTIONS':
continue
resource = conn.load_json_from_url(url=url)
current_app.config['APP_CACHE'].set(resource_name, resource)
try:
for resource_name, url in current_app.config["URL_RESOURCES"].items():
if resource_name == 'URL_NAME_CONVENTIONS':
continue
current_app.logger.debug(f'<<{__name__}>> Trying to load resource for: {resource_name}')
resource = conn.load_json_from_url(url=url)
current_app.config['APP_CACHE'].set(resource_name, resource)
except ValidationException as vex:
current_app.logger.error(f'<<{__name__}>> During loading of `{resource_name}` from {url} occured an error.'
f'Message: {vex.extern_logging}')
raise vex
i = 1
current_app.logger.info(f'<<{__name__}>> All Url resources were loaded to cache.')
else: # load resources for one model
......@@ -503,7 +513,7 @@ def parse_model():
# parse body to check if data is valid json or list of json
try:
data_decoded = literal_eval(request.data.decode('UTF-8'))
except (SyntaxError, ValueError):
except (SyntaxError, ValueError, TypeError):
raise ValidationErrorResponse(msg=f'Json-model can not be evaluated since it contains structural flaws',
module=__name__)
......
......@@ -144,8 +144,9 @@ class StructuredValue(Validator):
raise ValueError(f'Value "{v}" is not of fiware-type Text or Number')
else:
raise ValueError("Structured Value needs to be in form of key-value-pairs or as listed values")
except (SyntaxError, ValueError):
raise ValueError("Structured Value needs to be in form of key-value-pairs")
except (SyntaxError, ValueError, TypeError):
raise ValidationException(extern=f"Structured Value `{value}` needs to be in form of key-value-pairs or as "
f"list", module=__name__, func='StructuredValue.validate')
class Number(Validator):
......@@ -319,9 +320,17 @@ class Location(Validator):
def transform_geo_json(geo_json: dict) -> dict:
return {'type': f'geo:{geo_json["value"]["type"].lower()}', 'value': geo_json["value"]["coordinates"]}
def correct_coord_type(coord_type: str) -> any:
mapping = {'Line': 'geo:line', 'Point': 'geo:point', 'Box': 'geo:box', 'Polygon': 'geo:polygon'}
if coord_type in mapping.keys():
return mapping[coord_type]
return None
try:
value = ast.literal_eval(value) # Convert str to dict
coord_type = value.get('type', None) # geo:point, geo:line, geo:box, geo:polygon or geo:json
if correct_coord_type(coord_type=coord_type):
coord_type = correct_coord_type(coord_type=coord_type)
if not coord_type:
raise ValueError('Missing "type" attribute in location')
......@@ -390,6 +399,12 @@ class DateTime(Validator):
' YYYY-MM-ddTHH:mm:ssZ, YYYY-MM-ddTHH:mm:ss.SSSSZ ')
# TODO used in Device
class Boolean(Validator):
def validate(self, value):
pass
class FiwareElement(object):
""" Serves as a wrapper or container that initializes the desired Class that represents a fiware datatype.
It is mainly used by the workflow method and instantiated by the eval function but can of course be instantiated
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment