Commit 6f16d072 authored by Federico Sismondi's avatar Federico Sismondi
Browse files

Add files and updates for handling <python3 app.py push-locally-collected >...

Add files and updates for handling <python3 app.py push-locally-collected > for both fountain and watering use cases of carouge
parent 4825a029
......@@ -55,7 +55,7 @@ http_header_post = {
def get_ngsiv2_typed_description(val, force_date_modified=None):
"""
This function updates the attributes data format and meta data for aligining it to ngsi v2
This function updates the attributes data format and meta data for aligning it to ngsi v2
:param val: Value to be added to entity as attribute
:param force_date_modified: Optional datetime for updating dateModified meta data of attribute
:return: Dictionary of attribute typed and formated as NGSI-v2
......
{
"confirmed": false,
"cr_used": "4/5",
"dataFrame": "gAD/gw==",
"data_format": "base64",
"decrypted": true,
"devaddr": 20906033,
"deveui": "70b3d5fffebf6952",
"device_redundancy": 1,
"dr_used": "SF7BW125",
"early": false,
"fcnt": 62,
"freq": 867500000,
"gtw_info": [
{
"ant": 0,
"foff": 0,
"gtw_id": "000000000800000a",
"rssi": -62,
"snr": 9.8
}
],
"id": 1602783550187,
"live": true,
"mac_msg": "QDEAPwGAPgADvUapqxAsnsE=",
"port": 3,
"rssi": -62,
"sf_used": 7,
"snr": 9.8,
"time_on_air_ms": 51.456,
"timestamp": "2020-10-15T17:39:10.187Z"
}
{
"confirmed": false,
"cr_used": "4/5",
"dataFrame": "gAD/gw==",
"data_format": "base64",
"decrypted": true,
"devaddr": 20906033,
"deveui": "70b3d5fffebf6952",
"device_redundancy": 1,
"dr_used": "SF7BW125",
"early": false,
"fcnt": 85,
"freq": 868300000,
"gtw_info": [
{
"ant": 0,
"foff": 0,
"gtw_id": "000000000800000a",
"rssi": -60,
"snr": 8.2
}
],
"id": 1602824959350,
"live": true,
"mac_msg": "QDEAPwGAVQADNlKYndMUonM=",
"port": 3,
"rssi": -60,
"sf_used": 7,
"snr": 8.2,
"time_on_air_ms": 51.456,
"timestamp": "2020-10-16T05:09:19.350Z"
}
{
"confirmed": false,
"cr_used": "4/5",
"dataFrame": "+AACsgD/AGcARgMr",
"data_format": "base64",
"decrypted": true,
"devaddr": 4063278,
"deveui": "70b3d5fffebf6a44",
"device_redundancy": 1,
"dr_used": "SF7BW125",
"early": false,
"fcnt": 1626,
"freq": 868500000,
"gtw_info": [
{
"ant": 0,
"foff": 0,
"gtw_id": "000000000800000a",
"rssi": -45,
"snr": 8.2
}
],
"id": 1605542051059,
"live": true,
"mac_msg": "QC4APgCAWgYDf71VvqJ/CW0/c2cjGepvHA==",
"port": 3,
"rssi": -45,
"sf_used": 7,
"snr": 8.2,
"time_on_air_ms": 61.696,
"timestamp": "2020-11-16T15:54:11.059Z"
}
{
"confirmed": false,
"cr_used": "4/5",
"dataFrame": "+AACugD+AGAASQM1",
"data_format": "base64",
"decrypted": true,
"devaddr": 4063278,
"deveui": "70b3d5fffebf6a44",
"device_redundancy": 1,
"dr_used": "SF7BW125",
"early": false,
"fcnt": 1761,
"freq": 868300000,
"gtw_info": [
{
"ant": 0,
"foff": 0,
"gtw_id": "000000000800000a",
"rssi": -47,
"snr": 9.2
}
],
"id": 1605781541945,
"live": true,
"mac_msg": "QC4APgCA4QYDQivbkgh6jLg8AoIEtH0+SA==",
"port": 3,
"rssi": -47,
"sf_used": 7,
"snr": 9.2,
"time_on_air_ms": 61.696,
"timestamp": "2020-11-19T10:25:41.945Z"
}
dump dir for collected raw data
===============================
data structure pushed by devices LoRa devices is complex, please look at app.py to understand how values are multiplexed
into received dataFrame
......@@ -12,6 +12,7 @@ In simple words this module:
import re
import os
import json
import sys
import base64
import time
import dateutil.parser
......@@ -26,24 +27,21 @@ app = Flask(__name__, )
print("Starting to collect..")
# get config from environment
COLLECTION_DIR = os.getenv('COLLECTION_DIR', default='.')
COLLECTION_DIR = os.getenv('COLLECTION_DIR', default='raw_data')
URL_BASE = 'http://{}:1026'.format(os.getenv('ORION_HOST', default='localhost'))
LORA_PLATFORM_URL = os.getenv('LORA_PLATFORM_URL')
LORA_PLATFORM_SECRET_FILE = os.getenv('LORA_PLATFORM_SECRET_FILE', 'secret_carouge_lora')
with open(LORA_PLATFORM_SECRET_FILE, 'r') as file:
secret = file.read().replace('\n', '')
assert secret, 'No auth token defined for LoRa platform API'
# AUTH Header for lora platform
h = {'Authorization': secret}
# URN schema for Device entities
URN_BASE_DEVICE_ENTITY = "urn:ngsi-ld:Device:Device-"
URN_BASE_FLOWERBED_ENTITY = "urn:ngsi-ld:FlowerBed:FlowerBed-"
# HISTORICAL API (QuantumLeap):
NOTIFICATION_URL = 'http://{}:8668/v2/notify'.format(os.getenv('ORION_HOST', default='localhost'))
NOTIF_FLOWERBED = '5f071e73e44ddaac278abd3d'
NOTIF_DEVICE = '5fbf8489da7e7c9ac9757a7c'
# build header for POST
http_header_post = {
'Fiware-Service': 'carouge',
......@@ -58,32 +56,68 @@ http_header_get = {
}
def get_ngsiv2_typed_description(val):
def get_ngsiv2_typed_description(val, force_date_modified=None):
"""
This function updates the attributes data format and meta data for aligning it to ngsi v2
:param val: Value to be added to entity as attribute
:param force_date_modified: Optional datetime for updating dateModified meta data of attribute
:return: Dictionary of attribute typed and formated as NGSI-v2
"""
ret = {}
if force_date_modified and isinstance(force_date_modified, datetime.datetime):
# adds datetime metadata
ret.update(
{"metadata":
{"dateModified":
{"type": "DateTime", "value": str(force_date_modified.isoformat()).replace("+00:00", "Z")}
}
}
)
if force_date_modified and isinstance(force_date_modified, str):
# adds datetime metadata
ret.update(
{"metadata":
{"dateModified": {"type": "DateTime", "value": force_date_modified}}
}
)
assert "Z" in force_date_modified, "Only Zulu time is accepted"
if isinstance(val, Point):
return {'type': 'geo:json', 'value': dict(val)}
ret.update({'type': 'geo:json', 'value': dict(val)})
elif isinstance(val, int) or isinstance(val, float):
return {'type': 'Number', 'value': val}
ret.update({'type': 'Number', 'value': val})
elif isinstance(val, str):
pat = r'[<>"\'=;()]' # see https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html
r = re.compile(pat)
if r.search(val) is None:
new_val = val
else:
new_val = re.sub(pat, '', val)
app.logger.warning("Special char(s) deleted. This <{}> turned into this <{}>".format(val, new_val))
return {'type': 'Text', 'value': new_val}
ret.update({'type': 'Text', 'value': new_val})
elif isinstance(val, datetime.datetime):
# replace +00:00 is the same as Z but Orion doesnt like it :/
return {'type': 'DateTime', 'value': str(val.isoformat()).replace("+00:00", "Z")}
ret.update({'type': 'DateTime', 'value': str(val.isoformat()).replace("+00:00", "Z")})
elif isinstance(val, dict):
return {'value': val} # bypassed
ret.update({'value': val}) # bypassed
elif isinstance(val, list):
return {'value': val} # bypassed
ret.update({'value': val}) # bypassed
elif isinstance(val, type(None)):
return {'type': 'Text', 'value': None}
ret.update({'type': 'Text', 'value': None})
else:
raise NotImplementedError("Couldn't get NGSIv2 type for object {} of type {}".format(val, type(val)))
return ret
# = = = = = = = = API HANDLERS = = = = = = = = = = =
......@@ -213,7 +247,7 @@ def _get_reading(message_data):
# deplexing data:
# 42 -> it's a data frame!
# c0 ->
# c0 -> ?
# 01 -> configured as 0-10v input for channel A
# 02a0fb -> data channel A
# 01 -> configured as 0-10v input for channel A
......@@ -237,50 +271,16 @@ def _get_reading(message_data):
raise InvalidUsage(message=msg)
# = = = = = = = = CONTEXT BROKER FUNCTIONS = = = = = = = =
def _update_flowerbed_entity(soil_moisture, sensor_eui):
flowerbed_id = None
# check if refDevice attr defined for any FlowerBed
url = "{}/v2/entities/?q=refDevice=={}&type=FlowerBed".format(URL_BASE, sensor_eui)
r = client_request.get(
url=url,
headers=http_header_get
)
if r.ok and r.json():
flowerbed_id = r.json()[0]['id']
if not flowerbed_id:
msg = 'No FlowerBed associated to sensor {}'.format(sensor_eui)
app.logger.error(msg)
return msg
app.logger.debug('Found FlowerBed {} associated to sensor {}'.format(flowerbed_id, sensor_eui))
body = {"soilMoisture": get_ngsiv2_typed_description(soil_moisture)}
url = "{}/v2/entities/{}/attrs".format(URL_BASE, flowerbed_id)
r = client_request.patch(
url=url,
headers=http_header_post,
json=body
)
app.logger.debug('Sending data to CTX broker: {}'.format(body))
if not (200 <= r.status_code < 300):
app.logger.error('HTTP status code not 2xx, something went wrong..')
app.logger.error(r.status_code)
app.logger.error(r.reason)
app.logger.error(r.json())
return "Got error {}".format(r.reason)
else:
app.logger.info('Entity updated (code {}), url {}'.format(r.status_code, url))
return "ok"
def _update_device_entity(deveui, comment, longitude, latitude, last_reception, dataFrame,
dataFrame_rssi, dataFrame_timestamp):
""" Creates/Updates LoRa devs into ctx broker
def get_model_device(deveui,
comment = None,
longitude = None,
latitude = None,
last_reception = None,
dataFrame = None,
dataFrame_rssi = None,
dataFrame_timestamp = None,
meta_datetime=None):
""" Returns data dict of Device entity following Device data model
Output Data model:
-id (static)
......@@ -312,23 +312,25 @@ def _update_device_entity(deveui, comment, longitude, latitude, last_reception,
# static values
body.update({'id': '{}{}'.format(URN_BASE_DEVICE_ENTITY, deveui[-4:])})
body.update({'type': 'Device'})
body.update({'serialNumber': get_ngsiv2_typed_description(deveui)})
body.update({'controlledProperty': get_ngsiv2_typed_description(['soil_moisture'])})
body.update({'owner': get_ngsiv2_typed_description('Carouge')})
body.update({'source': get_ngsiv2_typed_description(LORA_PLATFORM_URL)})
body.update({'serialNumber': get_ngsiv2_typed_description(deveui,meta_datetime)})
body.update({'controlledProperty': get_ngsiv2_typed_description(['soil_moisture'],meta_datetime)})
body.update({'owner': get_ngsiv2_typed_description('Carouge',meta_datetime)})
body.update({'source': get_ngsiv2_typed_description(LORA_PLATFORM_URL,meta_datetime)})
# dynamic/real-time values
if longitude and latitude:
body.update({'location': get_ngsiv2_typed_description(Point([longitude, latitude]))})
body.update({'location': get_ngsiv2_typed_description(Point([longitude, latitude]),meta_datetime)})
else:
body.update({'location': get_ngsiv2_typed_description(Point([0, 0]), meta_datetime)})
if comment:
body.update({'description': get_ngsiv2_typed_description(comment)})
body.update({'description': get_ngsiv2_typed_description(comment,meta_datetime)})
# parse device name hidden inside comment field
pattern = r'{device_name}=([^\s]+)'
result = re.search(pattern, comment)
if result:
body.update({'name': get_ngsiv2_typed_description(result.groups()[0])})
body.update({'name': get_ngsiv2_typed_description(result.groups()[0],meta_datetime)})
if last_reception or dataFrame_timestamp:
in_date = None
......@@ -345,17 +347,189 @@ def _update_device_entity(deveui, comment, longitude, latitude, last_reception,
except TypeError:
pass
if in_date:
body.update({'dateLastValueReported': get_ngsiv2_typed_description(in_date)})
body.update({'dateLastValueReported': get_ngsiv2_typed_description(in_date,meta_datetime)})
else:
app.logger.warning("No date parsed for updaging Device field: dateLastValueReported")
app.logger.warning("No date parsed for updating Device field: dateLastValueReported")
if dataFrame:
moist, bat = _get_reading(dataFrame)
body.update({'value': get_ngsiv2_typed_description(moist)})
body.update({'batteryLevel': get_ngsiv2_typed_description(bat)})
body.update({'value': get_ngsiv2_typed_description(moist,meta_datetime)})
body.update({'batteryLevel': get_ngsiv2_typed_description(bat,meta_datetime)})
if dataFrame_rssi:
body.update({'rssi': get_ngsiv2_typed_description(dataFrame_rssi)})
body.update({'rssi': get_ngsiv2_typed_description(dataFrame_rssi,meta_datetime)})
return body
# = = = = = = = = CONTEXT BROKER FUNCTIONS = = = = = = = =
def _update_flowerbed_entity(soil_moisture, sensor_eui):
flowerbed_id = None
# check if refDevice attr defined for any FlowerBed
url = "{}/v2/entities/?q=refDevice=={}&type=FlowerBed".format(URL_BASE, sensor_eui)
r = client_request.get(
url=url,
headers=http_header_get
)
if r.ok and r.json():
flowerbed_id = r.json()[0]['id']
if not flowerbed_id:
msg = 'No FlowerBed associated to sensor {}'.format(sensor_eui)
app.logger.error(msg)
return msg
app.logger.debug('Found FlowerBed {} associated to sensor {}'.format(flowerbed_id, sensor_eui))
body = {"soilMoisture": get_ngsiv2_typed_description(soil_moisture)}
url = "{}/v2/entities/{}/attrs".format(URL_BASE, flowerbed_id)
r = client_request.patch(
url=url,
headers=http_header_post,
json=body
)
app.logger.debug('Sending data to CTX broker: {}'.format(body))
if not (200 <= r.status_code < 300):
app.logger.error('HTTP status code not 2xx, something went wrong..')
app.logger.error(r.status_code)
app.logger.error(r.reason)
app.logger.error(r.json())
return "Got error {}".format(r.reason)
else:
app.logger.info('Entity updated (code {}), url {}'.format(r.status_code, url))
return "ok"
def _update_historical_db_from_local_raw_files():
"""Posts data to Quantum Leap API directly as notification (without passing through Context Manager)
Used for pushing locally collected data to historical API service
"""
json_files = [os.path.join(COLLECTION_DIR, x) for x in os.listdir(COLLECTION_DIR) if x.endswith("json")]
data_devices = []
data_flowrebed = []
for json_file in json_files:
if 'merged_output' in json_file:
continue
# get data from file
with open(json_file, "r") as f:
json_data = json.load(f)
try:
sensor_eui = json_data['deveui']
except TypeError:
print("Error found trying to parse json for file {}".format(json_file))
print(json_data)
sys.exit(2)
flowerbed_id = None
if not json_data['dataFrame'] or json_data['dataFrame'] == '':
print("(!) Warning found emtpy data frame from device {}".format(sensor_eui))
# generate Device entity
device_entity = get_model_device(
deveui=sensor_eui,
dataFrame=json_data['dataFrame'],
dataFrame_rssi=json_data['rssi'],
dataFrame_timestamp=json_data['timestamp'],
meta_datetime=json_data['timestamp']
)
# add to list of data to be sent to quantum leap
data_devices.append(device_entity)
# generate FlowerBed entity
# first lets get the FlowerBed id from the deveui
# check if refDevice attr defined for any FlowerBed
url = "{}/v2/entities/?q=refDevice=={}&type=FlowerBed".format(URL_BASE, sensor_eui)
r = client_request.get(
url=url,
headers=http_header_get
)
if r.ok and r.json():
flowerbed_id = r.json()[0]['id']
if not flowerbed_id:
msg = 'No FlowerBed associated to sensor {}'.format(sensor_eui)
app.logger.error(msg)
continue
else:
moist, _ = _get_reading(json_data['dataFrame'])
flowerbed_entity = {
'id': flowerbed_id,
'type': 'FlowerBed',
'soilMoisture': get_ngsiv2_typed_description(moist, json_data['timestamp']),
'refDevice':get_ngsiv2_typed_description(sensor_eui, json_data['timestamp']),
}
# add to list of data to be sent to quantum leap
data_flowrebed.append(flowerbed_entity)
# dump to file to check what will be sent
with open(os.path.join(COLLECTION_DIR, "merged_output_device.json"), "w") as f:
json.dump(data_devices, f)
# dump to file to check what will be sent
with open(os.path.join(COLLECTION_DIR, "merged_output_flowerbed.json"), "w") as f:
json.dump(data_flowrebed, f)
## uncoment this for verifying content of files before pushing!
#sys.exit(2)
r = client_request.post(
url=NOTIFICATION_URL,
headers=http_header_post,
json={"subscriptionId": NOTIF_DEVICE, "data": data_devices}
)
if not (200 <= r.status_code < 300):
print('HTTP status code not 2xx, something went wrong..')
print(r.status_code)
print(r.reason)
print(r.json())
else:
print('History has been rewritten for DEVICE entities! (HTTP response is 2xx)')
r = client_request.post(
url=NOTIFICATION_URL,
headers=http_header_post,
json={"subscriptionId": NOTIF_FLOWERBED, "data": data_flowrebed}
)
if not (200 <= r.status_code < 300):
print('HTTP status code not 2xx, something went wrong..')
print(r.status_code)
print(r.reason)
print(r.json())
else:
print('History has been rewritten for FLOWERBED entities! (HTTP response is 2xx)')
def _update_device_entity(deveui, comment, longitude, latitude, last_reception, dataFrame,
dataFrame_rssi, dataFrame_timestamp):
""" Creates/Updates LoRa devs into ctx broker
"""
assert deveui
body = get_model_device(deveui,
comment,
longitude,
latitude,
last_reception,
dataFrame,
dataFrame_rssi,
dataFrame_timestamp)
app.logger.debug('Sending data to CTX broker: {}'.format(body))
......@@ -491,5 +665,27 @@ def main():
if __name__ == "__main__":
if 'push-locally-collected' in sys.argv: # call `python3 app.py push-locally-collected`
print("Pushing locally collected data at {} to IoT Platform historical component".format(COLLECTION_DIR))
_update_historical_db_from_local_raw_files()
sys.exit(0)
print("Running app as collector and push to IoT Platform")
with open(LORA_PLATFORM_SECRET_FILE, 'r') as file:
secret = file.read().replace('\n', '')
assert secret, 'No auth token defined for LoRa platform API'
# AUTH Header for lora platform
h = {'Authorization': secret}
# build header for GET
http_header_get = {
'Fiware-Service': 'carouge',
'Accept': 'application/json',
}
init()
main()
{
"id": 1595242307991,
"deveui": "0018b20000020a80",
"timestamp": "2020-07-20T10:51:47.991Z",
"devaddr": 295517731,
"live": true,
"dataFrame": "QkABACmaAQAAAA==",
"fcnt": 731,
"session_id": "b8f4dbb2-7e68-435a-9efe-9525b7eecc11",
"port": 1,
"rssi": -108,
"snr": -12,