Commit 5d62ab97 authored by Federico Sismondi's avatar Federico Sismondi

Merge branch 'weather_fixes' into 'master'

Feat: now app collects, transforms and pushes data to platform. It handles...

See merge request !21
parents d8f963f0 c94dae70
......@@ -17,15 +17,19 @@ import base64
import time
import dateutil.parser
import datetime
import logging
from geojson import Point
from flask import Flask, jsonify
from flask import request
import requests as client_request
app = Flask(__name__, )
print("Starting to collect..")
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
# get config from environment
COLLECTION_DIR = os.getenv('COLLECTION_DIR', default='raw_data')
URL_BASE = 'http://{}:1026'.format(os.getenv('ORION_HOST', default='localhost'))
......
......@@ -15,5 +15,5 @@ COPY . /code
RUN unzip /code/data_harmonization_tool.zip
CMD ["echo", "please override this CMD with some .sh or .py script call"]
# docker run -it --rm --name dca-carouge-weather --env ORION_HOST=10.81.6.109 naiades-client ./weather_02_get_all_raw_data.py
# docker run -it --rm --name dca-carouge-weather --env ORION_HOST=10.81.6.109 naiades-client ./security_01_get_token_with_password.sh
This diff is collapsed.
{
"city_info": {
"name": "Carouge",
"country": "Suisse",
"latitude": "46.1838613",
"longitude": "6.1385878",
"elevation": "388",
"sunrise": "05:57",
"sunset": "21:25"
},
"current_condition": {
"date": "15.07.2020",
"hour": "00:00",
"tmp": 17,
"wnd_spd": 7,
"wnd_gust": 11,
"wnd_dir": "SE",
"pressure": 1011.9,
"humidity": 77,
"condition": "Fortement nuageux",
"condition_key": "fortement-nuageux",
"icon": "https://prevision-meteo.ch/style/images/icon/fortement-nuageux.png",
"icon_big": "https://prevision-meteo.ch/style/images/icon/fortement-nuageux-big.png"
}
}
{
"city_info": {
"name": "Carouge",
"country": "Suisse",
"latitude": "46.1838613",
"longitude": "6.1385878",
"elevation": "388",
"sunrise": "08:02",
"sunset": "16:51"
},
"current_condition": {
"date": "07.12.2020",
"hour": "13:00",
"tmp": 3,
"wnd_spd": 6,
"wnd_gust": 16,
"wnd_dir": "O",
"pressure": 997.5,
"humidity": 83,
"condition": "Eclaircies",
"condition_key": "eclaircies",
"icon": "https://prevision-meteo.ch/style/images/icon/eclaircies.png",
"icon_big": "https://prevision-meteo.ch/style/images/icon/eclaircies-big.png"
}
}
{
"city_info": {
"name": "Carouge",
"country": "Suisse",
"latitude": "46.1838613",
"longitude": "6.1385878",
"elevation": "388",
"sunrise": "07:56",
"sunset": "16:53"
},
"current_condition": {
"date": "01.12.2020",
"hour": "06:00",
"tmp": 3,
"wnd_spd": 16,
"wnd_gust": 25,
"wnd_dir": "S",
"pressure": 1017.3,
"humidity": 83,
"condition": "Pluie mod\u00e9r\u00e9e",
"condition_key": "pluie-moderee",
"icon": "https://prevision-meteo.ch/style/images/icon/pluie-moderee.png",
"icon_big": "https://prevision-meteo.ch/style/images/icon/pluie-moderee-big.png"
}
}
This diff is collapsed.
{"raw_data": "b'eyJjaXR5X2luZm8iOiB7Im5hbWUiOiAiQ2Fyb3VnZSIsICJjb3VudHJ5IjogIlN1aXNzZSIsICJsYXRpdHVkZSI6ICI0Ni4xODM4NjEzIiwgImxvbmdpdHVkZSI6ICI2LjEzODU4NzgiLCAiZWxldmF0aW9uIjogIjM4OCIsICJzdW5yaXNlIjogIjA1OjU1IiwgInN1bnNldCI6ICIyMTowOCJ9LCAiY3VycmVudF9jb25kaXRpb24iOiB7ImRhdGUiOiAiMTkuMDUuMjAyMCIsICJob3VyIjogIjE1OjAwIiwgInRtcCI6IDIzLCAid25kX3NwZCI6IDM2LCAid25kX2d1c3QiOiA1OCwgInduZF9kaXIiOiAiTkUiLCAicHJlc3N1cmUiOiAxMDEyLjksICJodW1pZGl0eSI6IDQ2LCAiY29uZGl0aW9uIjogIkVuc29sZWlsbFx1MDBlOSIsICJjb25kaXRpb25fa2V5IjogImVuc29sZWlsbGUiLCAiaWNvbiI6ICJodHRwczovL3d3dy5wcmV2aXNpb24tbWV0ZW8uY2gvc3R5bGUvaW1hZ2VzL2ljb24vZW5zb2xlaWxsZS5wbmciLCAiaWNvbl9iaWciOiAiaHR0cHM6Ly93d3cucHJldmlzaW9uLW1ldGVvLmNoL3N0eWxlL2ltYWdlcy9pY29uL2Vuc29sZWlsbGUtYmlnLnBuZyJ9fQ=='", "source": "https://www.prevision-meteo.ch/services/json/carouge"}
\ No newline at end of file
#!/usr/bin/env python3
"""
Dumps weather data to disk each POST_DATA_PERIOD seconds
"""
import os
import json
import time
import logging
from weather_api import *
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.info("Starting to collect..")
COLLECTION_DIR = os.getenv('COLLECTION_DIR', default='.')
def collect():
logger.info("Querying service..")
weather_data = get_data_from_weather_service()
filepath = os.path.join(COLLECTION_DIR,'{}_weather_raw_data.json'.format(int(time.time())))
with open(filepath, encoding='utf-8', mode='w') as f:
json.dump(
obj=weather_data,
fp=f
)
logger.info("Dumping to FS {}".format(filepath))
collect()
# Send updates in a loop
while True:
time.sleep(POST_DATA_PERIOD)
collect()
#!/usr/bin/env python3
"""
Creates entity in IoT platform, and loops for ever sending updates on the weather data for Carouge
"""
import time
import logging
from weather_api import *
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# Create entity in case it doesnt exist
create_weather_entity_in_platform()
# Send updates in a loop
while True:
time.sleep(POST_DATA_PERIOD)
post_weather_data_to_platform()
#!/usr/bin/env python3
import logging
from weather_api import *
from pprint import pprint
logger = logging.getLogger()
logger.setLevel(logging.INFO)
pprint(get_all_raw_weather_data_from_platform())
\ No newline at end of file
#!/usr/bin/env python3
"""
Implements API calls related to weather use case
NOTES:
- id and type of message sent should follow convention defined for URN in spreadsheet
https://telecombretagneeu-my.sharepoint.com/:x:/g/personal/federico_sismondi_telecom-bretagne_eu/Ebw_b9iHempGoSPEz__sYCcB_qPu0pwxDAgAeKXFoMu6cA?e=6uuUGD
"""
import os
import json
import requests
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
# config environment
URL_weather_service = 'https://www.prevision-meteo.ch/services/json/carouge'
POST_DATA_PERIOD = int(os.getenv('POST_DATA_PERIOD', default='300')) # in seconds
URL_BASE = 'http://{}:1026'.format(os.getenv('ORION_HOST', default='localhost'))
dump_keys = ['city_info', 'current_condition']
# see note above about id and type
fiware_data = {
'id': 'urn:ngsi-ld:WeatherObservedRaw:carouge_weather_observed_raw',
'type': 'WeatherObservedRaw'
}
# build header for POST
http_header_post = {
'Fiware-Service': 'carouge',
'Content-Type': 'application/json',
'Accept': 'application/json',
}
# build header for GET
http_header_get = {
'Fiware-Service': 'carouge',
'Accept': 'application/json',
}
logger.info('Config: \n\tPOST_DATA_PERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE))
def get_data_from_weather_service():
"""
:return: Raw data (json as string)
"""
logger.info('Querying weather service..')
try:
r = requests.get(
url=URL_weather_service,
timeout=90,
)
except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
logger.info('Could not weather reach service')
return None
try:
weather_data = {k: v for k, v in r.json().items() if k in dump_keys}
except json.decoder.JSONDecodeError as e:
logger.error("Got something that doesnt look like a json: {}".format(r.content))
return None
return weather_data
def create_weather_entity_in_platform():
weather_data = get_data_from_weather_service()
assert weather_data, 'Weather data not defined'
logger.info('Creating entity')
response = None
try:
entity = {**weather_data, **fiware_data}
response = requests.post(
url=URL_BASE + '/v2/entities?options=keyValues',
headers=http_header_post,
json=entity
)
logger.info('Got {}'.format(response.status_code))
except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
logger.error('IoT Platform at {} seems unreachable'.format(URL_BASE))
if response and response.ok:
return True
else:
if response:
logger.error('Operation failed, got message: {}'.format(response.json()))
else:
logger.error('Operation failed, no response received')
return False
def post_weather_data_to_platform():
weather_data = get_data_from_weather_service()
assert weather_data, 'Weather data not defined'
logger.info('Posting weather data to platform')
response = None
try:
entity = {**weather_data}
with open('weather_raw_last_data.json', encoding='utf-8', mode='w') as f:
json.dump(
obj=entity,
fp=f
)
response = requests.put(
url=URL_BASE + '/v2/entities/{}/attrs?options=keyValues'.format(fiware_data['id']),
headers=http_header_post,
json=entity
)
logger.info('Got {}'.format(response.status_code))
except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
logger.error('IoT Platform at {} seems unreachable'.format(URL_BASE))
if response and response.ok:
return True
else:
logger.error('Operation failed, got message: {}'.format(response.json()))
return False
def get_all_raw_weather_data_from_platform():
logger.info('Getting all RAW weather data from platform')
response = None
try:
response = requests.get(
url=URL_BASE + '/v2/entities/?options=keyValues&type={}'.format(fiware_data['type']),
headers=http_header_get,
)
logger.info('Got {}'.format(response.status_code))
except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
logger.error('IoT Platform at {} seems unreachable'.format(URL_BASE))
if response and response.ok:
return response.json()
else:
logger.error('Operation failed, got message: {}'.format(response.json()))
return False
__all__ = [
'URL_weather_service',
'POST_DATA_PERIOD',
'URL_BASE',
'post_weather_data_to_platform',
'get_data_from_weather_service',
'create_weather_entity_in_platform',
'get_all_raw_weather_data_from_platform'
]
......@@ -108,7 +108,7 @@ services:
environment:
- LC_ALL=C.UTF-8
- LANG=C.UTF-8
- POST_DATA_PERIOD=900 #15mins
- POST_DATA_PERIOD=1800 #30mins
- ORION_HOST=${ORION}
- COLLECTION_DIR=/code/raw_data
volumes:
......@@ -116,7 +116,7 @@ services:
networks:
default:
ipv4_address: 172.18.2.15
command: ./weather_00_collect_raw_weather_data_carouge.py # no sensors, only 3rd party data for the moment
command: ./app.py # no sensors, only 3rd party data for the moment
# healthcheck:
# test: curl --fail -s http://dca-carouge-weather:${WMS_MODULE_PORT}/healthcheck || exit 1
# start_period: 40s
......
......@@ -340,7 +340,7 @@ services:
environment:
- LC_ALL=C.UTF-8
- LANG=C.UTF-8
- POST_DATA_PERIOD=900 #15mins
- POST_DATA_PERIOD=1800 #30 minutes
- ORION_HOST=orion
- COLLECTION_DIR=/code/raw_data
volumes:
......@@ -348,7 +348,7 @@ services:
networks:
default:
ipv4_address: 172.18.1.15
command: ./weather_00_collect_raw_weather_data_carouge.py # no sensors, only 3rd party data for the moment
command: ./app.py # no sensors, only 3rd party data for the moment
# healthcheck:
# test: curl --fail -s http://dca-carouge-weather:${WMS_MODULE_PORT}/healthcheck || exit 1
# start_period: 40s
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment