Commit 3f44788e authored by Federico Sismondi's avatar Federico Sismondi

Feat() data collection in raw format, and sketch of first version of collect...

Feat() data collection in raw format, and sketch of first version of collect to historical data base (w/ QuantumLeap)
parent 733df5b9
.cpython-36.pyc
.idea/ .idea/
weather_data_init.json
scripts/weather_raw_last_data.json
wms-example/__pycache__/ wms-example/__pycache__/
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
...@@ -30,6 +30,10 @@ services: ...@@ -30,6 +30,10 @@ services:
command: -dbhost mongo-db -logLevel INFO command: -dbhost mongo-db -logLevel INFO
healthcheck: healthcheck:
test: curl --fail -s http://orion:${ORION_PORT}/version || exit 1 test: curl --fail -s http://orion:${ORION_PORT}/version || exit 1
start_period: 40s
interval: 15m00s
timeout: 10s
retries: 3
# Keyrock is an Identity Management Front-End # Keyrock is an Identity Management Front-End
keyrock: keyrock:
...@@ -58,6 +62,10 @@ services: ...@@ -58,6 +62,10 @@ services:
- my_secret_data - my_secret_data
healthcheck: healthcheck:
test: curl --fail -s http://localhost:${KEYROCK_PORT}/version || exit 1 test: curl --fail -s http://localhost:${KEYROCK_PORT}/version || exit 1
start_period: 40s
interval: 15m00s
timeout: 10s
retries: 3
# PEP Proxy for Orion # PEP Proxy for Orion
orion-proxy: orion-proxy:
...@@ -89,6 +97,10 @@ services: ...@@ -89,6 +97,10 @@ services:
- PEP_PROXY_PUBLIC_PATHS=/version - PEP_PROXY_PUBLIC_PATHS=/version
healthcheck: healthcheck:
test: curl --fail -s http://orion-proxy:${ORION_PROXY_PORT}/version || exit 1 test: curl --fail -s http://orion-proxy:${ORION_PROXY_PORT}/version || exit 1
start_period: 40s
interval: 15m00s
timeout: 10s
retries: 3
# Databases # Databases
mongo-db: mongo-db:
...@@ -179,10 +191,13 @@ services: ...@@ -179,10 +191,13 @@ services:
healthcheck: healthcheck:
test: curl --fail -s http://fiware-quantumleap:${QUANTUMLEAP_PORT}/v2/version || exit 1 test: curl --fail -s http://fiware-quantumleap:${QUANTUMLEAP_PORT}/v2/version || exit 1
start_period: 40s start_period: 40s
interval: 1m30s interval: 15m00s
timeout: 10s timeout: 10s
retries: 3 retries: 3
# - - - services which act as clients of the NAIADES platform
wms-app-example: wms-app-example:
build: build:
context: ./wms-example context: ./wms-example
...@@ -204,6 +219,61 @@ services: ...@@ -204,6 +219,61 @@ services:
ipv4_address: 172.18.1.11 ipv4_address: 172.18.1.11
healthcheck: healthcheck:
test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1 test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
start_period: 40s
interval: 15m00s
timeout: 10s
retries: 3
weather-data-collection-carouge-raw:
build:
context: ./weather-data-collection-carouge
dockerfile: Dockerfile
image: weather-data-collection-carouge:latest
container_name: weather-data-collection-carouge-raw
environment:
- LC_ALL=C.UTF-8
- LANG=C.UTF-8
- POST_DATA_PERIOD=3600 #3600sec = 1hour
- ORION_HOST=orion
- COLLECTION_DIR=/code/raw_data
volumes:
- ./weather-data-collection-carouge:/code:rw
networks:
default:
ipv4_address: 172.18.1.13
command: ./weather_00_collect_raw_weather_data_carouge
# healthcheck:
# test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
# start_period: 40s
# interval: 15m00s
# timeout: 10s
# retries: 3
# weather-data-collection-carouge:
# build:
# context: ./weather-data-collection-carouge
# dockerfile: Dockerfile
# image: weather-data-collection-carouge:latest
# container_name: weather-data-collection-carouge
# environment:
# - LC_ALL=C.UTF-8
# - LANG=C.UTF-8
# - POST_DATA_PERIOD=15
# - ORION_HOST=orion
# volumes:
# - ./weather-data-collection-carouge:/code:rw
# networks:
# default:
# ipv4_address: 172.18.1.14
# command: ./weather_01_collect_and_post_weather_data_carouge.py
## healthcheck:
## test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
## start_period: 40s
## interval: 15m00s
## timeout: 10s
## retries: 3
networks: networks:
default: default:
......
...@@ -6,6 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}" ...@@ -6,6 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \ curl --location --request GET \
"http://$ORION_HOST:1026/v2/subscriptions/" \ "http://$ORION_HOST:1026/v2/subscriptions/" \
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
| python -mjson.tool | python -mjson.tool
...@@ -8,4 +8,4 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}" ...@@ -8,4 +8,4 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
curl -X GET \ curl -X GET \
"http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\ "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" --header "Fiware-ServicePath: /Watering"
...@@ -10,6 +10,6 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}" ...@@ -10,6 +10,6 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
curl -X GET \ curl -X GET \
"http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\ "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
--header "X-Auth-Token: $KEYROCK_TOKEN" --header "X-Auth-Token: $KEYROCK_TOKEN"
...@@ -10,7 +10,7 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}" ...@@ -10,7 +10,7 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
curl --location --request PUT \ curl --location --request PUT \
"http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\ "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
--header "X-Auth-Token: $KEYROCK_TOKEN" \ --header "X-Auth-Token: $KEYROCK_TOKEN" \
--header "Content-Type: text/plain" \ --header "Content-Type: text/plain" \
-d $(( RANDOM % 10 )) -d $(( RANDOM % 10 ))
......
...@@ -17,7 +17,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}" ...@@ -17,7 +17,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}"
curl --location --request POST \ curl --location --request POST \
"http://$ORION_HOST:1026/v2/subscriptions/" \ "http://$ORION_HOST:1026/v2/subscriptions/" \
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
--header "Content-Type: application/json" \ --header "Content-Type: application/json" \
--header "Accept: application/json" \ --header "Accept: application/json" \
--header "X-Auth-Token: $KEYROCK_TOKEN" \ --header "X-Auth-Token: $KEYROCK_TOKEN" \
......
...@@ -6,7 +6,7 @@ echo "Querying context broker at: ${ORION_HOST}" ...@@ -6,7 +6,7 @@ echo "Querying context broker at: ${ORION_HOST}"
curl -iX POST \ curl -iX POST \
"http://$ORION_HOST:1026/v2/entities/" \ "http://$ORION_HOST:1026/v2/entities/" \
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
--header "Content-Type: application/json" \ --header "Content-Type: application/json" \
--data '{ --data '{
"id": "urn:ngsi-ld:FlowerBed:FlowerBed-345", "id": "urn:ngsi-ld:FlowerBed:FlowerBed-345",
......
...@@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}" ...@@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \ curl --location --request GET \
"http://$ORION_HOST:1026/v2/entities/" \ "http://$ORION_HOST:1026/v2/entities/" \
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
| python -mjson.tool | python -mjson.tool
...@@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}" ...@@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \ curl --location --request GET \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345?options=keyValues" \ "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345?options=keyValues" \
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
| python -mjson.tool | python -mjson.tool
\ No newline at end of file
...@@ -6,11 +6,11 @@ echo "Querying context broker at: ${ORION_HOST}" ...@@ -6,11 +6,11 @@ echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \ curl --location --request GET \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value" \ "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value" \
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
| python -mjson.tool | python -mjson.tool
#curl --location --request GET \ #curl --location --request GET \
# "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location/value" \ # "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location/value" \
# --header "Fiware-Service: carouge" \ # --header "Fiware-Service: carouge" \
# --header "Fiware-ServicePath: /watering" \ # --header "Fiware-ServicePath: /Watering" \
# | python -mjson.tool # | python -mjson.tool
\ No newline at end of file
...@@ -6,11 +6,11 @@ echo "Querying context broker at: ${ORION_HOST}" ...@@ -6,11 +6,11 @@ echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \ curl --location --request GET \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc" \ "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc" \
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
| python -mjson.tool | python -mjson.tool
#curl --location --request GET \ #curl --location --request GET \
# "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location" \ # "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location" \
# --header "Fiware-Service: carouge" \ # --header "Fiware-Service: carouge" \
# --header "Fiware-ServicePath: /watering" \ # --header "Fiware-ServicePath: /Watering" \
# | python -mjson.tool # | python -mjson.tool
\ No newline at end of file
...@@ -6,6 +6,6 @@ echo "Querying context broker at: ${ORION_HOST}" ...@@ -6,6 +6,6 @@ echo "Querying context broker at: ${ORION_HOST}"
curl --location --request PUT \ curl --location --request PUT \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\ "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
--header "Content-Type: text/plain" \ --header "Content-Type: text/plain" \
-d $(( RANDOM % 10 )) -d $(( RANDOM % 10 ))
...@@ -10,7 +10,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}" ...@@ -10,7 +10,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}"
curl --location --request POST \ curl --location --request POST \
"http://$ORION_HOST:1026/v2/subscriptions/" \ "http://$ORION_HOST:1026/v2/subscriptions/" \
--header "Fiware-Service: carouge" \ --header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \ --header "Fiware-ServicePath: /Watering" \
--header "Content-Type: application/json" \ --header "Content-Type: application/json" \
--header "Accept: application/json" \ --header "Accept: application/json" \
--data '{ --data '{
......
{"city_info": {"name": "Carouge", "country": "Suisse", "latitude": "46.1838613", "longitude": "6.1385878", "elevation": "388", "sunrise": "05:56", "sunset": "21:06"}, "current_condition": {"date": "18.05.2020", "hour": "14:00", "tmp": 23, "wnd_spd": 17, "wnd_gust": 27, "wnd_dir": "NE", "pressure": 1017.6, "humidity": 45, "condition": "Ensoleill\u00e9", "condition_key": "ensoleille", "icon": "https://www.prevision-meteo.ch/style/images/icon/ensoleille.png", "icon_big": "https://www.prevision-meteo.ch/style/images/icon/ensoleille-big.png"}}
\ No newline at end of file
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
# This is for building docker images for running the scripts
FROM python:3.8-slim-buster
MAINTAINER Federico Sismondi <fsismondi@udgalliance.org>
RUN apt-get update
RUN mkdir /code
WORKDIR /code
COPY requirements.txt requirements.txt
RUN pip install --timeout 300 -r requirements.txt
COPY . /code
CMD ["echo", "please override this CMD with some .sh or .py script call"]
# docker run -it --rm --name weather-data-collection-carouge --env ORION_HOST=10.81.6.109 naiades-client ./weather_02_get_all_raw_data.py
# docker run -it --rm --name weather-use-case-collection-carouge --env ORION_HOST=10.81.6.109 naiades-client ./security_01_get_token_with_password.sh
{"raw_data": "b'eyJjaXR5X2luZm8iOiB7Im5hbWUiOiAiQ2Fyb3VnZSIsICJjb3VudHJ5IjogIlN1aXNzZSIsICJsYXRpdHVkZSI6ICI0Ni4xODM4NjEzIiwgImxvbmdpdHVkZSI6ICI2LjEzODU4NzgiLCAiZWxldmF0aW9uIjogIjM4OCIsICJzdW5yaXNlIjogIjA1OjU1IiwgInN1bnNldCI6ICIyMTowOCJ9LCAiY3VycmVudF9jb25kaXRpb24iOiB7ImRhdGUiOiAiMTkuMDUuMjAyMCIsICJob3VyIjogIjE1OjAwIiwgInRtcCI6IDIzLCAid25kX3NwZCI6IDM2LCAid25kX2d1c3QiOiA1OCwgInduZF9kaXIiOiAiTkUiLCAicHJlc3N1cmUiOiAxMDEyLjksICJodW1pZGl0eSI6IDQ2LCAiY29uZGl0aW9uIjogIkVuc29sZWlsbFx1MDBlOSIsICJjb25kaXRpb25fa2V5IjogImVuc29sZWlsbGUiLCAiaWNvbiI6ICJodHRwczovL3d3dy5wcmV2aXNpb24tbWV0ZW8uY2gvc3R5bGUvaW1hZ2VzL2ljb24vZW5zb2xlaWxsZS5wbmciLCAiaWNvbl9iaWciOiAiaHR0cHM6Ly93d3cucHJldmlzaW9uLW1ldGVvLmNoL3N0eWxlL2ltYWdlcy9pY29uL2Vuc29sZWlsbGUtYmlnLnBuZyJ9fQ=='", "source": "https://www.prevision-meteo.ch/services/json/carouge"}
\ No newline at end of file
requests
\ No newline at end of file
#!/usr/bin/env python3
"""
Dumps weather data to disk each POST_DATA_PERIOD seconds
"""
import os
import json
import time
import logging
from weather_api import *
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.info("Starting to collect..")
COLLECTION_DIR = os.getenv('COLLECTION_DIR', default='.')
# Send updates in a loop
while True:
time.sleep(POST_DATA_PERIOD)
logger.info("Querying service..")
weather_data = get_data_from_weather_service()
logger.info("Dumping to disk..")
filepath = os.path.join(COLLECTION_DIR,'{}_weather_raw_data.json'.format(int(time.time())))
with open(filepath, encoding='utf-8', mode='w') as f:
json.dump(
obj=weather_data,
fp=f
)
\ No newline at end of file
...@@ -27,14 +27,14 @@ dump_keys = ['city_info', 'current_condition'] ...@@ -27,14 +27,14 @@ dump_keys = ['city_info', 'current_condition']
# see note above about id and type # see note above about id and type
fiware_data = { fiware_data = {
'id': 'urn:ngsi-ld:WeatherRawData:carouge_weather_observed_raw', 'id': 'urn:ngsi-ld:WeatherObservedRaw:carouge_weather_observed_raw',
'type': 'WeatherRawData' 'type': 'WeatherObservedRaw'
} }
# build header for POST # build header for POST
http_header_post = { http_header_post = {
'Fiware-Service': 'carouge', 'Fiware-Service': 'carouge',
'Fiware-ServicePath': '/Carouge/Weather/Raw', 'Fiware-ServicePath': '/Weather/Raw',
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Accept': 'application/json', 'Accept': 'application/json',
} }
...@@ -43,10 +43,13 @@ http_header_post = { ...@@ -43,10 +43,13 @@ http_header_post = {
http_header_get = http_header_post.copy() http_header_get = http_header_post.copy()
http_header_get.pop('Content-Type') http_header_get.pop('Content-Type')
logger.info('Config: \n\tPERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE)) logger.info('Config: \n\tPOST_DATA_PERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE))
def get_data_from_weather_service(): def get_data_from_weather_service():
"""
:return: Raw data (json as string)
"""
logger.info('Querying weather service..') logger.info('Querying weather service..')
try: try:
r = requests.get( r = requests.get(
...@@ -72,12 +75,6 @@ def create_weather_entity_in_platform(): ...@@ -72,12 +75,6 @@ def create_weather_entity_in_platform():
try: try:
entity = {**weather_data, **fiware_data} entity = {**weather_data, **fiware_data}
with open('weather_raw_last_data.json', encoding='utf-8', mode='w') as f:
json.dump(
obj=entity,
fp=f
)
response = requests.post( response = requests.post(
url=URL_BASE + '/v2/entities?options=keyValues', url=URL_BASE + '/v2/entities?options=keyValues',
headers=http_header_post, headers=http_header_post,
...@@ -91,7 +88,11 @@ def create_weather_entity_in_platform(): ...@@ -91,7 +88,11 @@ def create_weather_entity_in_platform():
if response and response.ok: if response and response.ok:
return True return True
else: else:
logger.error('Operation failed, got message: {}'.format(response.json())) if response:
logger.error('Operation failed, got message: {}'.format(response.json()))
else:
logger.error('Operation failed, no response received')
return False return False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment