Feat() data collection in raw format, and sketch of first version of collect...

Feat() data collection in raw format, and sketch of first version of collect to historical data base (w/ QuantumLeap)

.gitignore

+.cpython-36.pyc
 .idea/
+weather_data_init.json
+scripts/weather_raw_last_data.json
 wms-example/__pycache__/

docker-compose.yml

@@ -30,6 +30,10 @@ services:
     command: -dbhost mongo-db -logLevel INFO
     healthcheck:
       test: curl --fail -s http://orion:${ORION_PORT}/version || exit 1
+      start_period: 40s
+      interval: 15m00s
+      timeout: 10s
+      retries: 3
 
   # Keyrock is an Identity Management Front-End
   keyrock:
@@ -58,6 +62,10 @@ services:
       - my_secret_data
     healthcheck:
       test: curl --fail -s http://localhost:${KEYROCK_PORT}/version || exit 1
+      start_period: 40s
+      interval: 15m00s
+      timeout: 10s
+      retries: 3
 
   # PEP Proxy for Orion
   orion-proxy:
@@ -89,6 +97,10 @@ services:
       - PEP_PROXY_PUBLIC_PATHS=/version
     healthcheck:
       test: curl --fail -s http://orion-proxy:${ORION_PROXY_PORT}/version || exit 1
+      start_period: 40s
+      interval: 15m00s
+      timeout: 10s
+      retries: 3
 
   # Databases
   mongo-db:
@@ -179,10 +191,13 @@ services:
     healthcheck:
       test: curl --fail -s http://fiware-quantumleap:${QUANTUMLEAP_PORT}/v2/version || exit 1
       start_period: 40s
-      interval: 1m30s
+      interval: 15m00s
       timeout: 10s
       retries: 3
 
+
+# - - - services which act as clients of the NAIADES platform
+
   wms-app-example:
     build:
       context: ./wms-example
@@ -204,6 +219,61 @@ services:
         ipv4_address: 172.18.1.11
     healthcheck:
       test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
+      start_period: 40s
+      interval: 15m00s
+      timeout: 10s
+      retries: 3
+
+
+  weather-data-collection-carouge-raw:
+    build:
+      context: ./weather-data-collection-carouge
+      dockerfile: Dockerfile
+    image: weather-data-collection-carouge:latest
+    container_name: weather-data-collection-carouge-raw
+    environment:
+      - LC_ALL=C.UTF-8
+      - LANG=C.UTF-8
+      - POST_DATA_PERIOD=3600 #3600sec = 1hour
+      - ORION_HOST=orion
+      - COLLECTION_DIR=/code/raw_data
+    volumes:
+      - ./weather-data-collection-carouge:/code:rw
+    networks:
+      default:
+        ipv4_address: 172.18.1.13
+    command: ./weather_00_collect_raw_weather_data_carouge
+#    healthcheck:
+#      test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
+#      start_period: 40s
+#      interval: 15m00s
+#      timeout: 10s
+#      retries: 3
+
+#  weather-data-collection-carouge:
+#    build:
+#      context: ./weather-data-collection-carouge
+#      dockerfile: Dockerfile
+#    image: weather-data-collection-carouge:latest
+#    container_name: weather-data-collection-carouge
+#    environment:
+#      - LC_ALL=C.UTF-8
+#      - LANG=C.UTF-8
+#      - POST_DATA_PERIOD=15
+#      - ORION_HOST=orion
+#    volumes:
+#      - ./weather-data-collection-carouge:/code:rw
+#    networks:
+#      default:
+#        ipv4_address: 172.18.1.14
+#    command: ./weather_01_collect_and_post_weather_data_carouge.py
+##    healthcheck:
+##      test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
+##      start_period: 40s
+##      interval: 15m00s
+##      timeout: 10s
+##      retries: 3
+
 
 networks:
   default:

scripts/admin_get_all_orion_susbcriptions.sh

@@ -6,6 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}"
 curl --location --request GET \
     "http://$ORION_HOST:1026/v2/subscriptions/" \
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
     | python -mjson.tool
 

scripts/security_02_request_without_token.sh

@@ -8,4 +8,4 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
 curl -X GET \
   "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
   --header "Fiware-Service: carouge" \
-  --header "Fiware-ServicePath: /watering"
+  --header "Fiware-ServicePath: /Watering"

scripts/security_03_request_with_token.sh

@@ -10,6 +10,6 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
 curl -X GET \
   "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
   --header "Fiware-Service: carouge" \
-  --header "Fiware-ServicePath: /watering" \
+  --header "Fiware-ServicePath: /Watering" \
   --header "X-Auth-Token: $KEYROCK_TOKEN"
 

scripts/security_04_update_entity_with_token.sh

@@ -10,7 +10,7 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
 curl --location --request PUT \
     "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     --header "X-Auth-Token: $KEYROCK_TOKEN" \
     --header "Content-Type: text/plain" \
     -d $(( RANDOM % 10 ))

scripts/security_05_subscribe_to_entity_with_token.sh

@@ -17,7 +17,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}"
 curl --location --request POST \
     "http://$ORION_HOST:1026/v2/subscriptions/" \
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     --header "Content-Type: application/json" \
     --header "Accept: application/json" \
     --header "X-Auth-Token: $KEYROCK_TOKEN" \

scripts/watering_01_create_entity.sh

@@ -6,7 +6,7 @@ echo "Querying context broker at: ${ORION_HOST}"
 curl -iX POST \
     "http://$ORION_HOST:1026/v2/entities/" \
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     --header "Content-Type: application/json" \
     --data '{
         "id": "urn:ngsi-ld:FlowerBed:FlowerBed-345",

scripts/watering_02_retrieve_entities.sh

@@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}"
 curl --location --request GET \
     "http://$ORION_HOST:1026/v2/entities/" \
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     | python -mjson.tool

scripts/watering_03_retrieve_entity_keyValues_by_id.sh

@@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}"
 curl --location --request GET \
     "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345?options=keyValues" \
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     | python -mjson.tool
\ No newline at end of file

scripts/watering_04_retrieve_attribute_value.sh

@@ -6,11 +6,11 @@ echo "Querying context broker at: ${ORION_HOST}"
 curl --location --request GET \
     "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value" \
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     | python -mjson.tool
 
 #curl --location --request GET \
 #    "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location/value" \
 #    --header "Fiware-Service: carouge" \
-#    --header "Fiware-ServicePath: /watering" \
+#    --header "Fiware-ServicePath: /Watering" \
 #    | python -mjson.tool
\ No newline at end of file

scripts/watering_05_retrieve_attribute.sh

@@ -6,11 +6,11 @@ echo "Querying context broker at: ${ORION_HOST}"
 curl --location --request GET \
     "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc" \
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     | python -mjson.tool
 
 #curl --location --request GET \
 #    "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location" \
 #    --header "Fiware-Service: carouge" \
-#    --header "Fiware-ServicePath: /watering" \
+#    --header "Fiware-ServicePath: /Watering" \
 #    | python -mjson.tool
\ No newline at end of file

scripts/watering_06_update_specific_attrs.sh

@@ -6,6 +6,6 @@ echo "Querying context broker at: ${ORION_HOST}"
 curl --location --request PUT \
     "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     --header "Content-Type: text/plain" \
     -d $(( RANDOM % 10 ))

scripts/watering_07_subscribe_to_entity.sh

@@ -10,7 +10,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}"
 curl --location --request POST \
     "http://$ORION_HOST:1026/v2/subscriptions/" \
     --header "Fiware-Service: carouge" \
-    --header "Fiware-ServicePath: /watering" \
+    --header "Fiware-ServicePath: /Watering" \
     --header "Content-Type: application/json" \
     --header "Accept: application/json" \
     --data '{

scripts/weather_raw_last_data.json

-{"city_info": {"name": "Carouge", "country": "Suisse", "latitude": "46.1838613", "longitude": "6.1385878", "elevation": "388", "sunrise": "05:56", "sunset": "21:06"}, "current_condition": {"date": "18.05.2020", "hour": "14:00", "tmp": 23, "wnd_spd": 17, "wnd_gust": 27, "wnd_dir": "NE", "pressure": 1017.6, "humidity": 45, "condition": "Ensoleill\u00e9", "condition_key": "ensoleille", "icon": "https://www.prevision-meteo.ch/style/images/icon/ensoleille.png", "icon_big": "https://www.prevision-meteo.ch/style/images/icon/ensoleille-big.png"}}
\ No newline at end of file

weather-data-collection-carouge/Dockerfile

+# This is for building docker images for running the scripts
+
+FROM python:3.8-slim-buster
+MAINTAINER Federico Sismondi <fsismondi@udgalliance.org>
+
+RUN apt-get update
+
+RUN mkdir /code
+WORKDIR /code
+
+COPY requirements.txt requirements.txt
+RUN pip install --timeout 300 -r requirements.txt
+
+COPY . /code
+CMD ["echo", "please override this CMD with some .sh or .py script call"]
+
+# docker run -it --rm --name weather-data-collection-carouge --env ORION_HOST=10.81.6.109 naiades-client ./weather_02_get_all_raw_data.py
+# docker run -it --rm --name weather-use-case-collection-carouge --env ORION_HOST=10.81.6.109 naiades-client ./security_01_get_token_with_password.sh

weather-data-collection-carouge/raw_data/weather_raw_last_data.json

+{"raw_data": "b'eyJjaXR5X2luZm8iOiB7Im5hbWUiOiAiQ2Fyb3VnZSIsICJjb3VudHJ5IjogIlN1aXNzZSIsICJsYXRpdHVkZSI6ICI0Ni4xODM4NjEzIiwgImxvbmdpdHVkZSI6ICI2LjEzODU4NzgiLCAiZWxldmF0aW9uIjogIjM4OCIsICJzdW5yaXNlIjogIjA1OjU1IiwgInN1bnNldCI6ICIyMTowOCJ9LCAiY3VycmVudF9jb25kaXRpb24iOiB7ImRhdGUiOiAiMTkuMDUuMjAyMCIsICJob3VyIjogIjE1OjAwIiwgInRtcCI6IDIzLCAid25kX3NwZCI6IDM2LCAid25kX2d1c3QiOiA1OCwgInduZF9kaXIiOiAiTkUiLCAicHJlc3N1cmUiOiAxMDEyLjksICJodW1pZGl0eSI6IDQ2LCAiY29uZGl0aW9uIjogIkVuc29sZWlsbFx1MDBlOSIsICJjb25kaXRpb25fa2V5IjogImVuc29sZWlsbGUiLCAiaWNvbiI6ICJodHRwczovL3d3dy5wcmV2aXNpb24tbWV0ZW8uY2gvc3R5bGUvaW1hZ2VzL2ljb24vZW5zb2xlaWxsZS5wbmciLCAiaWNvbl9iaWciOiAiaHR0cHM6Ly93d3cucHJldmlzaW9uLW1ldGVvLmNoL3N0eWxlL2ltYWdlcy9pY29uL2Vuc29sZWlsbGUtYmlnLnBuZyJ9fQ=='", "source": "https://www.prevision-meteo.ch/services/json/carouge"}
\ No newline at end of file

weather-data-collection-carouge/requirements.txt

+requests
\ No newline at end of file

weather-data-collection-carouge/weather_00_collect_raw_weather_data_carouge.py

+#!/usr/bin/env python3
+
+"""
+Dumps weather data to disk each POST_DATA_PERIOD seconds
+"""
+
+import os
+import json
+import time
+import logging
+from weather_api import *
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+logger.info("Starting to collect..")
+
+COLLECTION_DIR = os.getenv('COLLECTION_DIR', default='.')
+
+# Send updates in a loop
+while True:
+    time.sleep(POST_DATA_PERIOD)
+    logger.info("Querying service..")
+    weather_data = get_data_from_weather_service()
+    logger.info("Dumping to disk..")
+    filepath = os.path.join(COLLECTION_DIR,'{}_weather_raw_data.json'.format(int(time.time())))
+    with open(filepath, encoding='utf-8', mode='w') as f:
+        json.dump(
+            obj=weather_data,
+            fp=f
+        )
\ No newline at end of file

scripts/weather_api.py → weather-data-collection-carouge/weather_api.py

@@ -27,14 +27,14 @@ dump_keys = ['city_info', 'current_condition']
 
 # see note above about id and type
 fiware_data = {
-    'id': 'urn:ngsi-ld:WeatherRawData:carouge_weather_observed_raw',
-    'type': 'WeatherRawData'
+    'id': 'urn:ngsi-ld:WeatherObservedRaw:carouge_weather_observed_raw',
+    'type': 'WeatherObservedRaw'
 }
 
 # build header for POST
 http_header_post = {
     'Fiware-Service': 'carouge',
-    'Fiware-ServicePath': '/Carouge/Weather/Raw',
+    'Fiware-ServicePath': '/Weather/Raw',
     'Content-Type': 'application/json',
     'Accept': 'application/json',
 }
@@ -43,10 +43,13 @@ http_header_post = {
 http_header_get = http_header_post.copy()
 http_header_get.pop('Content-Type')
 
-logger.info('Config: \n\tPERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE))
+logger.info('Config: \n\tPOST_DATA_PERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE))
 
 
 def get_data_from_weather_service():
+    """
+    :return: Raw data (json as string)
+    """
     logger.info('Querying weather service..')
     try:
         r = requests.get(
@@ -72,12 +75,6 @@ def create_weather_entity_in_platform():
     try:
         entity = {**weather_data, **fiware_data}
 
-        with open('weather_raw_last_data.json', encoding='utf-8', mode='w') as f:
-            json.dump(
-                obj=entity,
-                fp=f
-            )
-
         response = requests.post(
             url=URL_BASE + '/v2/entities?options=keyValues',
             headers=http_header_post,
@@ -91,7 +88,11 @@ def create_weather_entity_in_platform():
     if response and response.ok:
         return True
     else:
-        logger.error('Operation failed, got message: {}'.format(response.json()))
+        if response:
+            logger.error('Operation failed, got message: {}'.format(response.json()))
+        else:
+            logger.error('Operation failed, no response received')
+
         return False