diff --git a/.gitignore b/.gitignore index 8f63cb5887a88420e26434b70841cf213fa8db1d..5f281e77e1339c19286e5f77bf1cf15d14eec675 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ +.cpython-36.pyc .idea/ +weather_data_init.json +scripts/weather_raw_last_data.json wms-example/__pycache__/ diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/docker-compose.yml b/docker-compose.yml old mode 100644 new mode 100755 index 42d9477c64d5aedb8f23dccbdf8769ec1d53f1a9..0a49d630db3a347db15618c8c4fead77c7beccf1 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,10 @@ services: command: -dbhost mongo-db -logLevel INFO healthcheck: test: curl --fail -s http://orion:${ORION_PORT}/version || exit 1 + start_period: 40s + interval: 15m00s + timeout: 10s + retries: 3 # Keyrock is an Identity Management Front-End keyrock: @@ -58,6 +62,10 @@ services: - my_secret_data healthcheck: test: curl --fail -s http://localhost:${KEYROCK_PORT}/version || exit 1 + start_period: 40s + interval: 15m00s + timeout: 10s + retries: 3 # PEP Proxy for Orion orion-proxy: @@ -89,6 +97,10 @@ services: - PEP_PROXY_PUBLIC_PATHS=/version healthcheck: test: curl --fail -s http://orion-proxy:${ORION_PROXY_PORT}/version || exit 1 + start_period: 40s + interval: 15m00s + timeout: 10s + retries: 3 # Databases mongo-db: @@ -179,10 +191,13 @@ services: healthcheck: test: curl --fail -s http://fiware-quantumleap:${QUANTUMLEAP_PORT}/v2/version || exit 1 start_period: 40s - interval: 1m30s + interval: 15m00s timeout: 10s retries: 3 + +# - - - services which act as clients of the NAIADES platform + wms-app-example: build: context: ./wms-example @@ -204,6 +219,61 @@ services: ipv4_address: 172.18.1.11 healthcheck: test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1 + start_period: 40s + interval: 15m00s + timeout: 10s + retries: 3 + + + weather-data-collection-carouge-raw: + build: + context: ./weather-data-collection-carouge + dockerfile: Dockerfile + image: weather-data-collection-carouge:latest + container_name: weather-data-collection-carouge-raw + environment: + - LC_ALL=C.UTF-8 + - LANG=C.UTF-8 + - POST_DATA_PERIOD=3600 #3600sec = 1hour + - ORION_HOST=orion + - COLLECTION_DIR=/code/raw_data + volumes: + - ./weather-data-collection-carouge:/code:rw + networks: + default: + ipv4_address: 172.18.1.13 + command: ./weather_00_collect_raw_weather_data_carouge +# healthcheck: +# test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1 +# start_period: 40s +# interval: 15m00s +# timeout: 10s +# retries: 3 + +# weather-data-collection-carouge: +# build: +# context: ./weather-data-collection-carouge +# dockerfile: Dockerfile +# image: weather-data-collection-carouge:latest +# container_name: weather-data-collection-carouge +# environment: +# - LC_ALL=C.UTF-8 +# - LANG=C.UTF-8 +# - POST_DATA_PERIOD=15 +# - ORION_HOST=orion +# volumes: +# - ./weather-data-collection-carouge:/code:rw +# networks: +# default: +# ipv4_address: 172.18.1.14 +# command: ./weather_01_collect_and_post_weather_data_carouge.py +## healthcheck: +## test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1 +## start_period: 40s +## interval: 15m00s +## timeout: 10s +## retries: 3 + networks: default: diff --git a/scripts/admin_get_all_orion_susbcriptions.sh b/scripts/admin_get_all_orion_susbcriptions.sh index de3b8ff29618ba4dcabfb217c58cfa42506a944d..f68f75a15dcdadd3742a858a870ffac8bb2428c6 100755 --- a/scripts/admin_get_all_orion_susbcriptions.sh +++ b/scripts/admin_get_all_orion_susbcriptions.sh @@ -6,6 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}" curl --location --request GET \ "http://$ORION_HOST:1026/v2/subscriptions/" \ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ | python -mjson.tool diff --git a/scripts/security_02_request_without_token.sh b/scripts/security_02_request_without_token.sh index 0c8edcc8bee5611c47818c587af85e427515d4c8..e1f63e7b166b70684ae17cec2c81d576cdbb44d3 100755 --- a/scripts/security_02_request_without_token.sh +++ b/scripts/security_02_request_without_token.sh @@ -8,4 +8,4 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}" curl -X GET \ "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" + --header "Fiware-ServicePath: /Watering" diff --git a/scripts/security_03_request_with_token.sh b/scripts/security_03_request_with_token.sh index 2c5ca24cb5a23824f505a0616c92f331b0e03145..bf49183850c48faea3184f9d4b97a8596d7e8f10 100755 --- a/scripts/security_03_request_with_token.sh +++ b/scripts/security_03_request_with_token.sh @@ -10,6 +10,6 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}" curl -X GET \ "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ --header "X-Auth-Token: $KEYROCK_TOKEN" diff --git a/scripts/security_04_update_entity_with_token.sh b/scripts/security_04_update_entity_with_token.sh index 0dc701370785dcfb2f6ce2fc0be23aea148de0c5..0e3b8e7ed12b0692401a4af57739090ae58e40f9 100755 --- a/scripts/security_04_update_entity_with_token.sh +++ b/scripts/security_04_update_entity_with_token.sh @@ -10,7 +10,7 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}" curl --location --request PUT \ "http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ --header "X-Auth-Token: $KEYROCK_TOKEN" \ --header "Content-Type: text/plain" \ -d $(( RANDOM % 10 )) diff --git a/scripts/security_05_subscribe_to_entity_with_token.sh b/scripts/security_05_subscribe_to_entity_with_token.sh index 1885335d3f0c71f1f7369bcd845169615ec77331..56f0f47a6d695b001084a7e2c2c38286226d6f83 100755 --- a/scripts/security_05_subscribe_to_entity_with_token.sh +++ b/scripts/security_05_subscribe_to_entity_with_token.sh @@ -17,7 +17,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}" curl --location --request POST \ "http://$ORION_HOST:1026/v2/subscriptions/" \ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ --header "Content-Type: application/json" \ --header "Accept: application/json" \ --header "X-Auth-Token: $KEYROCK_TOKEN" \ diff --git a/scripts/watering_01_create_entity.sh b/scripts/watering_01_create_entity.sh index 25fc74418e3e3d0ad7d9c9036aa7ae35af4d33ea..69d5e692235e58e2c227d0487e84d5293373f047 100755 --- a/scripts/watering_01_create_entity.sh +++ b/scripts/watering_01_create_entity.sh @@ -6,7 +6,7 @@ echo "Querying context broker at: ${ORION_HOST}" curl -iX POST \ "http://$ORION_HOST:1026/v2/entities/" \ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ --header "Content-Type: application/json" \ --data '{ "id": "urn:ngsi-ld:FlowerBed:FlowerBed-345", diff --git a/scripts/watering_02_retrieve_entities.sh b/scripts/watering_02_retrieve_entities.sh index c0071ca740a2a07fb5d3e162001597deacee6c88..f4ed0da0fc2dfb0768ccc1f77c6b11db1039bd07 100755 --- a/scripts/watering_02_retrieve_entities.sh +++ b/scripts/watering_02_retrieve_entities.sh @@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}" curl --location --request GET \ "http://$ORION_HOST:1026/v2/entities/" \ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ | python -mjson.tool diff --git a/scripts/watering_03_retrieve_entity_keyValues_by_id.sh b/scripts/watering_03_retrieve_entity_keyValues_by_id.sh index 1ed4e4b98f6cbb509257b0c7c369e76645a6d9fc..4c06d6c961c2be2ae7b5427f838d70646e29530a 100755 --- a/scripts/watering_03_retrieve_entity_keyValues_by_id.sh +++ b/scripts/watering_03_retrieve_entity_keyValues_by_id.sh @@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}" curl --location --request GET \ "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345?options=keyValues" \ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ | python -mjson.tool \ No newline at end of file diff --git a/scripts/watering_04_retrieve_attribute_value.sh b/scripts/watering_04_retrieve_attribute_value.sh index ea6e93e4c2d409eeae00a9b7fe6b347dbc4f0208..160d512a5ead0a67f01c338028d48f1f3b543b1d 100755 --- a/scripts/watering_04_retrieve_attribute_value.sh +++ b/scripts/watering_04_retrieve_attribute_value.sh @@ -6,11 +6,11 @@ echo "Querying context broker at: ${ORION_HOST}" curl --location --request GET \ "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value" \ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ | python -mjson.tool #curl --location --request GET \ # "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location/value" \ # --header "Fiware-Service: carouge" \ -# --header "Fiware-ServicePath: /watering" \ +# --header "Fiware-ServicePath: /Watering" \ # | python -mjson.tool \ No newline at end of file diff --git a/scripts/watering_05_retrieve_attribute.sh b/scripts/watering_05_retrieve_attribute.sh index c5439857fabe066d64cfdd546042ee0225efb406..ba547c19cdc4d6bf744d865e86e4c8017ee7896b 100755 --- a/scripts/watering_05_retrieve_attribute.sh +++ b/scripts/watering_05_retrieve_attribute.sh @@ -6,11 +6,11 @@ echo "Querying context broker at: ${ORION_HOST}" curl --location --request GET \ "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc" \ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ | python -mjson.tool #curl --location --request GET \ # "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location" \ # --header "Fiware-Service: carouge" \ -# --header "Fiware-ServicePath: /watering" \ +# --header "Fiware-ServicePath: /Watering" \ # | python -mjson.tool \ No newline at end of file diff --git a/scripts/watering_06_update_specific_attrs.sh b/scripts/watering_06_update_specific_attrs.sh index aa863051e930034517831b04b4071b134c0f51b3..91d91842fef16630ea1c9e2e0630e7fdae02a308 100755 --- a/scripts/watering_06_update_specific_attrs.sh +++ b/scripts/watering_06_update_specific_attrs.sh @@ -6,6 +6,6 @@ echo "Querying context broker at: ${ORION_HOST}" curl --location --request PUT \ "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ --header "Content-Type: text/plain" \ -d $(( RANDOM % 10 )) diff --git a/scripts/watering_07_subscribe_to_entity.sh b/scripts/watering_07_subscribe_to_entity.sh index 33fbd4453a1e9dfe9aae9883407fa786b777f915..e5ae0454a31daa66dd1637c46e0b87ce4ed7daa0 100755 --- a/scripts/watering_07_subscribe_to_entity.sh +++ b/scripts/watering_07_subscribe_to_entity.sh @@ -10,7 +10,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}" curl --location --request POST \ "http://$ORION_HOST:1026/v2/subscriptions/" \ --header "Fiware-Service: carouge" \ - --header "Fiware-ServicePath: /watering" \ + --header "Fiware-ServicePath: /Watering" \ --header "Content-Type: application/json" \ --header "Accept: application/json" \ --data '{ diff --git a/scripts/weather_raw_last_data.json b/scripts/weather_raw_last_data.json deleted file mode 100644 index b38f2f1c3546c84ab94b9c2b3c05514fbb9a2a8f..0000000000000000000000000000000000000000 --- a/scripts/weather_raw_last_data.json +++ /dev/null @@ -1 +0,0 @@ -{"city_info": {"name": "Carouge", "country": "Suisse", "latitude": "46.1838613", "longitude": "6.1385878", "elevation": "388", "sunrise": "05:56", "sunset": "21:06"}, "current_condition": {"date": "18.05.2020", "hour": "14:00", "tmp": 23, "wnd_spd": 17, "wnd_gust": 27, "wnd_dir": "NE", "pressure": 1017.6, "humidity": 45, "condition": "Ensoleill\u00e9", "condition_key": "ensoleille", "icon": "https://www.prevision-meteo.ch/style/images/icon/ensoleille.png", "icon_big": "https://www.prevision-meteo.ch/style/images/icon/ensoleille-big.png"}} \ No newline at end of file diff --git a/scripts/xx_todo.sh b/scripts/xx_todo.sh old mode 100644 new mode 100755 diff --git a/secrets b/secrets old mode 100644 new mode 100755 diff --git a/weather-data-collection-carouge/Dockerfile b/weather-data-collection-carouge/Dockerfile new file mode 100755 index 0000000000000000000000000000000000000000..bd3b6b034ee029e2f3ec16d9b263f5f0c01b8c48 --- /dev/null +++ b/weather-data-collection-carouge/Dockerfile @@ -0,0 +1,18 @@ +# This is for building docker images for running the scripts + +FROM python:3.8-slim-buster +MAINTAINER Federico Sismondi + +RUN apt-get update + +RUN mkdir /code +WORKDIR /code + +COPY requirements.txt requirements.txt +RUN pip install --timeout 300 -r requirements.txt + +COPY . /code +CMD ["echo", "please override this CMD with some .sh or .py script call"] + +# docker run -it --rm --name weather-data-collection-carouge --env ORION_HOST=10.81.6.109 naiades-client ./weather_02_get_all_raw_data.py +# docker run -it --rm --name weather-use-case-collection-carouge --env ORION_HOST=10.81.6.109 naiades-client ./security_01_get_token_with_password.sh diff --git a/weather-data-collection-carouge/raw_data/weather_raw_last_data.json b/weather-data-collection-carouge/raw_data/weather_raw_last_data.json new file mode 100755 index 0000000000000000000000000000000000000000..7b4cf213517a7bed1cc44ade051e9410d85bdd29 --- /dev/null +++ b/weather-data-collection-carouge/raw_data/weather_raw_last_data.json @@ -0,0 +1 @@ +{"raw_data": "b'eyJjaXR5X2luZm8iOiB7Im5hbWUiOiAiQ2Fyb3VnZSIsICJjb3VudHJ5IjogIlN1aXNzZSIsICJsYXRpdHVkZSI6ICI0Ni4xODM4NjEzIiwgImxvbmdpdHVkZSI6ICI2LjEzODU4NzgiLCAiZWxldmF0aW9uIjogIjM4OCIsICJzdW5yaXNlIjogIjA1OjU1IiwgInN1bnNldCI6ICIyMTowOCJ9LCAiY3VycmVudF9jb25kaXRpb24iOiB7ImRhdGUiOiAiMTkuMDUuMjAyMCIsICJob3VyIjogIjE1OjAwIiwgInRtcCI6IDIzLCAid25kX3NwZCI6IDM2LCAid25kX2d1c3QiOiA1OCwgInduZF9kaXIiOiAiTkUiLCAicHJlc3N1cmUiOiAxMDEyLjksICJodW1pZGl0eSI6IDQ2LCAiY29uZGl0aW9uIjogIkVuc29sZWlsbFx1MDBlOSIsICJjb25kaXRpb25fa2V5IjogImVuc29sZWlsbGUiLCAiaWNvbiI6ICJodHRwczovL3d3dy5wcmV2aXNpb24tbWV0ZW8uY2gvc3R5bGUvaW1hZ2VzL2ljb24vZW5zb2xlaWxsZS5wbmciLCAiaWNvbl9iaWciOiAiaHR0cHM6Ly93d3cucHJldmlzaW9uLW1ldGVvLmNoL3N0eWxlL2ltYWdlcy9pY29uL2Vuc29sZWlsbGUtYmlnLnBuZyJ9fQ=='", "source": "https://www.prevision-meteo.ch/services/json/carouge"} \ No newline at end of file diff --git a/weather-data-collection-carouge/requirements.txt b/weather-data-collection-carouge/requirements.txt new file mode 100755 index 0000000000000000000000000000000000000000..663bd1f6a2ae02f29df59fb4963c17934034f731 --- /dev/null +++ b/weather-data-collection-carouge/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/weather-data-collection-carouge/weather_00_collect_raw_weather_data_carouge.py b/weather-data-collection-carouge/weather_00_collect_raw_weather_data_carouge.py new file mode 100755 index 0000000000000000000000000000000000000000..2a307b59f1b169f899252db099a89224400837bd --- /dev/null +++ b/weather-data-collection-carouge/weather_00_collect_raw_weather_data_carouge.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +""" +Dumps weather data to disk each POST_DATA_PERIOD seconds +""" + +import os +import json +import time +import logging +from weather_api import * + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +logger.info("Starting to collect..") + +COLLECTION_DIR = os.getenv('COLLECTION_DIR', default='.') + +# Send updates in a loop +while True: + time.sleep(POST_DATA_PERIOD) + logger.info("Querying service..") + weather_data = get_data_from_weather_service() + logger.info("Dumping to disk..") + filepath = os.path.join(COLLECTION_DIR,'{}_weather_raw_data.json'.format(int(time.time()))) + with open(filepath, encoding='utf-8', mode='w') as f: + json.dump( + obj=weather_data, + fp=f + ) \ No newline at end of file diff --git a/scripts/weather_01_send_caroute_data.py b/weather-data-collection-carouge/weather_01_collect_and_post_weather_data_carouge.py similarity index 100% rename from scripts/weather_01_send_caroute_data.py rename to weather-data-collection-carouge/weather_01_collect_and_post_weather_data_carouge.py diff --git a/scripts/weather_02_get_all_raw_data.py b/weather-data-collection-carouge/weather_02_get_all_raw_data.py similarity index 100% rename from scripts/weather_02_get_all_raw_data.py rename to weather-data-collection-carouge/weather_02_get_all_raw_data.py diff --git a/scripts/weather_api.py b/weather-data-collection-carouge/weather_api.py old mode 100644 new mode 100755 similarity index 89% rename from scripts/weather_api.py rename to weather-data-collection-carouge/weather_api.py index 3f786ceedb10788b972f2b54614241cfb1c6aee7..fb34cbe441553adb62ac6238f3b39c2cb2664d17 --- a/scripts/weather_api.py +++ b/weather-data-collection-carouge/weather_api.py @@ -27,14 +27,14 @@ dump_keys = ['city_info', 'current_condition'] # see note above about id and type fiware_data = { - 'id': 'urn:ngsi-ld:WeatherRawData:carouge_weather_observed_raw', - 'type': 'WeatherRawData' + 'id': 'urn:ngsi-ld:WeatherObservedRaw:carouge_weather_observed_raw', + 'type': 'WeatherObservedRaw' } # build header for POST http_header_post = { 'Fiware-Service': 'carouge', - 'Fiware-ServicePath': '/Carouge/Weather/Raw', + 'Fiware-ServicePath': '/Weather/Raw', 'Content-Type': 'application/json', 'Accept': 'application/json', } @@ -43,10 +43,13 @@ http_header_post = { http_header_get = http_header_post.copy() http_header_get.pop('Content-Type') -logger.info('Config: \n\tPERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE)) +logger.info('Config: \n\tPOST_DATA_PERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE)) def get_data_from_weather_service(): + """ + :return: Raw data (json as string) + """ logger.info('Querying weather service..') try: r = requests.get( @@ -72,12 +75,6 @@ def create_weather_entity_in_platform(): try: entity = {**weather_data, **fiware_data} - with open('weather_raw_last_data.json', encoding='utf-8', mode='w') as f: - json.dump( - obj=entity, - fp=f - ) - response = requests.post( url=URL_BASE + '/v2/entities?options=keyValues', headers=http_header_post, @@ -91,7 +88,11 @@ def create_weather_entity_in_platform(): if response and response.ok: return True else: - logger.error('Operation failed, got message: {}'.format(response.json())) + if response: + logger.error('Operation failed, got message: {}'.format(response.json())) + else: + logger.error('Operation failed, no response received') + return False