Commit 1fcaf70b authored by Federico Sismondi's avatar Federico Sismondi

Merge branch 'develop' into 'master'

Develop

See merge request !3
parents 733df5b9 7b1b1063
.cpython-36.pyc
.idea/
*weather_raw_data.json
weather_data_init.json
scripts/weather_raw_last_data.json
wms-example/__pycache__/
File mode changed from 100644 to 100755
......@@ -114,10 +114,18 @@ UPDATE|
cd naiades-platform-poc
docker-compose up -d
```
## Put platform down
```bash
cd naiades-platform-poc
docker-compose down
```
## Check services' status
```bash
>>> docker-compose ps
docker-compose ps
Name Command State Ports
-------------------------------------------------------------------------------------------------------------------------
......@@ -133,61 +141,6 @@ mongo-express tini -- /docker-entrypoint ... Up 0.
## Run demo using shell scripts in `scripts` folder
Let's start by looking at the type of REST requests used in the scripts:
```bash
cat 01_create_entity.sh
curl --location --request POST 'http://localhost:1026/v2/entities/' \
--header 'Content-Type: application/json' \
--data-raw '{
"id": "urn:ngsi-ld:FlowerBed:FlowerBed-345",
"type": "FlowerBed",
"category": {
"type": "Property",
"value": ["urbanTreeSpot"]
},
"soilMoistureVwc": {
"type": "Property",
"value": 0.85
},
"dateLastWatering": {
"type": "DateTime",
"value": "2020-02-11T08:00:00Z"
},
"soilTemperature": {
"type": "Property",
"value": 17
},
"address": {
"type": "Property",
"value": {
"addressCountry": "Switzerland",
"streetAddress": "Place de Sardaigne",
"adressLocality": "Carouge",
"type": "PostalAddress"
}
},
"location": {
"type": "GeoProperty",
"value": {
"type": "Point",
"coordinates": [46.184255,6.1378375]
}
}
}'
```
Now, let's run the script:
```bash
cd scripts
./01_create_entity.sh
```
Please see `getting started` documentation at this [link](https://gitlab.distantaccess.com/naiades/naiades-platform-poc/-/wikis/API-client-examples-(curl-and-python-scripts))
## Put platform down
``` bash
cd naiades-platform-poc
docker-compose down
```
......@@ -30,6 +30,10 @@ services:
command: -dbhost mongo-db -logLevel INFO
healthcheck:
test: curl --fail -s http://orion:${ORION_PORT}/version || exit 1
start_period: 40s
interval: 15m00s
timeout: 10s
retries: 3
# Keyrock is an Identity Management Front-End
keyrock:
......@@ -58,6 +62,10 @@ services:
- my_secret_data
healthcheck:
test: curl --fail -s http://localhost:${KEYROCK_PORT}/version || exit 1
start_period: 40s
interval: 15m00s
timeout: 10s
retries: 3
# PEP Proxy for Orion
orion-proxy:
......@@ -89,6 +97,10 @@ services:
- PEP_PROXY_PUBLIC_PATHS=/version
healthcheck:
test: curl --fail -s http://orion-proxy:${ORION_PROXY_PORT}/version || exit 1
start_period: 40s
interval: 15m00s
timeout: 10s
retries: 3
# Databases
mongo-db:
......@@ -179,10 +191,13 @@ services:
healthcheck:
test: curl --fail -s http://fiware-quantumleap:${QUANTUMLEAP_PORT}/v2/version || exit 1
start_period: 40s
interval: 1m30s
interval: 15m00s
timeout: 10s
retries: 3
# - - - services which act as clients of the NAIADES platform
wms-app-example:
build:
context: ./wms-example
......@@ -204,6 +219,62 @@ services:
ipv4_address: 172.18.1.11
healthcheck:
test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
start_period: 40s
interval: 15m00s
timeout: 10s
retries: 3
weather-data-collection-carouge-raw:
build:
context: ./weather-data-collection-carouge
dockerfile: Dockerfile
image: weather-data-collection-carouge:latest
container_name: weather-data-collection-carouge-raw
environment:
- LC_ALL=C.UTF-8
- LANG=C.UTF-8
- POST_DATA_PERIOD=3600 #3600sec = 1hour
- ORION_HOST=orion
- COLLECTION_DIR=/code/raw_data
volumes:
- ./weather-data-collection-carouge:/code:rw
networks:
default:
ipv4_address: 172.18.1.13
command: ./weather_00_collect_raw_weather_data_carouge.py
# healthcheck:
# test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
# start_period: 40s
# interval: 15m00s
# timeout: 10s
# retries: 3
# weather-data-collection-carouge:
# build:
# context: ./weather-data-collection-carouge
# dockerfile: Dockerfile
# image: weather-data-collection-carouge:latest
# container_name: weather-data-collection-carouge
# environment:
# - LC_ALL=C.UTF-8
# - LANG=C.UTF-8
# - POST_DATA_PERIOD=15
# - ORION_HOST=orion
# volumes:
# - ./weather-data-collection-carouge:/code:rw
# networks:
# default:
# ipv4_address: 172.18.1.14
# command: ./weather_01_collect_and_post_weather_data_carouge.py
## healthcheck:
## test: curl --fail -s http://wms-app-example:${WMS_MODULE_PORT}/healthcheck || exit 1
## start_period: 40s
## interval: 15m00s
## timeout: 10s
## retries: 3
networks:
default:
......
# This is for building docker images for running the scripts
FROM python:3.8-slim-buster
MAINTAINER Federico Sismondi <fsismondi@udgalliance.org>
RUN apt-get update
RUN mkdir /scripts
WORKDIR /scripts
COPY requirements.txt requirements.txt
RUN pip install --timeout 300 -r requirements.txt
COPY . /scripts
CMD ["echo", "please override this CMD with some .sh or .py script call"]
......@@ -6,6 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \
"http://$ORION_HOST:1026/v2/subscriptions/" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
| python -mjson.tool
#!/usr/bin/env bash
[[ -z "$CRATE_HOST" ]] && echo "Please set CRATE_HOST env var. E.g. export CRATE_HOST=127.0.0.1" && exit
echo "Querying historical data at: ${CRATE_HOST}"
curl --location --request POST \
"http://CRATE_HOST:4200/_sql" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /Watering" \
--header "Content-Type: application/json" \
--header "Accept: application/json" \
--data '{
"stmt":"SHOW TABLES"
}'
\ No newline at end of file
#!/usr/bin/env bash
[[ -z "$ORION_HOST" ]] && echo "Please set ORION_HOST env var. E.g. export ORION_HOST=127.0.0.1" && exit
echo "Querying context broker at: ${ORION_HOST}"
# API NOTES:
# "condition": { "attrs": [] } -> means monitor all the attributes in the entity
curl --location --request POST \
"http://$ORION_HOST:1026/v2/subscriptions/" \
--header "Fiware-Service: carouge" \
--header "Content-Type: application/json" \
--header "Accept: application/json" \
--data '{
"description": "Notify QuantumLeap, the historic API, of all WeatherObservedRaw changes",
"subject": {
"entities": [
{
"idPattern": ".*",
"type": "WeatherObservedRaw"
}
],
"condition": {
"attrs": []
}
},
"notification": {
"http": {
"url": "http://172.18.1.7:8668/v2/notify"
},
"attrs": [],
"attrsFormat" : "keyValues",
"metadata": ["dateCreated", "dateModified"]
}
}'
#!/usr/bin/env bash
[[ -z "$ORION_HOST" ]] && echo "Please set ORION_HOST env var. E.g. export ORION_HOST=127.0.0.1" && exit
echo "Querying context broker at: ${ORION_HOST}"
# API NOTES:
# "condition": { "attrs": [] } -> means monitor all the attributes in the entity
curl --location --request POST \
"http://$ORION_HOST:1026/v2/subscriptions/" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /Watering" \
--header "Content-Type: application/json" \
--header "Accept: application/json" \
--data '{
"description": "Notify QuantumLeap, the historic API, of all FlowerBed changes",
"subject": {
"entities": [
{
"idPattern": ".*",
"type": "FlowerBed"
}
],
"condition": {
"attrs": []
}
},
"notification": {
"http": {
"url": "http://172.18.1.7:8668/v2/notify"
},
"attrs": [],
"metadata": ["dateCreated", "dateModified"]
}
}'
#!/usr/bin/env bash
[[ -z "$QUANTUM_HOST" ]] && echo "Please set QUANTUM_HOST env var. E.g. export QUANTUM_HOST=127.0.0.1" && exit
echo "Querying QuantumLeap API at: ${QUANTUM_HOST}"
curl --location --request GET \
"http://$QUANTUM_HOST:8668/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-1/attrs/soilMoistureVwc/value?limit=100" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /Watering" \
--header "Accept: application/json"
#| python -mjson.tool
requests
\ No newline at end of file
......@@ -8,4 +8,4 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
curl -X GET \
"http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering"
--header "Fiware-ServicePath: /Watering"
......@@ -10,6 +10,6 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
curl -X GET \
"http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
--header "X-Auth-Token: $KEYROCK_TOKEN"
......@@ -10,7 +10,7 @@ echo "Querying Fiware entrypoint (PEP_PROXY) at: ${FIWARE_PROXY_HOST}"
curl --location --request PUT \
"http://$FIWARE_PROXY_HOST:1027/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
--header "X-Auth-Token: $KEYROCK_TOKEN" \
--header "Content-Type: text/plain" \
-d $(( RANDOM % 10 ))
......
......@@ -17,7 +17,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}"
curl --location --request POST \
"http://$ORION_HOST:1026/v2/subscriptions/" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
--header "Content-Type: application/json" \
--header "Accept: application/json" \
--header "X-Auth-Token: $KEYROCK_TOKEN" \
......
......@@ -4,47 +4,23 @@
echo "Querying context broker at: ${ORION_HOST}"
curl -iX POST \
"http://$ORION_HOST:1026/v2/entities/" \
"http://$ORION_HOST:1026/v2/entities/?options=keyValues" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
--header "Content-Type: application/json" \
--data '{
"id": "urn:ngsi-ld:FlowerBed:FlowerBed-345",
"id": "urn:ngsi-ld:FlowerBed:FlowerBed-1",
"type": "FlowerBed",
"category": {
"type": "Property",
"value": ["urbanTreeSpot"]
},
"soilMoistureVwc": {
"type": "Property",
"value": 0.85
},
"dateLastWatering": {
"type": "Property",
"value": {
"@type": "DateTime",
"@value": "2020-02-11T08:00:00Z"
}
},
"soilTemperature": {
"type": "Property",
"value": 17
},
"dateModified":"2017-03-31T08:00",
"category": ["urbanTreeSpot"],
"dateLastWatering": "2017-03-31T08:00",
"nextWateringDeadline":"2017-04-31T08:00",
"address": {
"type": "Property",
"value": {
"addressCountry": "Switzerland",
"streetAddress": "Place de Sardaigne",
"adressLocality": "Carouge",
"type": "PostalAddress"
}
"addressCountry": "Switzerland"
},
"location": {
"type": "GeoProperty",
"value": {
"type": "Point",
"coordinates": [46.184255,6.1378375]
}
}
"soilTemperature": 17,
"soilMoistureVwc": 0.85
}
'
'
......@@ -6,5 +6,5 @@ echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \
"http://$ORION_HOST:1026/v2/entities/" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
| python -mjson.tool
......@@ -4,7 +4,7 @@
echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345?options=keyValues" \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-1?options=keyValues" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
| python -mjson.tool
\ No newline at end of file
......@@ -4,13 +4,13 @@
echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value" \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-1/attrs/soilMoistureVwc/value" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
| python -mjson.tool
#curl --location --request GET \
# "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location/value" \
# --header "Fiware-Service: carouge" \
# --header "Fiware-ServicePath: /watering" \
# --header "Fiware-ServicePath: /Watering" \
# | python -mjson.tool
\ No newline at end of file
......@@ -4,13 +4,8 @@
echo "Querying context broker at: ${ORION_HOST}"
curl --location --request GET \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc" \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-1/attrs/soilMoistureVwc" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
| python -mjson.tool
#curl --location --request GET \
# "http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/location" \
# --header "Fiware-Service: carouge" \
# --header "Fiware-ServicePath: /watering" \
# | python -mjson.tool
\ No newline at end of file
......@@ -4,8 +4,8 @@
echo "Querying context broker at: ${ORION_HOST}"
curl --location --request PUT \
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-345/attrs/soilMoistureVwc/value"\
"http://$ORION_HOST:1026/v2/entities/urn:ngsi-ld:FlowerBed:FlowerBed-1/attrs/soilMoistureVwc/value"\
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
--header "Content-Type: text/plain" \
-d $(( RANDOM % 10 ))
......@@ -10,7 +10,7 @@ echo "Callback URL for subscriptions is: ${SUBSCRIPTION_URL}"
curl --location --request POST \
"http://$ORION_HOST:1026/v2/subscriptions/" \
--header "Fiware-Service: carouge" \
--header "Fiware-ServicePath: /watering" \
--header "Fiware-ServicePath: /Watering" \
--header "Content-Type: application/json" \
--header "Accept: application/json" \
--data '{
......@@ -18,7 +18,7 @@ curl --location --request POST \
"subject": {
"entities": [
{
"id": "urn:ngsi-ld:FlowerBed:FlowerBed-345",
"idPattern": ".*",
"type": "FlowerBed"
}
],
......
{"city_info": {"name": "Carouge", "country": "Suisse", "latitude": "46.1838613", "longitude": "6.1385878", "elevation": "388", "sunrise": "05:56", "sunset": "21:06"}, "current_condition": {"date": "18.05.2020", "hour": "14:00", "tmp": 23, "wnd_spd": 17, "wnd_gust": 27, "wnd_dir": "NE", "pressure": 1017.6, "humidity": 45, "condition": "Ensoleill\u00e9", "condition_key": "ensoleille", "icon": "https://www.prevision-meteo.ch/style/images/icon/ensoleille.png", "icon_big": "https://www.prevision-meteo.ch/style/images/icon/ensoleille-big.png"}}
\ No newline at end of file
curl --request DELETE 'localhost:1026/v2/subscriptions/5e54f6dc00043ebcd1ef0850'
#!/usr/bin/env bash
[[ -z "$ORION_HOST" ]] && echo "Please set ORION_HOST env var. E.g. export ORION_HOST=127.0.0.1" && exit
echo "Querying context broker at: ${ORION_HOST}"
curl --location --request DELETE "http://$ORION_HOST:1026/v2/subscriptions/5ec4fbfe91d551c2b4612f42"\
--header "Fiware-Service: carouge"
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
# This is for building docker images for running the scripts
FROM python:3.8-slim-buster
MAINTAINER Federico Sismondi <fsismondi@udgalliance.org>
RUN apt-get update
RUN mkdir /code
WORKDIR /code
COPY requirements.txt requirements.txt
RUN pip install --timeout 300 -r requirements.txt
COPY . /code
CMD ["echo", "please override this CMD with some .sh or .py script call"]
# docker run -it --rm --name weather-data-collection-carouge --env ORION_HOST=10.81.6.109 naiades-client ./weather_02_get_all_raw_data.py
# docker run -it --rm --name weather-use-case-collection-carouge --env ORION_HOST=10.81.6.109 naiades-client ./security_01_get_token_with_password.sh
{"raw_data": "b'eyJjaXR5X2luZm8iOiB7Im5hbWUiOiAiQ2Fyb3VnZSIsICJjb3VudHJ5IjogIlN1aXNzZSIsICJsYXRpdHVkZSI6ICI0Ni4xODM4NjEzIiwgImxvbmdpdHVkZSI6ICI2LjEzODU4NzgiLCAiZWxldmF0aW9uIjogIjM4OCIsICJzdW5yaXNlIjogIjA1OjU1IiwgInN1bnNldCI6ICIyMTowOCJ9LCAiY3VycmVudF9jb25kaXRpb24iOiB7ImRhdGUiOiAiMTkuMDUuMjAyMCIsICJob3VyIjogIjE1OjAwIiwgInRtcCI6IDIzLCAid25kX3NwZCI6IDM2LCAid25kX2d1c3QiOiA1OCwgInduZF9kaXIiOiAiTkUiLCAicHJlc3N1cmUiOiAxMDEyLjksICJodW1pZGl0eSI6IDQ2LCAiY29uZGl0aW9uIjogIkVuc29sZWlsbFx1MDBlOSIsICJjb25kaXRpb25fa2V5IjogImVuc29sZWlsbGUiLCAiaWNvbiI6ICJodHRwczovL3d3dy5wcmV2aXNpb24tbWV0ZW8uY2gvc3R5bGUvaW1hZ2VzL2ljb24vZW5zb2xlaWxsZS5wbmciLCAiaWNvbl9iaWciOiAiaHR0cHM6Ly93d3cucHJldmlzaW9uLW1ldGVvLmNoL3N0eWxlL2ltYWdlcy9pY29uL2Vuc29sZWlsbGUtYmlnLnBuZyJ9fQ=='", "source": "https://www.prevision-meteo.ch/services/json/carouge"}
\ No newline at end of file
requests
\ No newline at end of file
#!/usr/bin/env python3
"""
Dumps weather data to disk each POST_DATA_PERIOD seconds
"""
import os
import json
import time
import logging
from weather_api import *
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.info("Starting to collect..")
COLLECTION_DIR = os.getenv('COLLECTION_DIR', default='.')
def collect():
logger.info("Querying service..")
weather_data = get_data_from_weather_service()
logger.info("Dumping to disk..")
filepath = os.path.join(COLLECTION_DIR,'{}_weather_raw_data.json'.format(int(time.time())))
with open(filepath, encoding='utf-8', mode='w') as f:
json.dump(
obj=weather_data,
fp=f
)
collect()
# Send updates in a loop
while True:
time.sleep(POST_DATA_PERIOD)
collect()
......@@ -27,14 +27,14 @@ dump_keys = ['city_info', 'current_condition']
# see note above about id and type
fiware_data = {
'id': 'urn:ngsi-ld:WeatherRawData:carouge_weather_observed_raw',
'type': 'WeatherRawData'
'id': 'urn:ngsi-ld:WeatherObservedRaw:carouge_weather_observed_raw',
'type': 'WeatherObservedRaw'
}
# build header for POST
http_header_post = {
'Fiware-Service': 'carouge',
'Fiware-ServicePath': '/Carouge/Weather/Raw',
'Fiware-ServicePath': '/Weather/Raw',
'Content-Type': 'application/json',
'Accept': 'application/json',
}
......@@ -43,10 +43,13 @@ http_header_post = {
http_header_get = http_header_post.copy()
http_header_get.pop('Content-Type')
logger.info('Config: \n\tPERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE))
logger.info('Config: \n\tPOST_DATA_PERIOD: {} seconds \n\tURL: {}'.format(POST_DATA_PERIOD, URL_BASE))
def get_data_from_weather_service():
"""
:return: Raw data (json as string)
"""
logger.info('Querying weather service..')
try:
r = requests.get(
......@@ -56,8 +59,11 @@ def get_data_from_weather_service():
except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError):
logger.info('Could not weather reach service')
return None
try:
weather_data = {k: v for k, v in r.json().items() if k in dump_keys}
except json.decoder.JSONDecodeError as e:
logger.error("Got something that doesnt look like a json: {}".format(r.content))
return None
return weather_data
......@@ -72,12 +78,6 @@ def create_weather_entity_in_platform():
try:
entity = {**weather_data, **fiware_data}
with open('weather_raw_last_data.json', encoding='utf-8', mode='w') as f:
json.dump(
obj=entity,
fp=f
)
response = requests.post(
url=URL_BASE + '/v2/entities?options=keyValues',
headers=http_header_post,
......@@ -91,7 +91,11 @@ def create_weather_entity_in_platform():
if response and response.ok:
return True
else:
if response:
logger.error('Operation failed, got message: {}'.format(response.json()))
else:
logger.error('Operation failed, no response received')
return False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment