Commit 563938f9 authored by Hamza Baqa's avatar Hamza Baqa

SemTest Module

parents
{
"variables": [],
"info": {
"name": "F-interop SemTest",
"_postman_id": "b59abcd1-4f97-5c9a-6424-ef2e8c9e8c4a",
"description": "",
"schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json"
},
"item": [
{
"name": "Ping",
"request": {
"url": "{{server}}:{{port}}/SemTest/ping",
"method": "GET",
"header": [],
"body": {},
"description": "Test the service\n"
},
"response": []
},
{
"name": "Upload Ontology",
"event": [
{
"listen": "test",
"script": {
"type": "text/javascript",
"exec": [
"var resp = JSON.parse(responseBody);",
"postman.setEnvironmentVariable(\"file_id\", resp.file_id);",
"postman.setEnvironmentVariable(\"format\", resp.format);"
]
}
}
],
"request": {
"url": "{{server}}:{{port}}/SemTest/ontology",
"method": "POST",
"header": [],
"body": {
"mode": "formdata",
"formdata": [
{
"key": "file",
"description": "",
"type": "file",
"src": "crowdmodelling.owl"
}
]
},
"description": "Upload your semantic discription"
},
"response": []
},
{
"name": "Test",
"request": {
"url": "{{server}}:{{port}}/SemTest/ontology/test",
"method": "POST",
"header": [
{
"key": "Content-Type",
"value": "application/x-www-form-urlencoded",
"description": ""
}
],
"body": {
"mode": "raw",
"raw": "{\n\t\"format\": {{format}},\n\t\"campaign\" : \"123\",\n\t\"ErrType\": [\"Namespace and URI validation\", \"Literal\", \"Predicate and Class validation\"],\n\t\"file_id\": {{file_id}},\n\t\"ref_ontologies\":[\n\t\t{\n \"OntoPath\": \"http://www.w3.org/2002/07/owl#\",\n \"OntoName\": \"owl\"\n },\n {\n \"OntoPath\": \"http://owl.man.ac.uk/2006/07/sssw/people#\",\n \"OntoName\": \"ns0\"\n }\n ]\n}"
},
"description": "After uploading your ontology.\nThis servlet will generate the report of the semantic test "
},
"response": []
},
{
"name": "Generate report",
"request": {
"url": "{{server}}:{{port}}/SemTest/report",
"method": "POST",
"header": [],
"body": {
"mode": "formdata",
"formdata": [
{
"key": "file",
"value": "",
"description": "",
"type": "file"
}
]
},
"description": "This is the only request that the F-interop user need to call to generate the validation report, this servlet will call implicitly:\n\n- Upload ontology Servlet\n- Test Servlet"
},
"response": []
}
]
}
\ No newline at end of file
version = 1.0
DOCKER_GW = 172.17.42.1
info:
@echo $(info_message)
version:
@echo SemTest v$(version)
show_env: ## set env vars
@echo "AMQP_URL " $(AMQP_URL) " AMQP_EXCHANGE " $(AMQP_EXCHANGE)
env: ## set env vars
@echo "Creating var env "
export AMQP_URL=$(AMQP_URL)
@echo $(AMQP_URL)
export AMQP_EXCHANGE=$(AMQP_EXCHANGE)
help: ## Help dialog.
@IFS=$$'\n' ; \
help_lines=(`fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//'`); \
for help_line in $${help_lines[@]}; do \
IFS=$$'#' ; \
help_split=($$help_line) ; \
help_command=`echo $${help_split[0]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \
help_info=`echo $${help_split[2]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \
printf "%-30s %s\n" $$help_command $$help_info ; \
done
build-tools: ## builds all testing tool docker images (only testing tool)
@echo $(info_message)
@echo "Starting to build docker images.. "
$(MAKE) _docker-build-semtest
sniff-bus: ## Listen and echo all messages in the event bus
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
ioppytest-cli connect
stop-all:
@echo "Stop the SemTest cointainer"
docker stop easyglobalmarket-semtest
run-semtest-testing-tool: ## Run SemTest testing tool in docker container
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
docker run -p 9093:8080 -d --rm --env AMQP_EXCHANGE=$(AMQP_EXCHANGE) --env AMQP_URL=$(AMQP_URL) --env DOCKER_GW=$(DOCKER_GW) --name easyglobalmarket-semtest easyglobalmarket-semtest
run-tests: ## runs all unittests
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
$(MAKE) _test_utils
run-u2u-test: ## runs all unittests
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
$(MAKE) _u2u_test
run-conf-test: ## runs all unittests
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
$(MAKE) _test_utils_conf
_test_utils_conf:
python3 conf.py
_test_utils:
cd semtest/test ;python3 test.py
_u2u_test:
python3 u2u.py
install-python-dependencies: ## installs all python pip dependencies
@echo 'installing py3 dependencies...'
@python3 -m pip install ioppytest-utils --user
_check-sudo:
@runner=`whoami` ;\
if test $$runner != "root" ;\
then \
echo "(!) You are not root. This command requires 'sudo -E' \n"; \
fi
_docker-build-semtest:
@echo "Starting to build SemTest testing tools.."
# let's build the testing tool image (same for interop and conformance)
docker build --quiet -t easyglobalmarket-semtest$(version) -f semtest/SemTest/Dockerfile .
docker build --quiet -t easyglobalmarket-semtest-launcher$(version) -f semtest/SemTest_launcher/Dockerfile .
# tag all last version images also with a version-less name
docker tag easyglobalmarket-semtest$(version):latest easyglobalmarket-semtest
docker tag easyglobalmarket-semtest-launcher$(version):latest easyglobalmarket-semtest-launcher
info_message = """ \\n\
******************************************************************************************\n\
docker images naming must follow the following conventions: \n\
\n\
resource_type-sub_type-resource_name-version \n\
\n\
resource_type, sub_type and resource_name cannot contain any special character, nor '-' \n\
version format must comply to vx.x \n\
\n\
examples: \n\
\n\
testing_tool-semtest-v$(version) \n\
\n\
******************************************************************************************\n\\n\
"""
version = 1.0
AMQP_URL=amqp://2XU190DG:BM7F9VB8@mq.dev.f-interop.eu:443/1db49a57-fe56-45a5-aafa-d00f86a3744f
AMQP_EXCHANGE=amq.topic
DOCKER_GW = 172.17.42.1
info:
@echo $(info_message)
version:
@echo SemTest v$(version)
env: ## set env vars
@echo "Creating var env "
export AMQP_URL=$(AMQP_URL)
@echo $(AMQP_URL)
export AMQP_EXCHANGE=$(AMQP_EXCHANGE)
help: ## Help dialog.
@IFS=$$'\n' ; \
help_lines=(`fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//'`); \
for help_line in $${help_lines[@]}; do \
IFS=$$'#' ; \
help_split=($$help_line) ; \
help_command=`echo $${help_split[0]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \
help_info=`echo $${help_split[2]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \
printf "%-30s %s\n" $$help_command $$help_info ; \
done
build-tools: ## builds all testing tool docker images (only testing tool)
@echo $(info_message)
@echo "Starting to build docker images.. "
$(MAKE) _docker-build-semtest
sniff-bus: ## Listen and echo all messages in the event bus
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
ioppytest-cli connect
stop-all:
@echo "Stop the SemTest cointainer"
docker stop easyglobalmarket-semtest
run-semtest-testing-tool: ## Run SemTest testing tool in docker container
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
docker run -p 9093:8080 -d --rm --env AMQP_EXCHANGE=$(AMQP_EXCHANGE) --env AMQP_URL=$(AMQP_URL) --name easyglobalmarket-semtest easyglobalmarket-semtest
docker run -d --rm --env AMQP_EXCHANGE=$(AMQP_EXCHANGE) --env AMQP_URL=$(AMQP_URL) --env DOCKER_GW=$(DOCKER_GW) --name easyglobalmarket-semtest-launcher easyglobalmarket-semtest-launcher
run-tests: ## runs all unittests
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
$(MAKE) _test_utils
run-con-test: ## runs all unittests
@echo "Using AMQP env vars: {url : $(AMQP_URL), exchange : $(AMQP_EXCHANGE)}"
$(MAKE) _test_utils_conf
_test_utils_conf:
cd semtest/test ;python3 test_conf.py
_test_utils:
cd semtest/test ;python3 test.py
install-python-dependencies: ## installs all python pip dependencies
@echo 'installing py3 dependencies...'
@python3 -m pip install ioppytest-utils --user
_check-sudo:
@runner=`whoami` ;\
if test $$runner != "root" ;\
then \
echo "(!) You are not root. This command requires 'sudo -E' \n"; \
fi
_docker-build-semtest:
@echo "Starting to build SemTest testing tools.."
# let's build the testing tool image (same for interop and conformance)
docker build --quiet -t easyglobalmarket-semtest$(version) -f semtest/SemTest/Dockerfile .
docker build --quiet -t easyglobalmarket-semtest-launcher$(version) -f semtest/SemTest_launcher/Dockerfile .
# tag all last version images also with a version-less name
docker tag easyglobalmarket-semtest$(version):latest easyglobalmarket-semtest
docker tag easyglobalmarket-semtest-launcher$(version):latest easyglobalmarket-semtest-launcher
info_message = """ \\n\
******************************************************************************************\n\
docker images naming must follow the following conventions: \n\
\n\
resource_type-sub_type-resource_name-version \n\
\n\
resource_type, sub_type and resource_name cannot contain any special character, nor '-' \n\
version format must comply to vx.x \n\
\n\
examples: \n\
\n\
testing_tool-semtest-v$(version) \n\
\n\
******************************************************************************************\n\\n\
"""
import os
from utils.ioppytest_cli import ioppytest_cli
import requests
from subprocess import call
import json
print("The env vars are:")
DOCKER_GW = os.environ['DOCKER_GW']
retval = os.getcwd() + "/tmp"
#call(["ioppytest-cli", "gui_request_file_upload",retval,"--text-message", "please upload file now :D"])
ioppytest_cli.gui_request_file_upload(retval+"/conf","please upload file now :D","")
file_to_validate_name = os.listdir(retval+"/conf")[1]
file_to_validate_path = retval + "/conf/" + file_to_validate_name
#Upload a the file
URL_to_send = "http://"+DOCKER_GW+":9093/SemTest/ontology"
files = {'file': open(file_to_validate_path, 'rb')}
r = requests.post(URL_to_send, files=files)
j = json.loads(r.text)
JSON_to_Send = json.dumps(
{
"format": j["format"],
"campaign" : "123",
"ErrType": ["Namespace and URI validation", "Literal", "Predicate and Class validation"],
"file_id": j["file_id"],
"ref_ontologies": j["ref_ontologies"]
}
)
#send the file for validation
URL_to_send_validation = "http://"+os.environ['DOCKER_GW']+":9093/SemTest/ontology/test"
res = requests.post(URL_to_send_validation, data=JSON_to_Send)
os.remove(file_to_validate_path)
#Display result
#call(["ioppytest-cli", "gui_display_message",res.text])
ioppytest_cli.gui_display_message(res.text,"")
FROM tomcat:latest
COPY semtest/SemTest/SemTest.war $CATALINA_HOME/webapps/SemTest.war
FROM tomcat:latest
COPY semtest/SemTest/SemTest.war $CATALINA_HOME/webapps/SemTest.war
ADD semtest/SemTest/test.py /root/test.sh
FROM ubuntu:16.04
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update
RUN apt-get -y -qq install apt-utils
RUN apt-get -y -qq install python3-dev
RUN apt-get -y -qq install build-essential
RUN apt-get -y -qq install python3-setuptools
RUN apt-get -y -qq install python3-pip
RUN apt-get -y -qq install python-pip
RUN apt-get -y -qq install supervisor
RUN apt-get -y -qq install net-tools
# upgrade pip
RUN python -m pip install --upgrade pip
RUN python3 -m pip install --upgrade pip
ADD semtest/SemTest_launcher/test.py /
FROM ubuntu:16.04
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update
RUN apt-get -y -qq install apt-utils
RUN apt-get -y -qq install python3-dev
RUN apt-get -y -qq install build-essential
RUN apt-get -y -qq install python3-setuptools
RUN apt-get -y -qq install python3-pip
RUN apt-get -y -qq install python-pip
RUN apt-get -y -qq install supervisor
RUN apt-get -y -qq install net-tools
# upgrade pip
RUN python -m pip install --upgrade pip
RUN python3 -m pip install --upgrade pip
ADD semtest/SemTest_launcher/test.py /
import os
import requests
from subprocess import call
import json
#os.environ["AMQP_URL"] = "amqp://2XU190DG:BM7F9VB8@mq.dev.f-interop.eu:443/1db49a57-fe56-45a5-aafa-d00f86a3744f"
#os.environ['AMQP_EXCHANGE'] = "amq.topic"
print(os.environ["AMQP_URL"])
print(os.environ['AMQP_EXCHANGE'])
print(os.environ['DOCKER_GW'])
#call(["ioppytest-cli", "connect"])
DOCKER_GW = os.environ['DOCKER_GW']
retval = os.getcwd() + "/tmp"
#print("Current working directory", retval)
#ioppytest-cli gui_request_file_upload tmp --text-message "please upload file now :D"
call(["ioppytest-cli", "gui_request_file_upload",retval,"--text-message", "please upload file now :D"])
file_to_validate_name = os.listdir(retval)[0]
file_to_validate_path = retval + "/" + file_to_validate_name
print("My file to upload path is: ",file_to_validate_path)
#Upload a the file
URL_to_send = "http://"+DOCKER_GW+":9093/SemTest/ontology"
files = {'file': open(file_to_validate_path, 'rb')}
r = requests.post(URL_to_send, files=files)
j = json.loads(r.text)
JSON_to_Send = json.dumps(
{
"format": j["format"],
"campaign" : "123",
"ErrType": ["Namespace and URI validation", "Literal", "Predicate and Class validation"],
"file_id": j["file_id"],
"ref_ontologies": j["ref_ontologies"]
}
)
#send the file for validation
URL_to_send_validation = "http://"+os.environ['DOCKER_GW']+":9093/SemTest/ontology/test"
res = requests.post(URL_to_send_validation, data=JSON_to_Send)
os.remove(file_to_validate_path)
#Display result
call(["ioppytest-cli", "gui_display_message",res.text])
print(os.environ['AMQP_EXCHANGE'])
print(os.environ['AMQP_URL'])
\ No newline at end of file
---
services:
semtest:
environment:
- GW_DOCKER=http://172.17.42.1:8185/
image: easyglobalmarket/semtest
ports:
- "8185:8080"
version: "2"
from utils-master.ioppytest_cli import ioppytest_cli
import pika
import os
retval = os.getcwd() + "/tmp"
print(retval)
req = ioppytest_cli.MsgUiRequestSessionConfiguration()
ui_reply = ioppytest_cli._amqp_request(req,'dummy_component')
ioppytest_cli.gui_request_file_upload(retval,"please upload file now :D", ui_reply.users[0])
ioppytest_cli.gui_request_file_upload(retval,"please upload file now :D", ui_reply.users[2])
#ui_reply.users
#print(ui_reply.users)
#print(req)
This diff is collapsed.
java -jar start.jar
\ No newline at end of file
This diff is collapsed.
from utils.ioppytest_cli import ioppytest_cli
import pika
import requests
import os
import json
retval = os.getcwd() + "/tmp"
print(retval)
req = ioppytest_cli.MsgUiRequestSessionConfiguration()
ui_reply = ioppytest_cli._amqp_request(req,'dummy_component')
ioppytest_cli.gui_request_file_upload(retval+"/user1","please upload file now :D", ui_reply.users[0])
ioppytest_cli.gui_request_file_upload(retval+"/user2","please upload file now :D", ui_reply.users[2])
file_1_to_validate_name = os.listdir(retval+"/user1")[1]
file_2_to_validate_name = os.listdir(retval+"/user2")[1]
file_1_to_validate_path = retval + "/user1/" + file_1_to_validate_name
file_2_to_validate_path = retval + "/user2/" + file_2_to_validate_name
URL_to_send = "http://"+os.environ['DOCKER_GW']+":9093/SemTest/ontology"
#Upload file 1
file_1 = {'file': open(file_1_to_validate_path, 'rb')}
r_1 = requests.post(URL_to_send, files=file_1)
j_1 = json.loads(r_1.text)
#Upload file 2
file_2 = {'file': open(file_2_to_validate_path, 'rb')}
r_2 = requests.post(URL_to_send, files=file_2)
j_2 = json.loads(r_2.text)
def dico_to_ont_list(my_dico):
li = []
for my_ob in my_dico['ref_ontologies']:
li.append(my_ob['OntoPath'])
return li
print("User 1 has these ontologies: ",dico_to_ont_list(j_1))
print("User 2 has these ontologies: ",dico_to_ont_list(j_2))
def lengths(x):
if isinstance(x,list):
yield len(x)
for y in x:
yield from lengths(y)
def common_onto(a, b):
a_set = set(a)
b_set = set(b)
if (a_set & b_set):
to_send = " Interoperable at: {0} %".format((len(a_set & b_set)/len(min(a,b)))*100)
else:
to_send = "No shared vocabulary"
return to_send
ioppytest_cli.gui_display_message(common_onto(dico_to_ont_list(j_1),dico_to_ont_list(j_2)),"")
# Created by .ignore support plugin (hsz.mobi)
### Python template
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# dotenv
.env
# virtualenv
.venv/
venv/
ENV/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
### JetBrains template
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff:
.idea/workspace.xml
.idea/tasks.xml
# Sensitive or high-churn files:
.idea/dataSources/
.idea/dataSources.ids
.idea/dataSources.xml
.idea/dataSources.local.xml
.idea/sqlDataSources.xml
.idea/dynamic.xml
.idea/uiDesigner.xml
# Gradle:
.idea/gradle.xml
.idea/libraries
# Mongo Explorer plugin:
.idea/mongoSettings.xml
## File-based project format:
*.iws
## Plugin-specific files:
# IntelliJ
/out/
# mpeltonen/sbt-idea plugin
.idea_modules/
.idea/
# JIRA plugin
atlassian-ide-plugin.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
properties([[$class: 'GitLabConnectionProperty', gitLabConnection: 'figitlab']])
if(env.JOB_NAME =~ 'utils/'){
node('sudo'){