Commit 754fbd31 authored by Federico Sismondi's avatar Federico Sismondi
Browse files

Runner for all automated interop tests

parent bd4def25
......@@ -960,424 +960,40 @@ if(env.JOB_NAME =~ 'ioppytest-coap-automated-iuts/'){
}
}
stage("RUN mini-plugtest: libcoap_clie VS august_cellars_serv"){
gitlabCommitStatus("START resources for mini-plugtest: libcoap_clie VS august_cellars_serv") {
gitlabCommitStatus("Docker run") {
long startTime = System.currentTimeMillis()
long timeoutInSeconds = 120
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
sudo -E make _run-coap-mini-interop-libcoap-cli-vs-august-cellars-server
'''
}
} catch (err) {
long timePassed = System.currentTimeMillis() - startTime
if (timePassed >= timeoutInSeconds * 1000) {
echo 'Docker container kept on running!'
currentBuild.result = 'SUCCESS'
} else {
currentBuild.result = 'FAILURE'
}
}
}
}
gitlabCommitStatus("EXECUTE mini-plugtest: libcoap_clie VS august_cellars_serv") {
long timeoutInSeconds = 600
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
python3 -m pytest -s -p no:cacheprovider tests/integration_test__full_coap_interop_session.py -v
'''
}
}
catch (e){
sh '''
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
finally {
sh '''
sudo -E make stop-all
sudo -E docker ps
'''
}
}
}
stage("RUN mini-plugtest: libcoap_clie VS californium_serv"){
gitlabCommitStatus("START resources for mini-plugtest: libcoap_clie VS californium_serv") {
stage("RUN all interop tests in the cloud"){
gitlabCommitStatus("RUN all interop tests in the cloud") {
gitlabCommitStatus("Docker run") {
long startTime = System.currentTimeMillis()
long timeoutInSeconds = 120
long timeoutInSeconds = 9000
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
sudo -E make _run-coap-mini-interop-libcoap-cli-vs-californium-server
sudo -E python3 -m automation.automated_interop --all-interops --result-logger
'''
}
} catch (err) {
long timePassed = System.currentTimeMillis() - startTime
if (timePassed >= timeoutInSeconds * 1000) {
echo 'Docker container kept on running!'
currentBuild.result = 'SUCCESS'
} else {
currentBuild.result = 'FAILURE'
}
}
}
}
gitlabCommitStatus("EXECUTE mini-plugtest: libcoap_clie VS californium_serv") {
long timeoutInSeconds = 600
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
catch (e){
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
python3 -m pytest -s -p no:cacheprovider tests/integration_test__full_coap_interop_session.py -v
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
}
catch (e){
sh '''
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
finally {
sh '''
sudo -E make stop-all
sudo -E docker ps
'''
}
}
}
stage("RUN mini-plugtest: aiocoap_clie VS californium_serv"){
gitlabCommitStatus("START resources for mini-plugtest: aiocoap_clie VS californium_serv") {
gitlabCommitStatus("Docker run") {
long startTime = System.currentTimeMillis()
long timeoutInSeconds = 120
finally {
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
sudo -E make _run-coap-mini-interop-aiocoap-cli-vs-californium-server
'''
}
} catch (err) {
long timePassed = System.currentTimeMillis() - startTime
if (timePassed >= timeoutInSeconds * 1000) {
echo 'Docker container kept on running!'
currentBuild.result = 'SUCCESS'
} else {
currentBuild.result = 'FAILURE'
}
}
}
}
gitlabCommitStatus("EXECUTE mini-plugtest: aiocoap_clie VS californium_serv") {
long timeoutInSeconds = 600
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
python3 -m pytest -s -p no:cacheprovider tests/integration_test__full_coap_interop_session.py -v
sudo -E make stop-all
sudo -E docker ps
ls data/results/*
'''
archiveArtifacts artifacts: 'data/results/**/*.json', fingerprint: true
archiveArtifacts artifacts: 'data/results/**/*.pcap', fingerprint: true
}
}
catch (e){
sh '''
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
finally {
sh '''
sudo -E make stop-all
sudo -E docker ps
'''
}
}
}
stage("RUN mini-plugtest: aiocoap_clie VS coaphton_serv"){
gitlabCommitStatus("START resources for mini-plugtest: aiocoap_clie VS coaphton_serv") {
gitlabCommitStatus("Docker run") {
long startTime = System.currentTimeMillis()
long timeoutInSeconds = 120
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
sudo -E make _run-coap-mini-interop-aiocoap-cli-vs-coapthon-server
'''
}
} catch (err) {
long timePassed = System.currentTimeMillis() - startTime
if (timePassed >= timeoutInSeconds * 1000) {
echo 'Docker container kept on running!'
currentBuild.result = 'SUCCESS'
} else {
currentBuild.result = 'FAILURE'
}
}
}
}
gitlabCommitStatus("EXECUTE mini-plugtest: aiocoap_clie VS coapthon_server") {
long timeoutInSeconds = 600
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
python3 -m pytest -s -p no:cacheprovider tests/integration_test__full_coap_interop_session.py -v
'''
}
}
catch (e){
sh '''
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
finally {
sh '''
sudo -E make stop-all
sudo -E docker ps
'''
}
}
}
stage("RUN mini-plugtest: coapthon_clie VS californium_serv"){
gitlabCommitStatus("START resources for mini-plugtest: coapthon_clie VS californium_serv") {
gitlabCommitStatus("Docker run") {
long startTime = System.currentTimeMillis()
long timeoutInSeconds = 120
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
sudo -E make _run-coap-mini-interop-coapthon-cli-vs-californium-server
'''
}
} catch (err) {
long timePassed = System.currentTimeMillis() - startTime
if (timePassed >= timeoutInSeconds * 1000) {
echo 'Docker container kept on running!'
currentBuild.result = 'SUCCESS'
} else {
currentBuild.result = 'FAILURE'
}
}
}
}
gitlabCommitStatus("EXECUTE mini-plugtest: coapthon_clie VS californium_serv") {
long timeoutInSeconds = 600
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
python3 -m pytest -s -p no:cacheprovider tests/integration_test__full_coap_interop_session.py -v
'''
}
}
catch (e){
sh '''
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
finally {
sh '''
sudo -E make stop-all
sudo -E docker ps
'''
}
}
}
stage("RUN mini-plugtest: californium_clie VS californium_serv"){
gitlabCommitStatus("START resources for mini-plugtest: californium_clie VS californium_serv") {
gitlabCommitStatus("Docker run") {
long startTime = System.currentTimeMillis()
long timeoutInSeconds = 120
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
sudo -E make _run-coap-mini-interop-californium-cli-vs-californium-server
'''
}
} catch (err) {
long timePassed = System.currentTimeMillis() - startTime
if (timePassed >= timeoutInSeconds * 1000) {
echo 'Docker container kept on running!'
currentBuild.result = 'SUCCESS'
} else {
currentBuild.result = 'FAILURE'
}
}
}
}
gitlabCommitStatus("EXECUTE mini-plugtest: californium_clie VS californium_serv") {
long timeoutInSeconds = 600
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
python3 -m pytest -s -p no:cacheprovider tests/integration_test__full_coap_interop_session.py -v
'''
}
}
catch (e){
sh '''
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
finally {
sh '''
sudo -E make stop-all
sudo -E docker ps
'''
}
}
}
stage("RUN mini-plugtest: californium_clie VS coapthon_serv"){
gitlabCommitStatus("START resources for mini-plugtest: californium_clie VS coapthon_serv") {
gitlabCommitStatus("Docker run") {
long startTime = System.currentTimeMillis()
long timeoutInSeconds = 120
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
sudo -E make _run-coap-mini-interop-californium-cli-vs-coapthon-server
'''
}
} catch (err) {
long timePassed = System.currentTimeMillis() - startTime
if (timePassed >= timeoutInSeconds * 1000) {
echo 'Docker container kept on running!'
currentBuild.result = 'SUCCESS'
} else {
currentBuild.result = 'FAILURE'
}
}
}
}
gitlabCommitStatus("EXECUTE mini-plugtest: californium_clie VS coapthon_serv") {
long timeoutInSeconds = 600
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
python3 -m pytest -s -p no:cacheprovider tests/integration_test__full_coap_interop_session.py -v
'''
}
}
catch (e){
sh '''
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
finally {
sh '''
sudo -E make stop-all
sudo -E docker ps
'''
}
}
}
stage("RUN mini-plugtest: coapthon_clie VS coapthon_serv"){
gitlabCommitStatus("START resources for mini-plugtest: coapthon_clie VS coapthon_serv") {
gitlabCommitStatus("Docker run") {
long startTime = System.currentTimeMillis()
long timeoutInSeconds = 120
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
sudo -E make _run-coap-mini-interop-coapthon-cli-vs-coapthon-server
'''
}
} catch (err) {
long timePassed = System.currentTimeMillis() - startTime
if (timePassed >= timeoutInSeconds * 1000) {
echo 'Docker container kept on running!'
currentBuild.result = 'SUCCESS'
} else {
currentBuild.result = 'FAILURE'
}
}
}
}
gitlabCommitStatus("EXECUTE mini-plugtest: coapthon_clie VS coapthon_serv") {
long timeoutInSeconds = 600
try {
timeout(time: timeoutInSeconds, unit: 'SECONDS') {
sh '''
echo AMQP params: { url: $AMQP_URL , exchange: $AMQP_EXCHANGE}
python3 -m pytest -s -p no:cacheprovider tests/integration_test__full_coap_interop_session.py -v
'''
}
}
catch (e){
sh '''
echo Something broke while running the interop in the cloud :/
echo docker container logs :
sudo make get-logs
'''
throw e
}
finally {
sh '''
sudo -E make stop-all
sudo -E docker ps
'''
}
}
}
}
......
......@@ -319,9 +319,9 @@ _docker-build-coap-additional-resources:
docker build --quiet -t automated_iut-coap_client-libcoap-v$(version) -f automation/coap_client_libcoap/Dockerfile .
docker tag automated_iut-coap_client-libcoap-v$(version):latest automated_iut-coap_client-libcoap
# automated_iut-coap_server-august_cellars (WIP)
docker build --quiet -t automated_iut-coap_server-august_cellars-v$(version) -f automation/coap_server_august_cellars/Dockerfile .
docker tag automated_iut-coap_server-august_cellars-v$(version):latest automated_iut-coap_server-august_cellars
# # automated_iut-coap_server-august_cellars (WIP)
# docker build --quiet -t automated_iut-coap_server-august_cellars-v$(version) -f automation/coap_server_august_cellars/Dockerfile .
# docker tag automated_iut-coap_server-august_cellars-v$(version):latest automated_iut-coap_server-august_cellars
_docker-build-lwm2m-additional-resources:
@echo "Starting to build lwm2m-additional-resources.. "
......
- name: "aiocoap-cli-vs-californium-server"
target_start: "_setup-coap-mini-interop-aiocoap-cli-vs-californium-server"
target_start: "_run-coap-mini-interop-aiocoap-cli-vs-californium-server"
target_stop: "stop-all"
delim: "aiocoap-cli-vs-californium-server"
- name: "libcoap-cli-vs-californium-server"
target_start: "_setup-coap-mini-interop-libcoap-cli-vs-californium-server"
target_start: "_run-coap-mini-interop-libcoap-cli-vs-californium-server"
target_stop: "stop-all"
delim: "libcoap-cli-vs-californium-server"
- name: "californium-cli-vs-californium-server"
target_start: "_setup-coap-mini-interop-californium-cli-vs-californium-server"
target_start: "_run-coap-mini-interop-californium-cli-vs-californium-server"
target_stop: "stop-all"
delim: "californium-cli-vs-californium-server"
- name: "coapthon-cli-vs-californium-server"
target_start: "_setup-coap-mini-interop-coapthon-cli-vs-californium-server"
target_start: "_run-coap-mini-interop-coapthon-cli-vs-californium-server"
target_stop: "stop-all"
delim: "coapthon-cli-vs-californium-server"
- name: "californium-cli-vs-coapthon-server"
target_start: "_setup-coap-mini-interop-californium-cli-vs-coapthon-server"
target_start: "_run-coap-mini-interop-californium-cli-vs-coapthon-server"
target_stop: "stop-all"
delim: "californium-cli-vs-coapthon-server"
- name: "coapthon-cli-vs-coapthon-server"
target_start: "_setup-coap-mini-interop-coapthon-cli-vs-coapthon-server"
target_start: "_run-coap-mini-interop-coapthon-cli-vs-coapthon-server"
target_stop: "stop-all"
delim: "coapthon-cli-vs-coapthon-server"
# -*- coding: utf-8 -*-
import logging
import os
import threading
import time
import base64
import json
import pika
import base64
import threading
import logging
from collections import OrderedDict
from event_bus_utils import AmqpListener, publish_message
from event_bus_utils.rmq_handler import RabbitMQHandler, JsonFormatter
from ioppytest import get_from_environment, AMQP_URL, AMQP_EXCHANGE, RESULTS_DIR
from event_bus_utils import AmqpListener, publish_message
from messages import MsgTestingToolTerminate, MsgSessionLog, MsgTestCaseReady, MsgTestingToolReady, \
MsgTestingToolConfigured, MsgTestSuiteReport, MsgTestCaseVerdict, MsgStepVerifyExecute, \
MsgTestingToolComponentReady, Message, MsgStepVerifyExecuted, MsgTestSuiteStart, MsgTestCaseStart, MsgTestCaseSkip, \
MsgTestingToolComponentShutdown, MsgSniffingGetCaptureReply
from ioppytest.ui_adaptor.message_rendering import (testsuite_results_to_ascii_table,
testcase_verdict_to_ascii_table,
testsuite_state_to_ascii_table)
from messages import (MsgTestingToolTerminate, MsgSessionLog,
MsgTestCaseReady, MsgTestingToolReady,
MsgTestingToolConfigured, MsgTestSuiteReport,
MsgTestCaseVerdict, MsgStepVerifyExecute,
MsgTestingToolComponentReady, Message,
MsgStepVerifyExecuted, MsgTestSuiteStart,
MsgTestCaseStart, MsgTestCaseSkip,
MsgTestingToolComponentShutdown, MsgSniffingGetCaptureReply,
MsgUiRequestSessionConfiguration, MsgUiSessionConfigurationReply)
logger = logging.getLogger(__name__)
......@@ -20,10 +31,16 @@ INTERACTIVE_SESSION = get_from_environment("INTERACTIVE_SESSION", True)
COAP_CLIENT_HOST = get_from_environment("COAP_CLIENT_HOST", 'bbbb::1')
COAP_SERVER_HOST = get_from_environment("COAP_SERVER_HOST", 'bbbb::2')
COAP_SERVER_PORT = get_from_environment("COAP_SERVER_PORT", '5683')
TESTSUITE_NAME = os.environ.get('TESTNAME', 'noname')
TESTSUITE_REPORT_DELIM = os.environ.get('DELIM', '===TESTRESULT===')
LOG_LEVEL = 30
MAX_LINE_LENGTH = 120
default_configuration = {
"testsuite.testcases": None # None => default config (all test cases)
}
def log_all_received_messages(event_list: list):
logger.info("Events sniffed in bus: %s" % len(event_list))
......@@ -60,53 +77,138 @@ COMPLETE LOG TRACE from log messages in event bus (MsgSessionLog)
logger.debug(traces_of_all_messages_in_event_bus)
class ResultsLogger(AmqpListener):
def __init__(self, amqp_url, amqp_exchange):
class ResultsLogToFile(AmqpListener):
def __init__(self, amqp_url, amqp_exchange, results_dir=RESULTS_DIR):
AmqpListener.__init__(self, amqp_url, amqp_exchange,
callback=self.process_message,
topics=['#'],
use_message_typing=True)
self.results_dir = results_dir
self.messages_list = []
self.messages_by_type_dict = {}
@classmethod
def process_message(cls, message):
print('Event received: %s' % type(message))
def process_message(self, message):
if isinstance(message, MsgTestSuiteReport):
# Save report
json_file = os.path.join(
RESULTS_DIR,
'final_report.json'
)
json_file = os.path.join(self.results_dir, 'final_report.json')
with open(json_file, 'w') as f:
f.write(message.to_json())
print("Saved test suite report")
logger.info("Saved test suite report file %s" % json_file)
elif isinstance(message, MsgSniffingGetCaptureReply):
if message.ok:
file_path = os.path.join(RESULTS_DIR, message.filename)
file_path = os.path.join(self.results_dir, message.filename)
with open(file_path, "wb") as pcap_file:
nb = pcap_file.write(base64.b64decode(message.value))
print("Saved pcap file with %s bytes" % nb)
logger.info("Saved pcap file %s with %s bytes" % (file_path, nb))
else:
print("Got Capture result reply with NOK field")
logger.warning("Got Capture result reply with NOK field")
elif isinstance(message, MsgTestCaseVerdict):
# Save verdict
json_file = os.path.join(
RESULTS_DIR,
message.testcase_id + '_verdict.json'
)
json_file = os.path.join(self.results_dir, message.testcase_id + '_verdict.json')
with open(json_file, 'w') as f:
f.write(message.to_json())
print("Verdict saved: %s" % message.testcase_id)
logger.info("Saved verdict file %s for testcase %s" % (json_file, message.testcase_id))
elif isinstance(message, MsgTestingToolTerminate):
logger.info("Received termination message. Stopping %s" % self.__class__.__name__)
self.stop()
else:
logger.debug("Ignoring msg: %s" % type(message))
class ResultsLogToStdout(AmqpListener):
"""
This listener just listens to certain AMQP messages and logs stuff using pretty formatting
"""
def __init__(self, amqp_url, amqp_exchange, use_special_delimiters_for_report=True):
self.use_special_delimiters_for_report = use_special_delimiters_for_report
topics = [
MsgTestingToolTerminate.routing_key,