Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
f-interop-contributors
ioppytest
Commits
723e027b
Commit
723e027b
authored
Dec 28, 2018
by
Federico Sismondi
Browse files
Small refact on FSM's testcase finish logic
Cleaned up /test_coordinator py files
parents
18d2b0eb
f915e273
Pipeline
#3144
canceled with stage
in 0 seconds
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
ioppytest/test_coordinator/amqp_connector.py
View file @
723e027b
...
...
@@ -18,7 +18,6 @@ from messages import *
# TODO get filter from config of the TEDs
COAP_CLIENT_IUT_MODE
=
'user-assisted'
COAP_SERVER_IUT_MODE
=
'automated'
ANALYSIS_MODE
=
'post_mortem'
# either step_by_step or post_mortem
# if left empty => packet_sniffer chooses the loopback
# TODO send flag to sniffer telling him to look for a tun interface instead!
...
...
ioppytest/test_coordinator/coordinator.py
View file @
723e027b
...
...
@@ -73,7 +73,7 @@ class Coordinator(CoordinatorAmqpInterface):
else
:
self
.
event
=
event
def
generate_testcases_verdict
(
self
,
received_event
):
def
save_current_testcase_report_to_file
(
self
,
received_event
):
verdict_info
=
{}
info1
=
self
.
testsuite
.
get_current_testcase
().
to_dict
(
verbose
=
True
)
info2
=
self
.
testsuite
.
get_testcase_report
()
...
...
@@ -252,10 +252,10 @@ class Coordinator(CoordinatorAmqpInterface):
def
finish_testcase
(
self
):
"""
:return:
"""
if
self
.
testsuite
.
check_testcase_finished
()
is
False
:
msg
=
'
e
xpected testcase to be finished'
msg
=
'
E
xpected testcase to be finished
.
'
logger
.
error
(
msg
)
ls_tc
,
ls_steps
=
self
.
testsuite
.
get_detailed_status
()
...
...
@@ -263,142 +263,140 @@ class Coordinator(CoordinatorAmqpInterface):
logger
.
error
(
'steps: %s'
%
ls_steps
)
raise
CoordinatorError
(
msg
)
# Get 'pointer' to current test case and update status
current_tc
=
self
.
testsuite
.
get_current_testcase
()
current_tc
.
change_state
(
'analyzing'
)
current_tc
.
current_step
=
None
#
g
et TC params
#
G
et TC params
tc_id
=
current_tc
.
id
tc_ref
=
current_tc
.
uri
#
Finish sniffer and get PCAP
#
Stop sniffing packets for this test case
logger
.
debug
(
"Sending sniffer stop request..."
)
self
.
call_service_sniffer_stop
()
time
.
sleep
(
0.5
)
time
.
sleep
(
0.1
)
# Get capture of test case
logger
.
debug
(
"Sending get capture request to sniffer..."
)
sniffer_response
=
self
.
call_service_sniffer_get_capture
(
capture_id
=
tc_id
)
# Save .pcap file locally
try
:
if
sniffer_response
.
ok
:
pcap_file_base64
=
sniffer_response
.
value
filename
=
sniffer_response
.
filename
# TODO break this function in smaller pieces
if
ANALYSIS_MODE
==
'post_mortem'
:
with
open
(
os
.
path
.
join
(
PCAP_DIR
,
filename
),
"wb"
)
as
pcap_file
:
nb
=
pcap_file
.
write
(
base64
.
b64decode
(
pcap_file_base64
))
logger
.
debug
(
"Pcap correctly saved (%d Bytes) at %s"
%
(
nb
,
TMPDIR
))
else
:
error_msg
=
'Error encountered with packet sniffer: %s'
%
repr
(
sniffer_response
)
logger
.
warning
(
error_msg
)
tat_response
=
None
gen_verdict
=
''
gen_description
=
''
report
=
[]
error_msg
=
''
# generate error verdict/report
current_tc
.
generate_testcase_report
(
err_msg
=
error_msg
)
self
.
save_current_testcase_report_to_file
(
None
)
logger
.
debug
(
"Sending get capture request to sniffer..."
)
sniffer_response
=
self
.
call_service_sniffer_get_capture
(
capture_id
=
tc_id
)
# change tc state
current_tc
.
change_state
(
'finished'
)
return
# let's try to save the file and then push it to results repo
try
:
if
sniffer_response
.
ok
:
pcap_file_base64
=
sniffer_response
.
value
filename
=
sniffer_response
.
filename
# save PCAP to file
with
open
(
os
.
path
.
join
(
PCAP_DIR
,
filename
),
"wb"
)
as
pcap_file
:
nb
=
pcap_file
.
write
(
base64
.
b64decode
(
pcap_file_base64
))
logger
.
debug
(
"Pcap correctly saved (%d Bytes) at %s"
%
(
nb
,
TMPDIR
))
logger
.
debug
(
"Sending PCAP file to TAT for analysis..."
)
# Forwards PCAP to TAT to get CHECKs results
try
:
tat_response
=
self
.
call_service_testcase_analysis
(
protocol
=
self
.
testsuite_name
,
testcase_id
=
tc_id
,
testcase_ref
=
tc_ref
,
file_enc
=
"pcap_base64"
,
filename
=
tc_id
+
".pcap"
,
value
=
pcap_file_base64
)
except
AmqpSynchCallTimeoutError
as
e
:
error_msg
+=
"TAT didnt answer to the analysis request"
logger
.
error
(
error_msg
)
if
tat_response
and
tat_response
.
ok
:
logger
.
info
(
"Response received from TAT: %s "
%
repr
(
tat_response
))
# Save the json object received
json_file
=
os
.
path
.
join
(
TMPDIR
,
'%s_analysis.json'
%
tc_id
)
with
open
(
json_file
,
'w'
)
as
f
:
f
.
write
(
tat_response
.
to_json
())
# let's process the partial verdicts from TAT's answer
# format : [[partial verdict : str, description : str]]
partial_verd
=
[]
step_count
=
0
ls_len
=
len
(
tat_response
.
partial_verdicts
)
for
item
in
tat_response
.
partial_verdicts
:
# let's partial verdict id
step_count
+=
1
p
=
(
"frame_check_[{}/{}]"
.
format
(
step_count
,
ls_len
),
item
[
0
],
item
[
1
])
partial_verd
.
append
(
p
)
logger
.
debug
(
"Processing partial verdict received from TAT: %s"
%
str
(
p
))
# generates a general verdict considering other steps partial verdicts besides TAT's
gen_verdict
,
gen_description
,
report
=
current_tc
.
generate_testcases_verdict
(
partial_verd
)
else
:
error_msg
+=
'Error message: %s (err.code: %s)'
%
(
tat_response
.
error_message
,
tat_response
.
error_code
)
logger
.
warning
(
error_msg
)
# generate verdict and verdict description
try
:
gen_description
=
error_msg
gen_verdict
=
'inconclusive'
except
AttributeError
:
gen_description
=
error_msg
gen_verdict
=
'error'
report
=
[]
else
:
error_msg
+=
'Error encountered with packet sniffer: %s'
%
repr
(
sniffer_response
)
logger
.
warning
(
error_msg
)
gen_verdict
=
'error'
gen_description
=
error_msg
report
=
[]
except
AttributeError
as
ae
:
error_msg
+=
'Failed to process Sniffer response. Wrongly formated response? : %s'
%
repr
(
sniffer_response
)
logger
.
error
(
error_msg
)
gen_verdict
=
'error'
gen_description
=
error_msg
report
=
[]
# TODO this should be hanlded directly by generate_testcases_verdict method
# save sent message in RESULTS dir
final_report
=
OrderedDict
()
final_report
[
'verdict'
]
=
gen_verdict
final_report
[
'description'
]
=
gen_description
final_report
[
'partial_verdicts'
]
=
report
# TODO this should be hanlded directly by generate_testcases_verdict method
# lets generate test case report
current_tc
.
report
=
final_report
# Save the final verdict as json
json_file
=
os
.
path
.
join
(
TMPDIR
,
tc_id
+
'_verdict.json'
)
with
open
(
json_file
,
'w'
)
as
f
:
json
.
dump
(
final_report
,
f
)
except
AttributeError
as
ae
:
error_msg
=
'Failed to process packet sniffer response. '
\
'Wrongly formatted response? : %s'
%
repr
(
sniffer_response
)
logger
.
error
(
error_msg
)
# generate error verdict/report
current_tc
.
generate_testcase_report
(
err_msg
=
error_msg
)
self
.
save_current_testcase_report_to_file
(
None
)
# change tc state
current_tc
.
change_state
(
'finished'
)
logger
.
info
(
"General verdict generated: %s"
%
json
.
dumps
(
current_tc
.
report
))
return
else
:
# TODO implement step-by-step analysis
raise
NotImplementedError
()
# Forward .pcap to TAT to get CHECKs results
try
:
tat_response
=
self
.
call_service_testcase_analysis
(
protocol
=
self
.
testsuite_name
,
testcase_id
=
tc_id
,
testcase_ref
=
tc_ref
,
file_enc
=
"pcap_base64"
,
filename
=
tc_id
+
".pcap"
,
value
=
pcap_file_base64
)
except
AmqpSynchCallTimeoutError
as
e
:
error_msg
=
"TAT service timeout error"
logger
.
error
(
error_msg
)
# generate error verdict/report
current_tc
.
generate_testcase_report
(
err_msg
=
error_msg
)
self
.
save_current_testcase_report_to_file
(
None
)
# change tc state
current_tc
.
change_state
(
'finished'
)
return
# Process TAT response and create final test case report
try
:
if
tat_response
.
ok
:
logger
.
info
(
"Response received from TAT: %s "
%
repr
(
tat_response
))
# Save the json object received
json_file
=
os
.
path
.
join
(
TMPDIR
,
'%s_analysis.json'
%
tc_id
)
with
open
(
json_file
,
'w'
)
as
f
:
f
.
write
(
tat_response
.
to_json
())
# let's process the partial verdicts from TAT's answer
# format : [[partial verdict : str, description : str]]
partial_verd
=
[]
step_count
=
0
ls_len
=
len
(
tat_response
.
partial_verdicts
)
for
item
in
tat_response
.
partial_verdicts
:
# let's partial verdict id
step_count
+=
1
p
=
(
"frame_check_[{}/{}]"
.
format
(
step_count
,
ls_len
),
item
[
0
],
item
[
1
])
partial_verd
.
append
(
p
)
logger
.
debug
(
"Processing partial verdict received from TAT: %s"
%
str
(
p
))
else
:
error_msg
=
'Error response from TAT : %s (err.code: %s)'
%
(
tat_response
.
error_message
,
tat_response
.
error_code
)
logger
.
warning
(
error_msg
)
# generate error verdict/report
current_tc
.
generate_testcase_report
(
err_msg
=
error_msg
)
self
.
save_current_testcase_report_to_file
(
None
)
# change tc state
current_tc
.
change_state
(
'finished'
)
return
except
AttributeError
:
error_msg
=
"TAT didn't send a correctly formatted message, we got %s"
%
repr
(
tat_response
)
logger
.
error
(
error_msg
)
# generate error verdict/report
current_tc
.
generate_testcase_report
(
err_msg
=
error_msg
)
self
.
save_current_testcase_report_to_file
(
None
)
# change tc state
current_tc
.
change_state
(
'finished'
)
return
# Create final test case report from TAT partial verdicts
# Note: report is generated from TAT partial verdicts (CHECKS), as well as IUTs' VERIFY steps
current_tc
.
generate_testcase_report
(
partial_verd
)
logger
.
info
(
"General verdict generated: %s"
%
json
.
dumps
(
current_tc
.
report
))
self
.
save_current_testcase_report_to_file
(
None
)
return
current_tc
.
report
# change tc state
current_tc
.
change_state
(
'finished'
)
self
.
save_current_testcase_report_to_file
(
None
)
def
handle_testcase_restart
(
self
,
received_event
):
"""
...
...
ioppytest/test_coordinator/states_and_transitions.py
View file @
723e027b
...
...
@@ -59,7 +59,7 @@ states = [
'name'
:
'testcase_finished'
,
'on_enter'
:
[
'notify_testcase_finished'
,
'
generate_testcases_verdict
'
,
'
save_current_testcase_report_to_file
'
,
'notify_testcase_verdict'
,
'to_preparing_next_testcase'
],
# jumps to following state, this makes testcase_finished a transition state
'on_exit'
:
[]
...
...
ioppytest/test_suite/testsuite.py
View file @
723e027b
...
...
@@ -118,8 +118,6 @@ COMPONENT_ID = 'testsuite'
logger
=
logging
.
getLogger
(
COMPONENT_ID
)
logger
.
setLevel
(
LOG_LEVEL
)
ANALYSIS_MODE
=
'post_mortem'
# either step_by_step or post_mortem
# AMQP log handler with f-interop's json formatter
rabbitmq_handler
=
RabbitMQHandler
(
AMQP_URL
,
COMPONENT_ID
)
json_formatter
=
JsonFormatter
()
...
...
@@ -320,13 +318,7 @@ class TestSuite:
report_item
=
{
'testcase_id'
:
tc
.
id
}
if
tc
.
report
is
None
:
logger
.
warning
(
"Empty report found. Generating dummy report for skipped testcase : %s"
%
tc
.
id
)
# TODO this should be hanlded directly by generate_testcases_verdict method
gen_verdict
,
gen_description
,
rep
=
tc
.
generate_testcases_verdict
(
None
)
final_report
=
OrderedDict
()
final_report
[
'verdict'
]
=
gen_verdict
final_report
[
'description'
]
=
gen_description
final_report
[
'partial_verdicts'
]
=
rep
tc
.
report
=
final_report
tc
.
generate_testcase_report
(
err_msg
=
"No test case verdict/report found for test case"
)
report_item
.
update
(
tc
.
report
)
report
.
append
(
report_item
)
...
...
@@ -662,13 +654,10 @@ class TestSuite:
if
testcase_id
:
assert
type
(
testcase_id
)
is
str
# assigns the one which is not None:
tc
=
self
.
get_testcase
(
testcase_id
)
or
self
.
get_current_testcase
()
if
tc
:
return
tc
.
report
else
:
return
None
return
tc
.
report
if
tc
else
None
def
reinit_testcase
(
self
,
testcase_id
):
"""
...
...
@@ -883,11 +872,16 @@ class TestConfig:
class
Step
():
def
__init__
(
self
,
step_id
,
type
,
description
,
node
=
None
):
self
.
id
=
step_id
assert
type
in
(
"stimuli"
,
"check"
,
"verify"
,
"feature"
)
self
.
id
=
step_id
self
.
type
=
type
self
.
state
=
None
self
.
description
=
description
# stimuli steps dont have a verdict, they are IUT actions
if
type
!=
'stimuli'
:
self
.
partial_verdict
=
Verdict
()
# stimuli and verify step MUST have a iut field in the YAML file
if
type
==
'stimuli'
or
type
==
'verify'
:
assert
node
is
not
None
...
...
@@ -895,11 +889,6 @@ class Step():
else
:
self
.
iut
=
None
# stimulis step dont get partial verdict
self
.
partial_verdict
=
Verdict
()
self
.
state
=
None
def
__repr__
(
self
):
node
=
''
mode
=
''
...
...
@@ -911,16 +900,16 @@ class Step():
def
reinit
(
self
):
if
self
.
type
in
(
'check'
,
'verify'
,
'feature'
):
logger
.
debug
(
'Step (re)initing for: step_id: %s, step_type: %s'
%
(
self
.
id
,
self
.
type
))
if
self
.
type
in
(
'check'
,
'feature'
):
self
.
partial_verdict
=
Verdict
()
self
.
change_state
(
'postponed'
)
elif
self
.
type
==
'verify'
:
self
.
partial_verdict
=
Verdict
()
self
.
change_state
(
None
)
# when using post_mortem analysis mode all checks are postponed , and analysis is done at the end of the TC
logger
.
debug
(
'Processing step init, step_id: %s, step_type: %s, ANALYSIS_MODE is %s'
%
(
self
.
id
,
self
.
type
,
ANALYSIS_MODE
))
if
self
.
type
==
'check'
or
self
.
type
==
'feature'
and
ANALYSIS_MODE
==
'post_mortem'
:
self
.
change_state
(
'postponed'
)
else
:
self
.
change_state
(
None
)
else
:
# its a stimuli
self
.
change_state
(
None
)
...
...
@@ -931,7 +920,7 @@ class Step():
step_dict
[
'step_type'
]
=
self
.
type
step_dict
[
'step_info'
]
=
self
.
description
step_dict
[
'step_state'
]
=
self
.
state
#
it the step is a stimuli then lets add the IUT info(note that checks dont have that info)
#
let's return IUT info for steps associated to an IUT
if
self
.
type
==
'stimuli'
or
self
.
type
==
'verify'
:
step_dict
.
update
(
self
.
iut
.
to_dict
())
return
step_dict
...
...
@@ -1078,21 +1067,29 @@ class TestCase:
logger
.
debug
(
"[TESTCASE] - all steps in TC are either finished (or pending)."
)
return
True
def
generate_testcase
s_verdic
t
(
self
,
tat_
post_mortem_
analysis_report
=
None
):
def
generate_testcase
_repor
t
(
self
,
tat_analysis_report
=
None
,
err_msg
=
None
):
"""
Generates the final verdict of TC and report taking into account the CHECKs and VERIFYs of the testcase
:return: tuple: (final_verdict, verdict_description, tc_report) ,
where final_verdict in ("None", "error", "inconclusive", "pass" , "fail")
where description is String type
where tc report is a list :
[(step, step_partial_verdict, step_verdict_info, associated_frame_id (can be null))]
Generates the final verdict (report) of TC taking into consideration the CHECKs and VERIFYs of the testcase.
Updates report attribute from tc object
:return:
tc report format: [(step, step_partial_verdict, step_verdict_info, associated_frame_id (can be null))]
"""
if
err_msg
:
self
.
report
=
{
'verdict'
:
'error'
,
'description'
:
err_msg
,
'partial_verdicts'
:
[],
}
return
self
.
report
logger
.
debug
(
"[VERDICT GENERATION] starting the verdict generation"
)
if
self
.
state
is
None
or
self
.
state
==
'skipped'
or
self
.
state
==
'aborted'
:
return
(
'None'
,
'Testcase %s was %s.'
%
(
self
.
id
,
self
.
state
),
[])
# TODO hanlde frame id associated to the step , used for GUI purposes
if
self
.
check_all_steps_finished
()
is
False
:
logger
.
warning
(
"Found non finished steps: %s"
%
json
.
dumps
(
self
.
seq_to_dict
(
verbose
=
False
)))
return
...
...
@@ -1104,24 +1101,26 @@ class TestCase:
# for the verdict we use the info in the checks and verify steps
if
step
.
type
in
(
"check"
,
"verify"
,
"feature"
):
logger
.
debug
(
"[VERDICT GENERATION] Processing step %s"
%
step
.
id
)
if
step
.
state
==
"postponed"
:
tc_report
.
append
((
step
.
id
,
None
,
"%s step: postponed"
%
step
.
type
.
upper
(),
""
))
#tc_report.append((step.id, None, "%s step: postponed" % step.type.upper(), ""))
pass
elif
step
.
state
==
"finished"
:
tc_report
.
append
(
(
step
.
id
,
step
.
partial_verdict
.
get_value
(),
step
.
partial_verdict
.
get_message
(),
""
))
(
step
.
id
,
step
.
partial_verdict
.
get_value
(),
step
.
partial_verdict
.
get_message
(),
""
)
)
# update global verdict
final_verdict
.
update
(
step
.
partial_verdict
.
get_value
(),
step
.
partial_verdict
.
get_message
())
else
:
msg
=
"step %s not ready for analysis"
%
(
step
.
id
)
logger
.
error
(
"[VERDICT GENERATION] "
+
msg
)
raise
TestSuiteError
(
msg
)
# append at the end of the report the analysis done a posteriori (if any)
if
tat_
post_mortem_
analysis_report
and
len
(
tat_
post_mortem_
analysis_report
)
!=
0
:
logger
.
info
(
'Processing TAT partial verdict: '
+
str
(
tat_
post_mortem_
analysis_report
))
for
item
in
tat_
post_mortem_
analysis_report
:
if
tat_analysis_report
and
len
(
tat_analysis_report
)
!=
0
:
logger
.
info
(
'Processing TAT partial verdict: '
+
str
(
tat_analysis_report
))
for
item
in
tat_analysis_report
:
# TODO process the items correctly
tc_report
.
append
(
item
)
final_verdict
.
update
(
item
[
1
],
item
[
2
])
...
...
@@ -1133,12 +1132,16 @@ class TestCase:
# hack to overwrite the final verdict MESSAGE in case of pass
if
final_verdict
.
get_value
()
==
'pass'
:
final_verdict
.
update
(
'pass'
,
'No interoperability error was detected.'
)
logger
.
debug
(
"[VERDICT GENERATION] Test case executed correctly, a PASS was issued."
)
else
:
logger
.
debug
(
"[VERDICT GENERATION] Test case executed correctly, but FAIL was issued as verdict."
)
logger
.
debug
(
"[VERDICT GENERATION] info: %s' "
%
final_verdict
.
get_value
())
return
final_verdict
.
get_value
(),
final_verdict
.
get_message
(),
tc_report
self
.
report
=
{
'verdict'
:
final_verdict
.
get_value
(),
'description'
:
final_verdict
.
get_message
(),
'partial_verdicts'
:
tc_report
,
}
return
self
.
report
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
...
...
@@ -1170,7 +1173,7 @@ def testcase_constructor(loader, node):
instance
=
TestCase
.
__new__
(
TestCase
)
yield
instance
state
=
loader
.
construct_mapping
(
node
,
deep
=
True
)
logging
.
debug
(
"pasing test case: "
+
str
(
state
))
#
logging.debug("pasing test case: " + str(state))
instance
.
__init__
(
**
state
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment