Commit db5809e5 authored by Federico Sismondi's avatar Federico Sismondi
Browse files

midifications and fixes for using the agent with the new APIv.1.0

parent c5726116
...@@ -34,8 +34,6 @@ import multiprocessing ...@@ -34,8 +34,6 @@ import multiprocessing
from connectors.tun import TunConnector from connectors.tun import TunConnector
from connectors.core import CoreConnector from connectors.core import CoreConnector
from connectors.http import HTTPConnector from connectors.http import HTTPConnector
from connectors.ping import PingConnector
from connectors.zeromq import ZMQConnector
from connectors.serialconn import SerialConnector from connectors.serialconn import SerialConnector
from utils import arrow_down, arrow_up, finterop_banner from utils import arrow_down, arrow_up, finterop_banner
...@@ -46,7 +44,7 @@ try: ...@@ -46,7 +44,7 @@ try:
except ImportError: except ImportError:
from urlparse import urlparse from urlparse import urlparse
__version__ = (0, 0, 1) __version__ = (0, 1, 0)
DEFAULT_PLATFORM = 'f-interop.paris.inria.fr' DEFAULT_PLATFORM = 'f-interop.paris.inria.fr'
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
...@@ -163,8 +161,6 @@ For more information, visit: http://doc.f-interop.eu ...@@ -163,8 +161,6 @@ For more information, visit: http://doc.f-interop.eu
self.plugins["serial"] = SerialConnector(**data) self.plugins["serial"] = SerialConnector(**data)
else: else:
self.plugins["tun"] = TunConnector(**data) self.plugins["tun"] = TunConnector(**data)
# self.plugins["zmq"] = ZMQConnector(**data)
# self.plugins["ping"] = PingConnector(**data)
# self.plugins["http"] = HTTPConnector(**data) # self.plugins["http"] = HTTPConnector(**data)
for p in self.plugins.values(): for p in self.plugins.values():
......
...@@ -5,18 +5,15 @@ import json ...@@ -5,18 +5,15 @@ import json
import logging import logging
from multiprocessing import Process from multiprocessing import Process
from kombu import Connection from utils.messages import Message as EventBusMessage, MsgTestingToolComponentReady as EventBusMessageComponentReady
from kombu import Exchange from kombu import Connection, Queue, Exchange, Consumer
from kombu import Queue
from kombu.mixins import ConsumerMixin from kombu.mixins import ConsumerMixin
DEFAULT_EXCHANGE_NAME = "amq.topic"
class BaseConsumer(ConsumerMixin): class BaseConsumer(ConsumerMixin):
DEFAULT_EXCHANGE_NAME = "amq.topic" DEFAULT_EXCHANGE_NAME = "amq.topic"
def __init__(self, user, password, session, server, exchange, name, consumer_name): def __init__(self, user, password, session, server, exchange, name, consumer_name, topics):
""" """
Args: Args:
...@@ -27,6 +24,7 @@ class BaseConsumer(ConsumerMixin): ...@@ -27,6 +24,7 @@ class BaseConsumer(ConsumerMixin):
exchange: RMQ exchange for sending messages exchange: RMQ exchange for sending messages
name: Identity of the agent. Used by testing tools to identify/differentiate each agent on the session name: Identity of the agent. Used by testing tools to identify/differentiate each agent on the session
consumer_name: Name to easily identify a process consuming. consumer_name: Name to easily identify a process consuming.
topics: Topics subscriptions for the consumer
""" """
self.log = logging.getLogger(__name__) self.log = logging.getLogger(__name__)
self.log.setLevel(logging.DEBUG) self.log.setLevel(logging.DEBUG)
...@@ -53,109 +51,54 @@ class BaseConsumer(ConsumerMixin): ...@@ -53,109 +51,54 @@ class BaseConsumer(ConsumerMixin):
type="topic", type="topic",
durable=True) durable=True)
self.control_queue = Queue("control.{consumer_name}@{name}".format(name=name, # queues created for topic subscriptions
consumer_name=consumer_name), self.queues = []
exchange=self.exchange,
routing_key='control.{consumer_name}.toAgent.{name}'.format( # handle subscriptions
consumer_name=consumer_name, self.subscribe_to_topics(topics)
name=name),
durable=False, def subscribe_to_topics(self, topic_list):
auto_delete=True) for t in topic_list:
queue = Queue(
self.data_queue = Queue("data.{consumer_name}@{name}".format(name=name, name="{name}.{consumer_name}::{rkey}".format(
consumer_name=consumer_name), name=self.name,
exchange=self.exchange, consumer_name=self.consumer_name,
routing_key='data.{consumer_name}.toAgent.{name}'.format(consumer_name=consumer_name, rkey=t
name=name), ),
durable=False, exchange=self.exchange,
auto_delete=True) routing_key=t,
durable=False,
auto_delete=True
)
self.queues.append(
queue
)
def get_consumers(self, Consumer, channel): def get_consumers(self, Consumer, channel):
return [ return [
Consumer(queues=[self.control_queue], Consumer(self.queues, callbacks=[self.on_message], accept=['json']),
callbacks=[self.handle_control],
no_ack=True,
accept=['json']),
Consumer(queues=[self.data_queue],
callbacks=[self.handle_data],
no_ack=True,
accept=["json"])
] ]
def on_consume_ready(self, connection, channel, consumers, wakeup=True, **kwargs): def on_message(self, body, message):
# control plane info assert type(body) is dict # assert that kombu deserialized it already
self.log.info( json_body = json.dumps(body)
"{consumer_name} listening to control plane ".format(consumer_name=self.consumer_name))
self.log.info(
"Queue: control.{consumer_name}@{name} ".format(consumer_name=self.consumer_name, name=self.name))
self.log.info(
"Topic: control.{consumer_name}.toAgent.{name}".format(consumer_name=self.consumer_name, name=self.name))
# data plane info
self.log.info(
"{consumer_name} listening to data plane".format(consumer_name=self.consumer_name))
self.log.info(
"Queue: data.{consumer_name}@{name}".format(consumer_name=self.consumer_name, name=self.name))
self.log.info(
"Topic: data.{consumer_name}.toAgent.{name}".format(consumer_name=self.consumer_name, name=self.name))
def handle_control(self, body, message):
self.log.debug("DEFAULT HANDLE CONTROL")
self.log.debug(("Payload", message.payload))
self.log.debug(("Properties", message.properties))
self.log.debug(("Headers", message.headers))
self.log.debug(("body", message.body))
msg = None
try:
msg = json.loads(body)
self.log.debug(message)
except ValueError as e:
message.ack()
self.log.error(e)
self.log.error("Incorrect message: {0}".format(body))
if msg is not None:
self.log.debug("Just received that packet")
self.log.debug(msg)
def handle_data(self, body, message):
"""
Args:
msg:
Returns:
""" self.log.debug("DEFAULT on_message callback, got: {}".format(message.delivery_info.get('routing_key')))
self.log.debug("DEFAULT HANDLE DATA") msg = EventBusMessage.load(json_body, message.delivery_info.get('routing_key'))
self.log.debug(("Payload", message.payload)) self._on_message(msg)
self.log.debug(("Properties", message.properties))
self.log.debug(("Headers", message.headers))
self.log.debug(("body", message.body))
# msg = None
# try:
# msg = json.loads(body)
# self.log.debug(message)
# except ValueError as e:
# message.ack()
# self.log.error(e)
# self.log.error("Incorrect message: {0}".format(body))
#
# if msg is not None:
# self.log.debug("Just received that packet")
# self.log.debug(msg)
def test_connection(self):
"""
Test if a component can talk on the event bus.
Returns: def _on_message(self, message):
"Class to be overridden by children calss"
return NotImplementedError()
""" def on_consume_ready(self, connection, channel, consumers, wakeup=True, **kwargs):
# for key in [control key, data key]: # control plane info
# log.info("Testing on local routing key: %s" % key) for q in self.queues:
# self.basic_publish(key, "PING!!") self.log.info(
pass "Listening on {queue_name} bound to {rkey} ".format(queue_name=q.name,
rkey=q.routing_key)
)
class BaseController(Process): class BaseController(Process):
......
...@@ -3,13 +3,12 @@ Plugin to connect to the F-interop backend ...@@ -3,13 +3,12 @@ Plugin to connect to the F-interop backend
""" """
import json import json
import logging import logging
from utils.messages import *
from kombu import Producer from kombu import Producer
from .base import BaseController, BaseConsumer from .base import BaseController, BaseConsumer
__version__ = (0, 1, 0)
__version__ = (0, 0, 1)
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
...@@ -20,59 +19,33 @@ class CoreConsumer(BaseConsumer): ...@@ -20,59 +19,33 @@ class CoreConsumer(BaseConsumer):
""" """
def __init__(self, user, password, session, server, exchange, name, consumer_name): def __init__(self, user, password, session, server, exchange, name, consumer_name):
super(CoreConsumer, self).__init__(user, password, session, server, exchange, name, consumer_name) subscriptions = [MsgTestingToolComponentReady.routing_key]
super(CoreConsumer, self).__init__(user, password, session, server, exchange, name, consumer_name,
def get_consumers(self, Consumer, channel): subscriptions)
return [
Consumer(queues=[self.control_queue],
callbacks=[self.handle_control],
no_ack=True,
accept=['json']),
Consumer(queues=[self.data_queue],
callbacks=[self.handle_data],
no_ack=True,
accept=["json"])
]
def on_consume_ready(self, connection, channel, consumers, wakeup=True, **kwargs): def on_consume_ready(self, connection, channel, consumers, wakeup=True, **kwargs):
log.info("Backend ready to consume data") log.info("Backend ready to consume data")
# log.info("-------------------------------------------------")
# log.info("Go to this URL: http://{platform}/session/{session}".format(platform=self.server_url,
# session=self.session))
# log.info("-------------------------------------------------")
# let's send bootstrap message # let's send bootstrap message
msg = { msg = MsgTestingToolComponentReady(
'_type': 'testingtool.component.ready', component='agent.{}'.format(self.name),
'component': self.name, description="Component READY to start test suite."
"description": "Component READY to start test suite." )
}
producer = Producer(connection, serializer='json')
producer = Producer(connection,serializer='json') producer.publish(
producer.publish(msg, body=msg.to_dict(),
exchange=self.exchange, exchange=self.exchange,
routing_key='control.session' routing_key=msg.routing_key
) )
def handle_control(self, body, message): def _on_message(self, message):
log.debug("DEFAULT HANDLE CONTROL") self.log.debug(
log.debug(("Payload", message.payload)) "Consumer specialized handler <{consumer_name}> got: {message}".format(
log.debug(("Properties", message.properties)) consumer_name=self.consumer_name,
log.debug(("Headers", message.headers)) message=repr(message)
log.debug(("body", message.body)) )
msg = None )
try:
msg = json.loads(body)
log.debug(message)
except ValueError as e:
message.ack()
log.error(e)
log.error("Incorrect message: {0}".format(body))
if msg is not None:
log.debug("Just received that packet")
log.debug(msg)
class CoreConnector(BaseController): class CoreConnector(BaseController):
......
import json import json
__version__ = (0, 0, 1) __version__ = (0, 1, 0)
import logging import logging
from requests import Request, Session from requests import Request, Session
......
"""
Create ICMP packets to test if all nodes are up
"""
import json
__version__ = (0, 0, 1)
import logging
from .base import BaseController, BaseConsumer
import subprocess
class PingConsumer(BaseConsumer):
"""
AMQP helper
"""
def __init__(self, user, password, session, server, exchange, name, consumer_name):
super(PingConsumer, self).__init__(user, password, session, server, exchange, name, consumer_name)
def handle_control(self, body, message):
"""
Args:
body:
message:
Returns:
"""
required_keys = {"host"}
msg = None
try:
msg = json.loads(body)
self.log.debug(message)
except ValueError as e:
message.ack()
self.log.error(e)
self.log.error("Incorrect message: {0}".format(body))
if msg is not None:
present_keys = set(msg.keys())
if not present_keys.issuperset(required_keys):
self.log("Error you need those keys: %s" % required_keys)
else:
# Default version is ping
version = {"IPv6": "ping6", "IPv4": "ping"}
executable = version.get(msg["version"], "ping")
# default packet count is 1
count_option = msg.get("count", 1)
# network interface, default is None
interface_option = msg.get("interface", "")
# run the ping
command = [executable, msg["host"], "-c", str(count_option)]
self.log.info("command launched: %s" % command)
p = subprocess.call(command)
self.log.info("result: %s" % p)
class PingConnector(BaseController):
NAME = "ping"
def __init__(self, **kwargs):
super(PingConnector, self).__init__(name=PingConnector.NAME)
kwargs["consumer_name"] = PingConnector.NAME
self.consumer = PingConsumer(**kwargs)
self.consumer.log = logging.getLogger(__name__)
self.consumer.log.setLevel(logging.DEBUG)
def run(self):
self.consumer.run()
...@@ -9,19 +9,32 @@ import threading ...@@ -9,19 +9,32 @@ import threading
from connectors.base import BaseController, BaseConsumer from connectors.base import BaseController, BaseConsumer
from utils.serial_listener import SerialListener from utils.serial_listener import SerialListener
from utils import arrow_down, arrow_up, finterop_banner from utils import arrow_down, arrow_up, finterop_banner
from utils.messages import *
__version__ = (0, 0, 1) __version__ = (0, 1, 0)
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class SerialConsumer(BaseConsumer): class SerialConsumer(BaseConsumer):
""" """
AMQP helper Serial interface consumer:
- Set up process which communicates with serial interface of mote through USB port
- transmits in and out 802.15.4 packet messages
""" """
def __init__(self, user, password, session, server, exchange, name, consumer_name): def __init__(self, user, password, session, server, exchange, name, consumer_name):
super(SerialConsumer, self).__init__(user, password, session, server, exchange, name, consumer_name)
self.dispatcher = {
MsgPacketInjectRaw: self.handle_data,
}
subscriptions = [
MsgPacketInjectRaw.routing_key.replace('*', name).replace('ip.tun', '802154.serial'),
]
super(SerialConsumer, self).__init__(user, password, session, server, exchange, name, consumer_name,
subscriptions)
self.message_count = 0 self.message_count = 0
self.output = '' self.output = ''
self.serial_listener = None self.serial_listener = None
...@@ -49,24 +62,15 @@ class SerialConsumer(BaseConsumer): ...@@ -49,24 +62,15 @@ class SerialConsumer(BaseConsumer):
serial_listener_th.daemon = True serial_listener_th.daemon = True
serial_listener_th.start() serial_listener_th.start()
except KeyError as e: except KeyError as e:
logging.warning( logging.warning(
'Cannot retrieve environment variables for serial connection: ' 'Cannot retrieve environment variables for serial connection: '
'FINTEROP_CONNECTOR_SERIAL_PORT/FINTEROP_CONNECTOR_BAUDRATE ' 'FINTEROP_CONNECTOR_SERIAL_PORT/FINTEROP_CONNECTOR_BAUDRATE '
'If no sniffer/injector needed for test ignore this warning ') 'If no sniffer/injector needed for test ignore this warning ')
def handle_data(self, body, message): def handle_data(self, message):
""" """
Function that will handle serial management messages Forwards data packets from AMQP BUS to serial interface
exchange:
- default
example:
{
}
""" """
if self.serial_port is None: if self.serial_port is None:
...@@ -80,7 +84,7 @@ class SerialConsumer(BaseConsumer): ...@@ -80,7 +84,7 @@ class SerialConsumer(BaseConsumer):
try: try:
self.output = 'c0' self.output = 'c0'
for c in body['data']: for c in message.data:
if format(c, '02x') == 'c0': if format(c, '02x') == 'c0':
# endslip # endslip
self.output += 'db' self.output += 'db'
...@@ -100,28 +104,9 @@ class SerialConsumer(BaseConsumer): ...@@ -100,28 +104,9 @@ class SerialConsumer(BaseConsumer):
print(arrow_down) print(arrow_down)
log.info('\n # # # # # # # # # # # # SERIAL INTERFACE # # # # # # # # # # # # ' + log.info('\n # # # # # # # # # # # # SERIAL INTERFACE # # # # # # # # # # # # ' +
'\n data packet EventBus -> Serial' + '\n data packet EventBus -> Serial' +
'\n' + json.dumps(body) + '\n' + message.to_json() +
'\n # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # ' '\n # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # '
) )
message.ack()
def handle_control(self, body, message):
msg = None
try:
msg = json.loads(body)
log.debug(message)
except ValueError as e:
message.ack()
log.error(e)
log.error("Incorrect message: {0}".format(body))
return
if msg["_type"] in self.dispatcher.keys():
self.dispatcher[msg["_type"]](msg)
else:
log.warning("Not supported action")
class SerialConnector(BaseController): class SerialConnector(BaseController):
......
...@@ -7,48 +7,54 @@ import json ...@@ -7,48 +7,54 @@ import json
import logging import logging
import sys import sys
import datetime import datetime
from utils.opentun import OpenTunLinux, OpenTunMACOS
from utils import arrow_down, arrow_up, finterop_banner
from utils.messages import *
from kombu import Producer from kombu import Producer
from connectors.base import BaseController, BaseConsumer from connectors.base import BaseController, BaseConsumer
from utils import arrow_down, arrow_up, finterop_banner
from utils.opentun import OpenTunLinux, OpenTunMACOS
__version__ = (0, 0, 1) __version__ = (0, 1, 0)
class TunConsumer(BaseConsumer): class TunConsumer(BaseConsumer):
""" """
AMQP helper Tun interface consumer:
- creates tunnel interface (RAW_IP)
- inyects IPv6 packets comming from event bus into tun interaface
- sniffs and forwards packets from tun to event bus
""" """
def __init__(self, user, password, session, server, exchange, name, consumer_name): def __init__(self, user, password, session, server, exchange, name, consumer_name):
super(TunConsumer, self).__init__(user, password, session, server, exchange, name, consumer_name)
self.dispatcher = { self.dispatcher = {
"tun.start": self.handle_start, MsgAgentTunStart: self.handle_start,
MsgPacketInjectRaw: self.handle_raw_packet_to_inject,