~singpolyma/biboumi

ref: 0bb4f144fcded6b5753b5de7493b7b10474c9a1f biboumi/tests/end_to_end/__main__.py -rw-r--r-- 12.2 KiB
0bb4f144 — Félix Baylac-Jacqué CLI: Add a test config flag 2 years ago
                                                                                
51a34f83 Florent Le Coz
bd41bc8b louiz’
606700f4 louiz’
057ef25e louiz’
bd41bc8b louiz’
17366413 louiz’
5ae33d6b louiz’
51a34f83 Florent Le Coz
71b3f8f5 louiz’
17366413 louiz’
d57b8bb8 louiz’
51a34f83 Florent Le Coz
fdf336af louiz’
51a34f83 Florent Le Coz
d57b8bb8 louiz’
51a34f83 Florent Le Coz
bd41bc8b louiz’
bd625aa5 louiz’
bd41bc8b louiz’
bd625aa5 louiz’
bd41bc8b louiz’
55d1d817 Florent Le Coz
bd41bc8b louiz’
55d1d817 Florent Le Coz
51a34f83 Florent Le Coz
7e5cd2f1 louiz’
51a34f83 Florent Le Coz
55e53852 louiz’
51a34f83 Florent Le Coz
fdf336af louiz’
51a34f83 Florent Le Coz
440e04c6 louiz’
55d1d817 Florent Le Coz
51a34f83 Florent Le Coz
0b1bf924 louiz’
51a34f83 Florent Le Coz
440e04c6 louiz’
3584979c louiz’
440e04c6 louiz’
7536a1b3 louiz’
51a34f83 Florent Le Coz
55d1d817 Florent Le Coz
bd625aa5 louiz’
55d1d817 Florent Le Coz
51a34f83 Florent Le Coz
440e04c6 louiz’
6ca8cd50 louiz’
5ae33d6b louiz’
bd41bc8b louiz’
5ae33d6b louiz’
51a34f83 Florent Le Coz
6ca8cd50 louiz’
51a34f83 Florent Le Coz
eb976918 louiz’
51a34f83 Florent Le Coz
d57b8bb8 louiz’
606700f4 louiz’
51a34f83 Florent Le Coz
0b1bf924 louiz’
51a34f83 Florent Le Coz
eb976918 louiz’
51a34f83 Florent Le Coz
eb976918 louiz’
51a34f83 Florent Le Coz
606700f4 louiz’
64f341ee louiz’
606700f4 louiz’
6ca8cd50 louiz’
606700f4 louiz’
6ca8cd50 louiz’
606700f4 louiz’
8cef7303 louiz’
606700f4 louiz’
45f442b6 louiz’
606700f4 louiz’
2e1ddeb6 louiz’
e4550d32 louiz’
057ef25e louiz’
81fb7389 louiz’
606700f4 louiz’
51a34f83 Florent Le Coz
0ce75ab5 louiz’
51a34f83 Florent Le Coz
6ca8cd50 louiz’
51a34f83 Florent Le Coz
1ee5f8e0 Florent Le Coz
6ca8cd50 louiz’
1ee5f8e0 Florent Le Coz
20b187da louiz’
f7fa3443 louiz’
20f36df0 louiz’
51a34f83 Florent Le Coz
6ca8cd50 louiz’
51a34f83 Florent Le Coz
82e0cf99 louiz’
6ca8cd50 louiz’
20f36df0 louiz’
51a34f83 Florent Le Coz
20f36df0 louiz’
51a34f83 Florent Le Coz
0b1bf924 louiz’
51a34f83 Florent Le Coz
d57b8bb8 louiz’
ea35a12d louiz’
1ee5f8e0 Florent Le Coz
98517e9a louiz’
d452c2ff louiz’
89868e4c louiz’
58b3345f louiz’
439ea262 louiz’
98517e9a louiz’
439ea262 louiz’
d452c2ff louiz’
89868e4c louiz’
bfcf2945 louiz’
51a34f83 Florent Le Coz
0b1bf924 louiz’
bd41bc8b louiz’
17366413 louiz’
bd41bc8b louiz’
efb695be louiz’
bd41bc8b louiz’
0b1bf924 louiz’
a4e2e56a louiz’
51a34f83 Florent Le Coz
bd41bc8b louiz’
51a34f83 Florent Le Coz
327c821f louiz’
606700f4 louiz’
45f442b6 louiz’
606700f4 louiz’
327c821f louiz’
3ddca868 louiz’
327c821f louiz’
3ddca868 louiz’
81fb7389 louiz’
606700f4 louiz’
45f442b6 louiz’
51a34f83 Florent Le Coz
bd41bc8b louiz’
6ca8cd50 louiz’
51a34f83 Florent Le Coz
6ca8cd50 louiz’
51a34f83 Florent Le Coz
e43885c4 louiz’
51a34f83 Florent Le Coz
45f442b6 louiz’
606700f4 louiz’
3fe55ab0 louiz’
606700f4 louiz’
51a34f83 Florent Le Coz
bd41bc8b louiz’
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
#!/usr/bin/env python3

from functions import StanzaError, SkipStepError

import collections
import subprocess
import importlib
import sequences
import datetime
import slixmpp
import asyncio
import logging
import signal
import atexit
import sys
import os

from slixmpp.xmlstream.matcher.base import MatcherBase

if not hasattr(asyncio, "ensure_future"):
    asyncio.ensure_future = getattr(asyncio, "async")

class MatchAll(MatcherBase):
    """match everything"""

    def match(self, xml):
        return True


class Scenario:
    """Defines a list of actions that are executed in sequence, until one of
    them throws an exception, or until the end.  An action can be something
    like “send a stanza”, “receive the next stanza and check that it matches
    the given XPath”, “send a signal”, “wait for the end of the process”,
    etc
    """

    def __init__(self, name, steps, conf):
        """
        Steps is a list of 2-tuple:
        [(action, answer), (action, answer)]
        """
        self.name = name
        self.steps = []
        self.conf = conf

        def unwrap_tuples(elements):
            """Yields all the value contained in the tuples, of tuples, of tuples…
            For example unwrap_tuples((1, 2, 3, (4, 5, (6,)))) will yield 1, 2, 3, 4, 5, 6
            This works with any depth"""
            if isinstance(elements, collections.abc.Iterable):
                for elem in elements:
                    yield from unwrap_tuples(elem)
            else:
                yield elements

        for step in unwrap_tuples(steps):
            self.steps.append(step)

class XMPPComponent(slixmpp.BaseXMPP):
    """
    XMPPComponent sending a “scenario” of stanzas, checking that the responses
    match the expected results.
    """

    def __init__(self, scenario, biboumi):
        super().__init__(jid="biboumi.localhost", default_ns="jabber:component:accept")
        self.is_component = True
        self.auto_authorize = None # Do not accept or reject subscribe requests automatically
        self.auto_subscribe = False
        self.stream_header = '<stream:stream %s %s from="%s" id="%s">' % (
            'xmlns="jabber:component:accept"',
            'xmlns:stream="%s"' % self.stream_ns,
            self.boundjid, self.new_id())
        self.stream_footer = "</stream:stream>"

        self.register_handler(slixmpp.Callback('Match All',
                                               MatchAll(None),
                                               self.handle_incoming_stanza))

        self.add_event_handler("session_end", self.on_end_session)

        asyncio.ensure_future(self.accept_routine())

        self.scenario = scenario
        self.biboumi = biboumi
        self.timeout_handler = None
        # A callable, taking a stanza as argument and raising a StanzaError
        # exception if the test should fail.
        self.stanza_checker = None
        self.failed = False
        self.accepting_server = None

        self.saved_values = {}

    def error(self, message):
        print("[31;1mFailure[0m: %s" % (message,))
        self.scenario.steps = []
        self.failed = True

    def on_timeout(self, xpaths):
        error_msg = "Timeout while waiting for a stanza that would match the expected xpath(s):"
        for xpath in xpaths:
            error_msg += "\n" + str(xpath)
        self.error(error_msg)
        self.run_scenario()

    def on_end_session(self, _):
        self.loop.stop()

    def handle_incoming_stanza(self, stanza):
        if self.stanza_checker:
            try:
                self.stanza_checker(stanza)
            except StanzaError as e:
                self.error(e)
            except SkipStepError:
                # Run the next step and then re-handle this same stanza
                self.run_scenario()
                return self.handle_incoming_stanza(stanza)
            self.stanza_checker = None
        self.run_scenario()

    def run_scenario(self):
        if self.timeout_handler is not None:
            self.timeout_handler.cancel()
            self.timeout_handler = None
        if self.scenario.steps:
            step = self.scenario.steps.pop(0)
            try:
                step(xmpp=self, biboumi=self.biboumi)
            except Exception as e:
                self.error(e)
                self.run_scenario()
        else:
            if self.biboumi:
                self.biboumi.stop()


    async def accept_routine(self):
        self.accepting_server = await self.loop.create_server(lambda: self,
                                                              "127.0.0.1", 8811, reuse_address=True)


class ProcessRunner:
    def __init__(self):
        self.process = None
        self.signal_sent = False
        self.create = None

    async def start(self):
        self.process = await self.create

    async def wait(self):
        code = await self.process.wait()
        return code

    def stop(self):
        if not self.signal_sent:
            self.signal_sent = True
            if self.process:
                self.process.send_signal(signal.SIGINT)

    def __del__(self):
        self.stop()


class BiboumiRunner(ProcessRunner):
    def __init__(self, name):
        super().__init__()
        self.name = name
        self.fd = open("biboumi_%s_output.txt" % (name,), "w")
        with_valgrind = os.environ.get("E2E_WITH_VALGRIND") is not None
        if with_valgrind:
            self.create = asyncio.create_subprocess_exec("valgrind", "--suppressions=" + (os.environ.get("E2E_BIBOUMI_SUPP_DIR") or "") + "biboumi.supp", "--leak-check=full", "--show-leak-kinds=all",
                                                         "--errors-for-leak-kinds=all", "--error-exitcode=16",
                                                         "./biboumi", "test.conf", stdin=None, stdout=self.fd,
                                                         stderr=self.fd, loop=None, limit=None)
        else:
            self.create = asyncio.create_subprocess_exec("./biboumi", "test.conf", stdin=None, stdout=self.fd,
                                                         stderr=self.fd, loop=None, limit=None)


class IrcServerRunner(ProcessRunner):
    def __init__(self):
        super().__init__()
        # Always start with a fresh state
        try:
            os.remove("ircd.db")
        except FileNotFoundError:
            pass
        subprocess.run(["oragono", "mkcerts", "--conf", os.getcwd() + "/../tests/end_to_end/ircd.yaml"])
        self.create = asyncio.create_subprocess_exec("oragono", "run", "--conf", os.getcwd() + "/../tests/end_to_end/ircd.yaml",
                                                     stderr=asyncio.subprocess.PIPE)

class BiboumiTest:
    """
    Spawns a biboumi process and a fake XMPP Component that will run a
    Scenario.  It redirects the outputs of the subprocess into separated
    files, and detects any failure in the running of the scenario.
    """

    def __init__(self, scenario, expected_code=0):
        self.scenario = scenario
        self.expected_code = expected_code

    def run(self):
        with_valgrind = os.environ.get("E2E_WITH_VALGRIND") is not None
        print("Running scenario: [33;1m%s[0m%s" % (self.scenario.name, " (with valgrind)" if with_valgrind else ''))
        # Redirect the slixmpp logging into a specific file
        output_filename = "slixmpp_%s_output.txt" % (self.scenario.name,)
        with open(output_filename, "w"):
            pass
        logging.basicConfig(level=logging.DEBUG,
                            format='%(levelname)-8s %(message)s',
                            filename=output_filename)

        with open("test.conf", "w") as fd:
            fd.write(confs[self.scenario.conf])

        try:
            os.remove("e2e_test.sqlite")
        except FileNotFoundError:
            pass

        start_datetime = datetime.datetime.now()

        # Start the XMPP component and biboumi
        biboumi = BiboumiRunner(self.scenario.name)
        xmpp = XMPPComponent(self.scenario, biboumi)
        asyncio.get_event_loop().run_until_complete(biboumi.start())

        asyncio.get_event_loop().call_soon(xmpp.run_scenario)

        xmpp.process()
        code = asyncio.get_event_loop().run_until_complete(biboumi.wait())
        xmpp.biboumi = None
        self.scenario.steps.clear()

        delta = datetime.datetime.now() - start_datetime

        failed = False
        if not xmpp.failed:
            if code != self.expected_code:
                xmpp.error("Wrong return code from biboumi's process: %d" % (code,))
                failed = True
            else:
                print("[32;1mSuccess![0m ({}s)".format(round(delta.total_seconds(), 2)))
        else:
            failed = True

        xmpp.saved_values.clear()

        if xmpp.server:
            xmpp.accepting_server.close()

        return not failed


confs = {
'basic':
"""hostname=biboumi.localhost
password=coucou
db_name=e2e_test.sqlite
port=8811
admin=admin@example.com
identd_port=1113
outgoing_bind=127.0.0.1""",

'fixed_server':
"""hostname=biboumi.localhost
password=coucou
db_name=e2e_test.sqlite
port=8811
fixed_irc_server=irc.localhost
admin=admin@example.com
identd_port=1113
""",

'persistent_by_default':
"""hostname=biboumi.localhost
password=coucou
db_name=e2e_test.sqlite
port=8811
persistent_by_default=true
""",}


def get_scenarios(test_path, provided_scenar_names):
    """
    :param test_path: The path containing all the tests
    :param provided_scenar_names: a list of scenario names provided on the
    command line by the user. May be empty
    :return: The list of scenarios to be run. If provided_scenar_names is
    empty, we return all the existing scenarios, otherwise we just return
    the one from that list
    """
    scenarios = []
    for entry in os.scandir(os.path.join(test_path, "scenarios")):
        if entry.is_file() and not entry.name.startswith('.') and entry.name.endswith('.py'):
            module_name = entry.name[:-3]
            if provided_scenar_names and module_name not in provided_scenar_names:
                continue
            if module_name == "__init__" or (provided_scenar_names and module_name not in provided_scenar_names):
                continue
            module_full_path = "scenarios.{}".format(module_name)
            mod = importlib.import_module(module_full_path)
            conf = "basic"
            if hasattr(mod, "conf"):
                conf = mod.conf
            # Every scenario needs to start with the handshake sequence.
            # Instead of repeating it everytime, we add it implicitely. This
            # is done here.
            scenarios.append(Scenario(module_name, (sequences.handshake(),) + mod.scenario, conf))
    return scenarios


if __name__ == '__main__':
    atexit.register(asyncio.get_event_loop().close)

    provided_scenar_names = sys.argv[1:]
    scenarios = get_scenarios(os.path.abspath(os.path.dirname(__file__)), provided_scenar_names)

    irc_output = open("irc_output.txt", "w")
    irc = IrcServerRunner()
    print("Starting irc server…")
    asyncio.get_event_loop().run_until_complete(irc.start())
    while True:
        res = asyncio.get_event_loop().run_until_complete(irc.process.stderr.readline())
        irc_output.write(res.decode())
        if not res:
            print("IRC server failed to start, see irc_output.txt for more details. Exiting…")
            sys.exit(1)
        if b"Server running" in res:
            break
    print("irc server started.")

    number_of_checks = len([s for s in scenarios if s.name in provided_scenar_names]) if provided_scenar_names else len(scenarios)
    print("Running %s checks for biboumi." % number_of_checks)

    failures = 0
    for s in scenarios:
        test = BiboumiTest(s)
        if not test.run():
            print("You can check the files slixmpp_%s_output.txt and biboumi_%s_output.txt to help you debug." %
                  (s.name, s.name))
            failures += 1
        sys.stdout.flush()

    print("Waiting for irc server to exit…")
    irc.stop()
    asyncio.get_event_loop().run_until_complete(irc.wait())

    if failures:
        print("%d test%s failed, please fix %s." % (failures, 's' if failures > 1 else '',
                                                    'them' if failures > 1 else 'it'))
        sys.exit(1)
    else:
        print("All tests passed successfully")