1#!/usr/bin/python3
2
3# Copyright (C) 2017 Netronome Systems, Inc.
4#
5# This software is licensed under the GNU General License Version 2,
6# June 1991 as shown in the file COPYING in the top-level directory of this
7# source tree.
8#
9# THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
10# WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
11# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
12# FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
13# OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
14# THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
15
16from datetime import datetime
17import argparse
18import json
19import os
20import pprint
21import random
22import string
23import struct
24import subprocess
25import time
26import traceback
27
28logfile = None
29log_level = 1
30skip_extack = False
31bpf_test_dir = os.path.dirname(os.path.realpath(__file__))
32pp = pprint.PrettyPrinter()
33devs = [] # devices we created for clean up
34files = [] # files to be removed
35netns = [] # net namespaces to be removed
36
37def log_get_sec(level=0):
38    return "*" * (log_level + level)
39
40def log_level_inc(add=1):
41    global log_level
42    log_level += add
43
44def log_level_dec(sub=1):
45    global log_level
46    log_level -= sub
47
48def log_level_set(level):
49    global log_level
50    log_level = level
51
52def log(header, data, level=None):
53    """
54    Output to an optional log.
55    """
56    if logfile is None:
57        return
58    if level is not None:
59        log_level_set(level)
60
61    if not isinstance(data, str):
62        data = pp.pformat(data)
63
64    if len(header):
65        logfile.write("\n" + log_get_sec() + " ")
66        logfile.write(header)
67    if len(header) and len(data.strip()):
68        logfile.write("\n")
69    logfile.write(data)
70
71def skip(cond, msg):
72    if not cond:
73        return
74    print("SKIP: " + msg)
75    log("SKIP: " + msg, "", level=1)
76    os.sys.exit(0)
77
78def fail(cond, msg):
79    if not cond:
80        return
81    print("FAIL: " + msg)
82    tb = "".join(traceback.extract_stack().format())
83    print(tb)
84    log("FAIL: " + msg, tb, level=1)
85    os.sys.exit(1)
86
87def start_test(msg):
88    log(msg, "", level=1)
89    log_level_inc()
90    print(msg)
91
92def cmd(cmd, shell=True, include_stderr=False, background=False, fail=True):
93    """
94    Run a command in subprocess and return tuple of (retval, stdout);
95    optionally return stderr as well as third value.
96    """
97    proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE,
98                            stderr=subprocess.PIPE)
99    if background:
100        msg = "%s START: %s" % (log_get_sec(1),
101                                datetime.now().strftime("%H:%M:%S.%f"))
102        log("BKG " + proc.args, msg)
103        return proc
104
105    return cmd_result(proc, include_stderr=include_stderr, fail=fail)
106
107def cmd_result(proc, include_stderr=False, fail=False):
108    stdout, stderr = proc.communicate()
109    stdout = stdout.decode("utf-8")
110    stderr = stderr.decode("utf-8")
111    proc.stdout.close()
112    proc.stderr.close()
113
114    stderr = "\n" + stderr
115    if stderr[-1] == "\n":
116        stderr = stderr[:-1]
117
118    sec = log_get_sec(1)
119    log("CMD " + proc.args,
120        "RETCODE: %d\n%s STDOUT:\n%s%s STDERR:%s\n%s END: %s" %
121        (proc.returncode, sec, stdout, sec, stderr,
122         sec, datetime.now().strftime("%H:%M:%S.%f")))
123
124    if proc.returncode != 0 and fail:
125        if len(stderr) > 0 and stderr[-1] == "\n":
126            stderr = stderr[:-1]
127        raise Exception("Command failed: %s\n%s" % (proc.args, stderr))
128
129    if include_stderr:
130        return proc.returncode, stdout, stderr
131    else:
132        return proc.returncode, stdout
133
134def rm(f):
135    cmd("rm -f %s" % (f))
136    if f in files:
137        files.remove(f)
138
139def tool(name, args, flags, JSON=True, ns="", fail=True, include_stderr=False):
140    params = ""
141    if JSON:
142        params += "%s " % (flags["json"])
143
144    if ns != "":
145        ns = "ip netns exec %s " % (ns)
146
147    if include_stderr:
148        ret, stdout, stderr = cmd(ns + name + " " + params + args,
149                                  fail=fail, include_stderr=True)
150    else:
151        ret, stdout = cmd(ns + name + " " + params + args,
152                          fail=fail, include_stderr=False)
153
154    if JSON and len(stdout.strip()) != 0:
155        out = json.loads(stdout)
156    else:
157        out = stdout
158
159    if include_stderr:
160        return ret, out, stderr
161    else:
162        return ret, out
163
164def bpftool(args, JSON=True, ns="", fail=True, include_stderr=False):
165    return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns,
166                fail=fail, include_stderr=include_stderr)
167
168def bpftool_prog_list(expected=None, ns=""):
169    _, progs = bpftool("prog show", JSON=True, ns=ns, fail=True)
170    # Remove the base progs
171    for p in base_progs:
172        if p in progs:
173            progs.remove(p)
174    if expected is not None:
175        if len(progs) != expected:
176            fail(True, "%d BPF programs loaded, expected %d" %
177                 (len(progs), expected))
178    return progs
179
180def bpftool_map_list(expected=None, ns=""):
181    _, maps = bpftool("map show", JSON=True, ns=ns, fail=True)
182    # Remove the base maps
183    for m in base_maps:
184        if m in maps:
185            maps.remove(m)
186    if expected is not None:
187        if len(maps) != expected:
188            fail(True, "%d BPF maps loaded, expected %d" %
189                 (len(maps), expected))
190    return maps
191
192def bpftool_prog_list_wait(expected=0, n_retry=20):
193    for i in range(n_retry):
194        nprogs = len(bpftool_prog_list())
195        if nprogs == expected:
196            return
197        time.sleep(0.05)
198    raise Exception("Time out waiting for program counts to stabilize want %d, have %d" % (expected, nprogs))
199
200def bpftool_map_list_wait(expected=0, n_retry=20):
201    for i in range(n_retry):
202        nmaps = len(bpftool_map_list())
203        if nmaps == expected:
204            return
205        time.sleep(0.05)
206    raise Exception("Time out waiting for map counts to stabilize want %d, have %d" % (expected, nmaps))
207
208def bpftool_prog_load(sample, file_name, maps=[], prog_type="xdp", dev=None,
209                      fail=True, include_stderr=False):
210    args = "prog load %s %s" % (os.path.join(bpf_test_dir, sample), file_name)
211    if prog_type is not None:
212        args += " type " + prog_type
213    if dev is not None:
214        args += " dev " + dev
215    if len(maps):
216        args += " map " + " map ".join(maps)
217
218    res = bpftool(args, fail=fail, include_stderr=include_stderr)
219    if res[0] == 0:
220        files.append(file_name)
221    return res
222
223def ip(args, force=False, JSON=True, ns="", fail=True, include_stderr=False):
224    if force:
225        args = "-force " + args
226    return tool("ip", args, {"json":"-j"}, JSON=JSON, ns=ns,
227                fail=fail, include_stderr=include_stderr)
228
229def tc(args, JSON=True, ns="", fail=True, include_stderr=False):
230    return tool("tc", args, {"json":"-p"}, JSON=JSON, ns=ns,
231                fail=fail, include_stderr=include_stderr)
232
233def ethtool(dev, opt, args, fail=True):
234    return cmd("ethtool %s %s %s" % (opt, dev["ifname"], args), fail=fail)
235
236def bpf_obj(name, sec=".text", path=bpf_test_dir,):
237    return "obj %s sec %s" % (os.path.join(path, name), sec)
238
239def bpf_pinned(name):
240    return "pinned %s" % (name)
241
242def bpf_bytecode(bytecode):
243    return "bytecode \"%s\"" % (bytecode)
244
245def mknetns(n_retry=10):
246    for i in range(n_retry):
247        name = ''.join([random.choice(string.ascii_letters) for i in range(8)])
248        ret, _ = ip("netns add %s" % (name), fail=False)
249        if ret == 0:
250            netns.append(name)
251            return name
252    return None
253
254def int2str(fmt, val):
255    ret = []
256    for b in struct.pack(fmt, val):
257        ret.append(int(b))
258    return " ".join(map(lambda x: str(x), ret))
259
260def str2int(strtab):
261    inttab = []
262    for i in strtab:
263        inttab.append(int(i, 16))
264    ba = bytearray(inttab)
265    if len(strtab) == 4:
266        fmt = "I"
267    elif len(strtab) == 8:
268        fmt = "Q"
269    else:
270        raise Exception("String array of len %d can't be unpacked to an int" %
271                        (len(strtab)))
272    return struct.unpack(fmt, ba)[0]
273
274class DebugfsDir:
275    """
276    Class for accessing DebugFS directories as a dictionary.
277    """
278
279    def __init__(self, path):
280        self.path = path
281        self._dict = self._debugfs_dir_read(path)
282
283    def __len__(self):
284        return len(self._dict.keys())
285
286    def __getitem__(self, key):
287        if type(key) is int:
288            key = list(self._dict.keys())[key]
289        return self._dict[key]
290
291    def __setitem__(self, key, value):
292        log("DebugFS set %s = %s" % (key, value), "")
293        log_level_inc()
294
295        cmd("echo '%s' > %s/%s" % (value, self.path, key))
296        log_level_dec()
297
298        _, out = cmd('cat %s/%s' % (self.path, key))
299        self._dict[key] = out.strip()
300
301    def _debugfs_dir_read(self, path):
302        dfs = {}
303
304        log("DebugFS state for %s" % (path), "")
305        log_level_inc(add=2)
306
307        _, out = cmd('ls ' + path)
308        for f in out.split():
309            p = os.path.join(path, f)
310            if os.path.isfile(p):
311                _, out = cmd('cat %s/%s' % (path, f))
312                dfs[f] = out.strip()
313            elif os.path.isdir(p):
314                dfs[f] = DebugfsDir(p)
315            else:
316                raise Exception("%s is neither file nor directory" % (p))
317
318        log_level_dec()
319        log("DebugFS state", dfs)
320        log_level_dec()
321
322        return dfs
323
324class NetdevSim:
325    """
326    Class for netdevsim netdevice and its attributes.
327    """
328
329    def __init__(self, link=None):
330        self.link = link
331
332        self.dev = self._netdevsim_create()
333        devs.append(self)
334
335        self.ns = ""
336
337        self.dfs_dir = '/sys/kernel/debug/netdevsim/%s' % (self.dev['ifname'])
338        self.sdev_dir = self.dfs_dir + '/sdev/'
339        self.dfs_refresh()
340
341    def __getitem__(self, key):
342        return self.dev[key]
343
344    def _netdevsim_create(self):
345        link = "" if self.link is None else "link " + self.link.dev['ifname']
346        _, old  = ip("link show")
347        ip("link add sim%d {link} type netdevsim".format(link=link))
348        _, new  = ip("link show")
349
350        for dev in new:
351            f = filter(lambda x: x["ifname"] == dev["ifname"], old)
352            if len(list(f)) == 0:
353                return dev
354
355        raise Exception("failed to create netdevsim device")
356
357    def remove(self):
358        devs.remove(self)
359        ip("link del dev %s" % (self.dev["ifname"]), ns=self.ns)
360
361    def dfs_refresh(self):
362        self.dfs = DebugfsDir(self.dfs_dir)
363        return self.dfs
364
365    def dfs_read(self, f):
366        path = os.path.join(self.dfs_dir, f)
367        _, data = cmd('cat %s' % (path))
368        return data.strip()
369
370    def dfs_num_bound_progs(self):
371        path = os.path.join(self.sdev_dir, "bpf_bound_progs")
372        _, progs = cmd('ls %s' % (path))
373        return len(progs.split())
374
375    def dfs_get_bound_progs(self, expected):
376        progs = DebugfsDir(os.path.join(self.sdev_dir, "bpf_bound_progs"))
377        if expected is not None:
378            if len(progs) != expected:
379                fail(True, "%d BPF programs bound, expected %d" %
380                     (len(progs), expected))
381        return progs
382
383    def wait_for_flush(self, bound=0, total=0, n_retry=20):
384        for i in range(n_retry):
385            nbound = self.dfs_num_bound_progs()
386            nprogs = len(bpftool_prog_list())
387            if nbound == bound and nprogs == total:
388                return
389            time.sleep(0.05)
390        raise Exception("Time out waiting for program counts to stabilize want %d/%d, have %d bound, %d loaded" % (bound, total, nbound, nprogs))
391
392    def set_ns(self, ns):
393        name = "1" if ns == "" else ns
394        ip("link set dev %s netns %s" % (self.dev["ifname"], name), ns=self.ns)
395        self.ns = ns
396
397    def set_mtu(self, mtu, fail=True):
398        return ip("link set dev %s mtu %d" % (self.dev["ifname"], mtu),
399                  fail=fail)
400
401    def set_xdp(self, bpf, mode, force=False, JSON=True, verbose=False,
402                fail=True, include_stderr=False):
403        if verbose:
404            bpf += " verbose"
405        return ip("link set dev %s xdp%s %s" % (self.dev["ifname"], mode, bpf),
406                  force=force, JSON=JSON,
407                  fail=fail, include_stderr=include_stderr)
408
409    def unset_xdp(self, mode, force=False, JSON=True,
410                  fail=True, include_stderr=False):
411        return ip("link set dev %s xdp%s off" % (self.dev["ifname"], mode),
412                  force=force, JSON=JSON,
413                  fail=fail, include_stderr=include_stderr)
414
415    def ip_link_show(self, xdp):
416        _, link = ip("link show dev %s" % (self['ifname']))
417        if len(link) > 1:
418            raise Exception("Multiple objects on ip link show")
419        if len(link) < 1:
420            return {}
421        fail(xdp != "xdp" in link,
422             "XDP program not reporting in iplink (reported %s, expected %s)" %
423             ("xdp" in link, xdp))
424        return link[0]
425
426    def tc_add_ingress(self):
427        tc("qdisc add dev %s ingress" % (self['ifname']))
428
429    def tc_del_ingress(self):
430        tc("qdisc del dev %s ingress" % (self['ifname']))
431
432    def tc_flush_filters(self, bound=0, total=0):
433        self.tc_del_ingress()
434        self.tc_add_ingress()
435        self.wait_for_flush(bound=bound, total=total)
436
437    def tc_show_ingress(self, expected=None):
438        # No JSON support, oh well...
439        flags = ["skip_sw", "skip_hw", "in_hw"]
440        named = ["protocol", "pref", "chain", "handle", "id", "tag"]
441
442        args = "-s filter show dev %s ingress" % (self['ifname'])
443        _, out = tc(args, JSON=False)
444
445        filters = []
446        lines = out.split('\n')
447        for line in lines:
448            words = line.split()
449            if "handle" not in words:
450                continue
451            fltr = {}
452            for flag in flags:
453                fltr[flag] = flag in words
454            for name in named:
455                try:
456                    idx = words.index(name)
457                    fltr[name] = words[idx + 1]
458                except ValueError:
459                    pass
460            filters.append(fltr)
461
462        if expected is not None:
463            fail(len(filters) != expected,
464                 "%d ingress filters loaded, expected %d" %
465                 (len(filters), expected))
466        return filters
467
468    def cls_filter_op(self, op, qdisc="ingress", prio=None, handle=None,
469                      chain=None, cls="", params="",
470                      fail=True, include_stderr=False):
471        spec = ""
472        if prio is not None:
473            spec += " prio %d" % (prio)
474        if handle:
475            spec += " handle %s" % (handle)
476        if chain is not None:
477            spec += " chain %d" % (chain)
478
479        return tc("filter {op} dev {dev} {qdisc} {spec} {cls} {params}"\
480                  .format(op=op, dev=self['ifname'], qdisc=qdisc, spec=spec,
481                          cls=cls, params=params),
482                  fail=fail, include_stderr=include_stderr)
483
484    def cls_bpf_add_filter(self, bpf, op="add", prio=None, handle=None,
485                           chain=None, da=False, verbose=False,
486                           skip_sw=False, skip_hw=False,
487                           fail=True, include_stderr=False):
488        cls = "bpf " + bpf
489
490        params = ""
491        if da:
492            params += " da"
493        if verbose:
494            params += " verbose"
495        if skip_sw:
496            params += " skip_sw"
497        if skip_hw:
498            params += " skip_hw"
499
500        return self.cls_filter_op(op=op, prio=prio, handle=handle, cls=cls,
501                                  chain=chain, params=params,
502                                  fail=fail, include_stderr=include_stderr)
503
504    def set_ethtool_tc_offloads(self, enable, fail=True):
505        args = "hw-tc-offload %s" % ("on" if enable else "off")
506        return ethtool(self, "-K", args, fail=fail)
507
508################################################################################
509def clean_up():
510    global files, netns, devs
511
512    for dev in devs:
513        dev.remove()
514    for f in files:
515        cmd("rm -f %s" % (f))
516    for ns in netns:
517        cmd("ip netns delete %s" % (ns))
518    files = []
519    netns = []
520
521def pin_prog(file_name, idx=0):
522    progs = bpftool_prog_list(expected=(idx + 1))
523    prog = progs[idx]
524    bpftool("prog pin id %d %s" % (prog["id"], file_name))
525    files.append(file_name)
526
527    return file_name, bpf_pinned(file_name)
528
529def pin_map(file_name, idx=0, expected=1):
530    maps = bpftool_map_list(expected=expected)
531    m = maps[idx]
532    bpftool("map pin id %d %s" % (m["id"], file_name))
533    files.append(file_name)
534
535    return file_name, bpf_pinned(file_name)
536
537def check_dev_info_removed(prog_file=None, map_file=None):
538    bpftool_prog_list(expected=0)
539    ret, err = bpftool("prog show pin %s" % (prog_file), fail=False)
540    fail(ret == 0, "Showing prog with removed device did not fail")
541    fail(err["error"].find("No such device") == -1,
542         "Showing prog with removed device expected ENODEV, error is %s" %
543         (err["error"]))
544
545    bpftool_map_list(expected=0)
546    ret, err = bpftool("map show pin %s" % (map_file), fail=False)
547    fail(ret == 0, "Showing map with removed device did not fail")
548    fail(err["error"].find("No such device") == -1,
549         "Showing map with removed device expected ENODEV, error is %s" %
550         (err["error"]))
551
552def check_dev_info(other_ns, ns, prog_file=None, map_file=None, removed=False):
553    progs = bpftool_prog_list(expected=1, ns=ns)
554    prog = progs[0]
555
556    fail("dev" not in prog.keys(), "Device parameters not reported")
557    dev = prog["dev"]
558    fail("ifindex" not in dev.keys(), "Device parameters not reported")
559    fail("ns_dev" not in dev.keys(), "Device parameters not reported")
560    fail("ns_inode" not in dev.keys(), "Device parameters not reported")
561
562    if not other_ns:
563        fail("ifname" not in dev.keys(), "Ifname not reported")
564        fail(dev["ifname"] != sim["ifname"],
565             "Ifname incorrect %s vs %s" % (dev["ifname"], sim["ifname"]))
566    else:
567        fail("ifname" in dev.keys(), "Ifname is reported for other ns")
568
569    maps = bpftool_map_list(expected=2, ns=ns)
570    for m in maps:
571        fail("dev" not in m.keys(), "Device parameters not reported")
572        fail(dev != m["dev"], "Map's device different than program's")
573
574def check_extack(output, reference, args):
575    if skip_extack:
576        return
577    lines = output.split("\n")
578    comp = len(lines) >= 2 and lines[1] == 'Error: ' + reference
579    fail(not comp, "Missing or incorrect netlink extack message")
580
581def check_extack_nsim(output, reference, args):
582    check_extack(output, "netdevsim: " + reference, args)
583
584def check_no_extack(res, needle):
585    fail((res[1] + res[2]).count(needle) or (res[1] + res[2]).count("Warning:"),
586         "Found '%s' in command output, leaky extack?" % (needle))
587
588def check_verifier_log(output, reference):
589    lines = output.split("\n")
590    for l in reversed(lines):
591        if l == reference:
592            return
593    fail(True, "Missing or incorrect message from netdevsim in verifier log")
594
595def check_multi_basic(two_xdps):
596    fail(two_xdps["mode"] != 4, "Bad mode reported with multiple programs")
597    fail("prog" in two_xdps, "Base program reported in multi program mode")
598    fail(len(two_xdps["attached"]) != 2,
599         "Wrong attached program count with two programs")
600    fail(two_xdps["attached"][0]["prog"]["id"] ==
601         two_xdps["attached"][1]["prog"]["id"],
602         "Offloaded and other programs have the same id")
603
604def test_spurios_extack(sim, obj, skip_hw, needle):
605    res = sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=skip_hw,
606                                 include_stderr=True)
607    check_no_extack(res, needle)
608    res = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
609                                 skip_hw=skip_hw, include_stderr=True)
610    check_no_extack(res, needle)
611    res = sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf",
612                            include_stderr=True)
613    check_no_extack(res, needle)
614
615def test_multi_prog(sim, obj, modename, modeid):
616    start_test("Test multi-attachment XDP - %s + offload..." %
617               (modename or "default", ))
618    sim.set_xdp(obj, "offload")
619    xdp = sim.ip_link_show(xdp=True)["xdp"]
620    offloaded = sim.dfs_read("bpf_offloaded_id")
621    fail("prog" not in xdp, "Base program not reported in single program mode")
622    fail(len(xdp["attached"]) != 1,
623         "Wrong attached program count with one program")
624
625    sim.set_xdp(obj, modename)
626    two_xdps = sim.ip_link_show(xdp=True)["xdp"]
627
628    fail(xdp["attached"][0] not in two_xdps["attached"],
629         "Offload program not reported after other activated")
630    check_multi_basic(two_xdps)
631
632    offloaded2 = sim.dfs_read("bpf_offloaded_id")
633    fail(offloaded != offloaded2,
634         "Offload ID changed after loading other program")
635
636    start_test("Test multi-attachment XDP - replace...")
637    ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
638    fail(ret == 0, "Replaced one of programs without -force")
639    check_extack(err, "XDP program already attached.", args)
640
641    if modename == "" or modename == "drv":
642        othermode = "" if modename == "drv" else "drv"
643        start_test("Test multi-attachment XDP - detach...")
644        ret, _, err = sim.unset_xdp(othermode, force=True,
645                                    fail=False, include_stderr=True)
646        fail(ret == 0, "Removed program with a bad mode")
647        check_extack(err, "program loaded with different flags.", args)
648
649    sim.unset_xdp("offload")
650    xdp = sim.ip_link_show(xdp=True)["xdp"]
651    offloaded = sim.dfs_read("bpf_offloaded_id")
652
653    fail(xdp["mode"] != modeid, "Bad mode reported after multiple programs")
654    fail("prog" not in xdp,
655         "Base program not reported after multi program mode")
656    fail(xdp["attached"][0] not in two_xdps["attached"],
657         "Offload program not reported after other activated")
658    fail(len(xdp["attached"]) != 1,
659         "Wrong attached program count with remaining programs")
660    fail(offloaded != "0", "Offload ID reported with only other program left")
661
662    start_test("Test multi-attachment XDP - reattach...")
663    sim.set_xdp(obj, "offload")
664    two_xdps = sim.ip_link_show(xdp=True)["xdp"]
665
666    fail(xdp["attached"][0] not in two_xdps["attached"],
667         "Other program not reported after offload activated")
668    check_multi_basic(two_xdps)
669
670    start_test("Test multi-attachment XDP - device remove...")
671    sim.remove()
672
673    sim = NetdevSim()
674    sim.set_ethtool_tc_offloads(True)
675    return sim
676
677# Parse command line
678parser = argparse.ArgumentParser()
679parser.add_argument("--log", help="output verbose log to given file")
680args = parser.parse_args()
681if args.log:
682    logfile = open(args.log, 'w+')
683    logfile.write("# -*-Org-*-")
684
685log("Prepare...", "", level=1)
686log_level_inc()
687
688# Check permissions
689skip(os.getuid() != 0, "test must be run as root")
690
691# Check tools
692ret, progs = bpftool("prog", fail=False)
693skip(ret != 0, "bpftool not installed")
694base_progs = progs
695_, base_maps = bpftool("map")
696
697# Check netdevsim
698ret, out = cmd("modprobe netdevsim", fail=False)
699skip(ret != 0, "netdevsim module could not be loaded")
700
701# Check debugfs
702_, out = cmd("mount")
703if out.find("/sys/kernel/debug type debugfs") == -1:
704    cmd("mount -t debugfs none /sys/kernel/debug")
705
706# Check samples are compiled
707samples = ["sample_ret0.o", "sample_map_ret0.o"]
708for s in samples:
709    ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False)
710    skip(ret != 0, "sample %s/%s not found, please compile it" %
711         (bpf_test_dir, s))
712
713# Check if iproute2 is built with libmnl (needed by extack support)
714_, _, err = cmd("tc qdisc delete dev lo handle 0",
715                fail=False, include_stderr=True)
716if err.find("Error: Failed to find qdisc with specified handle.") == -1:
717    print("Warning: no extack message in iproute2 output, libmnl missing?")
718    log("Warning: no extack message in iproute2 output, libmnl missing?", "")
719    skip_extack = True
720
721# Check if net namespaces seem to work
722ns = mknetns()
723skip(ns is None, "Could not create a net namespace")
724cmd("ip netns delete %s" % (ns))
725netns = []
726
727try:
728    obj = bpf_obj("sample_ret0.o")
729    bytecode = bpf_bytecode("1,6 0 0 4294967295,")
730
731    start_test("Test destruction of generic XDP...")
732    sim = NetdevSim()
733    sim.set_xdp(obj, "generic")
734    sim.remove()
735    bpftool_prog_list_wait(expected=0)
736
737    sim = NetdevSim()
738    sim.tc_add_ingress()
739
740    start_test("Test TC non-offloaded...")
741    ret, _ = sim.cls_bpf_add_filter(obj, skip_hw=True, fail=False)
742    fail(ret != 0, "Software TC filter did not load")
743
744    start_test("Test TC non-offloaded isn't getting bound...")
745    ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
746    fail(ret != 0, "Software TC filter did not load")
747    sim.dfs_get_bound_progs(expected=0)
748
749    sim.tc_flush_filters()
750
751    start_test("Test TC offloads are off by default...")
752    ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
753                                         fail=False, include_stderr=True)
754    fail(ret == 0, "TC filter loaded without enabling TC offloads")
755    check_extack(err, "TC offload is disabled on net device.", args)
756    sim.wait_for_flush()
757
758    sim.set_ethtool_tc_offloads(True)
759    sim.dfs["bpf_tc_non_bound_accept"] = "Y"
760
761    start_test("Test TC offload by default...")
762    ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
763    fail(ret != 0, "Software TC filter did not load")
764    sim.dfs_get_bound_progs(expected=0)
765    ingress = sim.tc_show_ingress(expected=1)
766    fltr = ingress[0]
767    fail(not fltr["in_hw"], "Filter not offloaded by default")
768
769    sim.tc_flush_filters()
770
771    start_test("Test TC cBPF bytcode tries offload by default...")
772    ret, _ = sim.cls_bpf_add_filter(bytecode, fail=False)
773    fail(ret != 0, "Software TC filter did not load")
774    sim.dfs_get_bound_progs(expected=0)
775    ingress = sim.tc_show_ingress(expected=1)
776    fltr = ingress[0]
777    fail(not fltr["in_hw"], "Bytecode not offloaded by default")
778
779    sim.tc_flush_filters()
780    sim.dfs["bpf_tc_non_bound_accept"] = "N"
781
782    start_test("Test TC cBPF unbound bytecode doesn't offload...")
783    ret, _, err = sim.cls_bpf_add_filter(bytecode, skip_sw=True,
784                                         fail=False, include_stderr=True)
785    fail(ret == 0, "TC bytecode loaded for offload")
786    check_extack_nsim(err, "netdevsim configured to reject unbound programs.",
787                      args)
788    sim.wait_for_flush()
789
790    start_test("Test non-0 chain offload...")
791    ret, _, err = sim.cls_bpf_add_filter(obj, chain=1, prio=1, handle=1,
792                                         skip_sw=True,
793                                         fail=False, include_stderr=True)
794    fail(ret == 0, "Offloaded a filter to chain other than 0")
795    check_extack(err, "Driver supports only offload of chain 0.", args)
796    sim.tc_flush_filters()
797
798    start_test("Test TC replace...")
799    sim.cls_bpf_add_filter(obj, prio=1, handle=1)
800    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1)
801    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
802
803    sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_sw=True)
804    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_sw=True)
805    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
806
807    sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=True)
808    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_hw=True)
809    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
810
811    start_test("Test TC replace bad flags...")
812    for i in range(3):
813        for j in range(3):
814            ret, _ = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
815                                            skip_sw=(j == 1), skip_hw=(j == 2),
816                                            fail=False)
817            fail(bool(ret) != bool(j),
818                 "Software TC incorrect load in replace test, iteration %d" %
819                 (j))
820        sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
821
822    start_test("Test spurious extack from the driver...")
823    test_spurios_extack(sim, obj, False, "netdevsim")
824    test_spurios_extack(sim, obj, True, "netdevsim")
825
826    sim.set_ethtool_tc_offloads(False)
827
828    test_spurios_extack(sim, obj, False, "TC offload is disabled")
829    test_spurios_extack(sim, obj, True, "TC offload is disabled")
830
831    sim.set_ethtool_tc_offloads(True)
832
833    sim.tc_flush_filters()
834
835    start_test("Test TC offloads work...")
836    ret, _, err = sim.cls_bpf_add_filter(obj, verbose=True, skip_sw=True,
837                                         fail=False, include_stderr=True)
838    fail(ret != 0, "TC filter did not load with TC offloads enabled")
839    check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
840
841    start_test("Test TC offload basics...")
842    dfs = sim.dfs_get_bound_progs(expected=1)
843    progs = bpftool_prog_list(expected=1)
844    ingress = sim.tc_show_ingress(expected=1)
845
846    dprog = dfs[0]
847    prog = progs[0]
848    fltr = ingress[0]
849    fail(fltr["skip_hw"], "TC does reports 'skip_hw' on offloaded filter")
850    fail(not fltr["in_hw"], "TC does not report 'in_hw' for offloaded filter")
851    fail(not fltr["skip_sw"], "TC does not report 'skip_sw' back")
852
853    start_test("Test TC offload is device-bound...")
854    fail(str(prog["id"]) != fltr["id"], "Program IDs don't match")
855    fail(prog["tag"] != fltr["tag"], "Program tags don't match")
856    fail(fltr["id"] != dprog["id"], "Program IDs don't match")
857    fail(dprog["state"] != "xlated", "Offloaded program state not translated")
858    fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
859
860    start_test("Test disabling TC offloads is rejected while filters installed...")
861    ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
862    fail(ret == 0, "Driver should refuse to disable TC offloads with filters installed...")
863
864    start_test("Test qdisc removal frees things...")
865    sim.tc_flush_filters()
866    sim.tc_show_ingress(expected=0)
867
868    start_test("Test disabling TC offloads is OK without filters...")
869    ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
870    fail(ret != 0,
871         "Driver refused to disable TC offloads without filters installed...")
872
873    sim.set_ethtool_tc_offloads(True)
874
875    start_test("Test destroying device gets rid of TC filters...")
876    sim.cls_bpf_add_filter(obj, skip_sw=True)
877    sim.remove()
878    bpftool_prog_list_wait(expected=0)
879
880    sim = NetdevSim()
881    sim.set_ethtool_tc_offloads(True)
882
883    start_test("Test destroying device gets rid of XDP...")
884    sim.set_xdp(obj, "offload")
885    sim.remove()
886    bpftool_prog_list_wait(expected=0)
887
888    sim = NetdevSim()
889    sim.set_ethtool_tc_offloads(True)
890
891    start_test("Test XDP prog reporting...")
892    sim.set_xdp(obj, "drv")
893    ipl = sim.ip_link_show(xdp=True)
894    progs = bpftool_prog_list(expected=1)
895    fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
896         "Loaded program has wrong ID")
897
898    start_test("Test XDP prog replace without force...")
899    ret, _ = sim.set_xdp(obj, "drv", fail=False)
900    fail(ret == 0, "Replaced XDP program without -force")
901    sim.wait_for_flush(total=1)
902
903    start_test("Test XDP prog replace with force...")
904    ret, _ = sim.set_xdp(obj, "drv", force=True, fail=False)
905    fail(ret != 0, "Could not replace XDP program with -force")
906    bpftool_prog_list_wait(expected=1)
907    ipl = sim.ip_link_show(xdp=True)
908    progs = bpftool_prog_list(expected=1)
909    fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
910         "Loaded program has wrong ID")
911    fail("dev" in progs[0].keys(),
912         "Device parameters reported for non-offloaded program")
913
914    start_test("Test XDP prog replace with bad flags...")
915    ret, _, err = sim.set_xdp(obj, "generic", force=True,
916                              fail=False, include_stderr=True)
917    fail(ret == 0, "Replaced XDP program with a program in different mode")
918    check_extack(err,
919                 "native and generic XDP can't be active at the same time.",
920                 args)
921    ret, _, err = sim.set_xdp(obj, "", force=True,
922                              fail=False, include_stderr=True)
923    fail(ret == 0, "Replaced XDP program with a program in different mode")
924    check_extack(err, "program loaded with different flags.", args)
925
926    start_test("Test XDP prog remove with bad flags...")
927    ret, _, err = sim.unset_xdp("", force=True,
928                                fail=False, include_stderr=True)
929    fail(ret == 0, "Removed program with a bad mode")
930    check_extack(err, "program loaded with different flags.", args)
931
932    start_test("Test MTU restrictions...")
933    ret, _ = sim.set_mtu(9000, fail=False)
934    fail(ret == 0,
935         "Driver should refuse to increase MTU to 9000 with XDP loaded...")
936    sim.unset_xdp("drv")
937    bpftool_prog_list_wait(expected=0)
938    sim.set_mtu(9000)
939    ret, _, err = sim.set_xdp(obj, "drv", fail=False, include_stderr=True)
940    fail(ret == 0, "Driver should refuse to load program with MTU of 9000...")
941    check_extack_nsim(err, "MTU too large w/ XDP enabled.", args)
942    sim.set_mtu(1500)
943
944    sim.wait_for_flush()
945    start_test("Test non-offload XDP attaching to HW...")
946    bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/nooffload")
947    nooffload = bpf_pinned("/sys/fs/bpf/nooffload")
948    ret, _, err = sim.set_xdp(nooffload, "offload",
949                              fail=False, include_stderr=True)
950    fail(ret == 0, "attached non-offloaded XDP program to HW")
951    check_extack_nsim(err, "xdpoffload of non-bound program.", args)
952    rm("/sys/fs/bpf/nooffload")
953
954    start_test("Test offload XDP attaching to drv...")
955    bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",
956                      dev=sim['ifname'])
957    offload = bpf_pinned("/sys/fs/bpf/offload")
958    ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)
959    fail(ret == 0, "attached offloaded XDP program to drv")
960    check_extack(err, "using device-bound program without HW_MODE flag is not supported.", args)
961    rm("/sys/fs/bpf/offload")
962    sim.wait_for_flush()
963
964    start_test("Test XDP offload...")
965    _, _, err = sim.set_xdp(obj, "offload", verbose=True, include_stderr=True)
966    ipl = sim.ip_link_show(xdp=True)
967    link_xdp = ipl["xdp"]["prog"]
968    progs = bpftool_prog_list(expected=1)
969    prog = progs[0]
970    fail(link_xdp["id"] != prog["id"], "Loaded program has wrong ID")
971    check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
972
973    start_test("Test XDP offload is device bound...")
974    dfs = sim.dfs_get_bound_progs(expected=1)
975    dprog = dfs[0]
976
977    fail(prog["id"] != link_xdp["id"], "Program IDs don't match")
978    fail(prog["tag"] != link_xdp["tag"], "Program tags don't match")
979    fail(str(link_xdp["id"]) != dprog["id"], "Program IDs don't match")
980    fail(dprog["state"] != "xlated", "Offloaded program state not translated")
981    fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
982
983    start_test("Test removing XDP program many times...")
984    sim.unset_xdp("offload")
985    sim.unset_xdp("offload")
986    sim.unset_xdp("drv")
987    sim.unset_xdp("drv")
988    sim.unset_xdp("")
989    sim.unset_xdp("")
990    bpftool_prog_list_wait(expected=0)
991
992    start_test("Test attempt to use a program for a wrong device...")
993    sim2 = NetdevSim()
994    sim2.set_xdp(obj, "offload")
995    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
996
997    ret, _, err = sim.set_xdp(pinned, "offload",
998                              fail=False, include_stderr=True)
999    fail(ret == 0, "Pinned program loaded for a different device accepted")
1000    check_extack_nsim(err, "program bound to different dev.", args)
1001    sim2.remove()
1002    ret, _, err = sim.set_xdp(pinned, "offload",
1003                              fail=False, include_stderr=True)
1004    fail(ret == 0, "Pinned program loaded for a removed device accepted")
1005    check_extack_nsim(err, "xdpoffload of non-bound program.", args)
1006    rm(pin_file)
1007    bpftool_prog_list_wait(expected=0)
1008
1009    sim = test_multi_prog(sim, obj, "", 1)
1010    sim = test_multi_prog(sim, obj, "drv", 1)
1011    sim = test_multi_prog(sim, obj, "generic", 2)
1012
1013    start_test("Test mixing of TC and XDP...")
1014    sim.tc_add_ingress()
1015    sim.set_xdp(obj, "offload")
1016    ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
1017                                         fail=False, include_stderr=True)
1018    fail(ret == 0, "Loading TC when XDP active should fail")
1019    check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
1020    sim.unset_xdp("offload")
1021    sim.wait_for_flush()
1022
1023    sim.cls_bpf_add_filter(obj, skip_sw=True)
1024    ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
1025    fail(ret == 0, "Loading XDP when TC active should fail")
1026    check_extack_nsim(err, "TC program is already loaded.", args)
1027
1028    start_test("Test binding TC from pinned...")
1029    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
1030    sim.tc_flush_filters(bound=1, total=1)
1031    sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True)
1032    sim.tc_flush_filters(bound=1, total=1)
1033
1034    start_test("Test binding XDP from pinned...")
1035    sim.set_xdp(obj, "offload")
1036    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp2", idx=1)
1037
1038    sim.set_xdp(pinned, "offload", force=True)
1039    sim.unset_xdp("offload")
1040    sim.set_xdp(pinned, "offload", force=True)
1041    sim.unset_xdp("offload")
1042
1043    start_test("Test offload of wrong type fails...")
1044    ret, _ = sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True, fail=False)
1045    fail(ret == 0, "Managed to attach XDP program to TC")
1046
1047    start_test("Test asking for TC offload of two filters...")
1048    sim.cls_bpf_add_filter(obj, da=True, skip_sw=True)
1049    ret, _, err = sim.cls_bpf_add_filter(obj, da=True, skip_sw=True,
1050                                         fail=False, include_stderr=True)
1051    fail(ret == 0, "Managed to offload two TC filters at the same time")
1052    check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
1053
1054    sim.tc_flush_filters(bound=2, total=2)
1055
1056    start_test("Test if netdev removal waits for translation...")
1057    delay_msec = 500
1058    sim.dfs["bpf_bind_verifier_delay"] = delay_msec
1059    start = time.time()
1060    cmd_line = "tc filter add dev %s ingress bpf %s da skip_sw" % \
1061               (sim['ifname'], obj)
1062    tc_proc = cmd(cmd_line, background=True, fail=False)
1063    # Wait for the verifier to start
1064    while sim.dfs_num_bound_progs() <= 2:
1065        pass
1066    sim.remove()
1067    end = time.time()
1068    ret, _ = cmd_result(tc_proc, fail=False)
1069    time_diff = end - start
1070    log("Time", "start:\t%s\nend:\t%s\ndiff:\t%s" % (start, end, time_diff))
1071
1072    fail(ret == 0, "Managed to load TC filter on a unregistering device")
1073    delay_sec = delay_msec * 0.001
1074    fail(time_diff < delay_sec, "Removal process took %s, expected %s" %
1075         (time_diff, delay_sec))
1076
1077    # Remove all pinned files and reinstantiate the netdev
1078    clean_up()
1079    bpftool_prog_list_wait(expected=0)
1080
1081    sim = NetdevSim()
1082    map_obj = bpf_obj("sample_map_ret0.o")
1083    start_test("Test loading program with maps...")
1084    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1085
1086    start_test("Test bpftool bound info reporting (own ns)...")
1087    check_dev_info(False, "")
1088
1089    start_test("Test bpftool bound info reporting (other ns)...")
1090    ns = mknetns()
1091    sim.set_ns(ns)
1092    check_dev_info(True, "")
1093
1094    start_test("Test bpftool bound info reporting (remote ns)...")
1095    check_dev_info(False, ns)
1096
1097    start_test("Test bpftool bound info reporting (back to own ns)...")
1098    sim.set_ns("")
1099    check_dev_info(False, "")
1100
1101    prog_file, _ = pin_prog("/sys/fs/bpf/tmp_prog")
1102    map_file, _ = pin_map("/sys/fs/bpf/tmp_map", idx=1, expected=2)
1103    sim.remove()
1104
1105    start_test("Test bpftool bound info reporting (removed dev)...")
1106    check_dev_info_removed(prog_file=prog_file, map_file=map_file)
1107
1108    # Remove all pinned files and reinstantiate the netdev
1109    clean_up()
1110    bpftool_prog_list_wait(expected=0)
1111
1112    sim = NetdevSim()
1113
1114    start_test("Test map update (no flags)...")
1115    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1116    maps = bpftool_map_list(expected=2)
1117    array = maps[0] if maps[0]["type"] == "array" else maps[1]
1118    htab = maps[0] if maps[0]["type"] == "hash" else maps[1]
1119    for m in maps:
1120        for i in range(2):
1121            bpftool("map update id %d key %s value %s" %
1122                    (m["id"], int2str("I", i), int2str("Q", i * 3)))
1123
1124    for m in maps:
1125        ret, _ = bpftool("map update id %d key %s value %s" %
1126                         (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1127                         fail=False)
1128        fail(ret == 0, "added too many entries")
1129
1130    start_test("Test map update (exists)...")
1131    for m in maps:
1132        for i in range(2):
1133            bpftool("map update id %d key %s value %s exist" %
1134                    (m["id"], int2str("I", i), int2str("Q", i * 3)))
1135
1136    for m in maps:
1137        ret, err = bpftool("map update id %d key %s value %s exist" %
1138                           (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1139                           fail=False)
1140        fail(ret == 0, "updated non-existing key")
1141        fail(err["error"].find("No such file or directory") == -1,
1142             "expected ENOENT, error is '%s'" % (err["error"]))
1143
1144    start_test("Test map update (noexist)...")
1145    for m in maps:
1146        for i in range(2):
1147            ret, err = bpftool("map update id %d key %s value %s noexist" %
1148                               (m["id"], int2str("I", i), int2str("Q", i * 3)),
1149                               fail=False)
1150        fail(ret == 0, "updated existing key")
1151        fail(err["error"].find("File exists") == -1,
1152             "expected EEXIST, error is '%s'" % (err["error"]))
1153
1154    start_test("Test map dump...")
1155    for m in maps:
1156        _, entries = bpftool("map dump id %d" % (m["id"]))
1157        for i in range(2):
1158            key = str2int(entries[i]["key"])
1159            fail(key != i, "expected key %d, got %d" % (key, i))
1160            val = str2int(entries[i]["value"])
1161            fail(val != i * 3, "expected value %d, got %d" % (val, i * 3))
1162
1163    start_test("Test map getnext...")
1164    for m in maps:
1165        _, entry = bpftool("map getnext id %d" % (m["id"]))
1166        key = str2int(entry["next_key"])
1167        fail(key != 0, "next key %d, expected %d" % (key, 0))
1168        _, entry = bpftool("map getnext id %d key %s" %
1169                           (m["id"], int2str("I", 0)))
1170        key = str2int(entry["next_key"])
1171        fail(key != 1, "next key %d, expected %d" % (key, 1))
1172        ret, err = bpftool("map getnext id %d key %s" %
1173                           (m["id"], int2str("I", 1)), fail=False)
1174        fail(ret == 0, "got next key past the end of map")
1175        fail(err["error"].find("No such file or directory") == -1,
1176             "expected ENOENT, error is '%s'" % (err["error"]))
1177
1178    start_test("Test map delete (htab)...")
1179    for i in range(2):
1180        bpftool("map delete id %d key %s" % (htab["id"], int2str("I", i)))
1181
1182    start_test("Test map delete (array)...")
1183    for i in range(2):
1184        ret, err = bpftool("map delete id %d key %s" %
1185                           (htab["id"], int2str("I", i)), fail=False)
1186        fail(ret == 0, "removed entry from an array")
1187        fail(err["error"].find("No such file or directory") == -1,
1188             "expected ENOENT, error is '%s'" % (err["error"]))
1189
1190    start_test("Test map remove...")
1191    sim.unset_xdp("offload")
1192    bpftool_map_list_wait(expected=0)
1193    sim.remove()
1194
1195    sim = NetdevSim()
1196    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1197    sim.remove()
1198    bpftool_map_list_wait(expected=0)
1199
1200    start_test("Test map creation fail path...")
1201    sim = NetdevSim()
1202    sim.dfs["bpf_map_accept"] = "N"
1203    ret, _ = sim.set_xdp(map_obj, "offload", JSON=False, fail=False)
1204    fail(ret == 0,
1205         "netdevsim didn't refuse to create a map with offload disabled")
1206
1207    sim.remove()
1208
1209    start_test("Test multi-dev ASIC program reuse...")
1210    simA = NetdevSim()
1211    simB1 = NetdevSim()
1212    simB2 = NetdevSim(link=simB1)
1213    simB3 = NetdevSim(link=simB1)
1214    sims = (simA, simB1, simB2, simB3)
1215    simB = (simB1, simB2, simB3)
1216
1217    bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA",
1218                      dev=simA['ifname'])
1219    progA = bpf_pinned("/sys/fs/bpf/nsimA")
1220    bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB",
1221                      dev=simB1['ifname'])
1222    progB = bpf_pinned("/sys/fs/bpf/nsimB")
1223
1224    simA.set_xdp(progA, "offload", JSON=False)
1225    for d in simB:
1226        d.set_xdp(progB, "offload", JSON=False)
1227
1228    start_test("Test multi-dev ASIC cross-dev replace...")
1229    ret, _ = simA.set_xdp(progB, "offload", force=True, JSON=False, fail=False)
1230    fail(ret == 0, "cross-ASIC program allowed")
1231    for d in simB:
1232        ret, _ = d.set_xdp(progA, "offload", force=True, JSON=False, fail=False)
1233        fail(ret == 0, "cross-ASIC program allowed")
1234
1235    start_test("Test multi-dev ASIC cross-dev install...")
1236    for d in sims:
1237        d.unset_xdp("offload")
1238
1239    ret, _, err = simA.set_xdp(progB, "offload", force=True, JSON=False,
1240                               fail=False, include_stderr=True)
1241    fail(ret == 0, "cross-ASIC program allowed")
1242    check_extack_nsim(err, "program bound to different dev.", args)
1243    for d in simB:
1244        ret, _, err = d.set_xdp(progA, "offload", force=True, JSON=False,
1245                                fail=False, include_stderr=True)
1246        fail(ret == 0, "cross-ASIC program allowed")
1247        check_extack_nsim(err, "program bound to different dev.", args)
1248
1249    start_test("Test multi-dev ASIC cross-dev map reuse...")
1250
1251    mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]
1252    mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]
1253
1254    ret, _ = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
1255                               dev=simB3['ifname'],
1256                               maps=["idx 0 id %d" % (mapB)],
1257                               fail=False)
1258    fail(ret != 0, "couldn't reuse a map on the same ASIC")
1259    rm("/sys/fs/bpf/nsimB_")
1260
1261    ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA_",
1262                                    dev=simA['ifname'],
1263                                    maps=["idx 0 id %d" % (mapB)],
1264                                    fail=False, include_stderr=True)
1265    fail(ret == 0, "could reuse a map on a different ASIC")
1266    fail(err.count("offload device mismatch between prog and map") == 0,
1267         "error message missing for cross-ASIC map")
1268
1269    ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
1270                                    dev=simB1['ifname'],
1271                                    maps=["idx 0 id %d" % (mapA)],
1272                                    fail=False, include_stderr=True)
1273    fail(ret == 0, "could reuse a map on a different ASIC")
1274    fail(err.count("offload device mismatch between prog and map") == 0,
1275         "error message missing for cross-ASIC map")
1276
1277    start_test("Test multi-dev ASIC cross-dev destruction...")
1278    bpftool_prog_list_wait(expected=2)
1279
1280    simA.remove()
1281    bpftool_prog_list_wait(expected=1)
1282
1283    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1284    fail(ifnameB != simB1['ifname'], "program not bound to originial device")
1285    simB1.remove()
1286    bpftool_prog_list_wait(expected=1)
1287
1288    start_test("Test multi-dev ASIC cross-dev destruction - move...")
1289    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1290    fail(ifnameB not in (simB2['ifname'], simB3['ifname']),
1291         "program not bound to remaining devices")
1292
1293    simB2.remove()
1294    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1295    fail(ifnameB != simB3['ifname'], "program not bound to remaining device")
1296
1297    simB3.remove()
1298    bpftool_prog_list_wait(expected=0)
1299
1300    start_test("Test multi-dev ASIC cross-dev destruction - orphaned...")
1301    ret, out = bpftool("prog show %s" % (progB), fail=False)
1302    fail(ret == 0, "got information about orphaned program")
1303    fail("error" not in out, "no error reported for get info on orphaned")
1304    fail(out["error"] != "can't get prog info: No such device",
1305         "wrong error for get info on orphaned")
1306
1307    print("%s: OK" % (os.path.basename(__file__)))
1308
1309finally:
1310    log("Clean up...", "", level=1)
1311    log_level_inc()
1312    clean_up()
1313