gh-74772: Adds autospec argument for Mock / MagicMock / AsyncMock

[claudiubelu]

Mock can accept a spec object / class as an argument, making sure that accessing attributes that do not exist in the spec will cause an AttributeError to be raised, but there is no guarantee that the spec's methods signatures are respected in any way. This creates the possibility to have faulty code with passing unittests and assertions.

Example:

from unittest import mock

class Something(object):
    def foo(self, a, b, c, d):
        pass

m = mock.Mock(spec=Something)
m.foo()

Adds the autospec argument to NonCallableMock and its mock_add_spec method. These changes are inherited by Mock, MagicMock, AsyncMock.

Passes the spec's attribute with the same name to the child mock (spec-ing the child), if the mock's autospec is True.

Sets _mock_check_sig only if the given spec is used via autospec, so that passing plain spec / spec_set does not change existing signature-checking behavior.

Adds unit tests to validate the fact that the autospecced method signatures are enforced.

https://bugs.python.org/issue30587
#74772

• Issue: Mock with spec object does not ensure method call signatures #74772

[claudiubelu]

A few things to note about this PR:

• The commit was essentially cherry-picked on the latest version from: bpo-30587: Adds signature checking for mock autospec object method calls #1982 (the PR was closed, and I could not reopen it)
• Added unit tests for async mocks.
• Note that before this PR, the autospec argument did nothing for Mock / MagicMock, yet there seems to be misconception that it actually exists: https://github.com/search?q=%2FMock%5C(autospec%3D%2F%20language%3APython%20&type=code . Because that argument does not (yet) exist, it is strictly worse than spec, and may hide potential issues as false positive tests.
• Interestingly, it seems some people have been / tried using the old PR mentioned above: https://discuss.python.org/t/how-can-i-get-a-patch-from-a-pr/14309 . This shows some interest from the community in having this argument added.

[claudiubelu]

An interesting note regarding mock.Mock(autospec=Thing) that slipped my mind is that it's significantly faster than mock.create_autospec(Thing) under most scenarios. Here are some numbers from the scripts [1][2] below:

./python benchmark/bench_flat.py
Sanity checks passed: invalid signatures raise TypeError under both autospec styles.

N=1000 iterations
create_autospec(BigClass) + call 10/10 methods:  17.145 ms/call
MagicMock(autospec=) + call  1/10 methods: 1.187 ms/call  (14.4x faster)
MagicMock(autospec=) + call  3/10 methods: 2.596 ms/call  (6.6x faster)
MagicMock(autospec=) + call  5/10 methods: 4.127 ms/call  (4.2x faster)
MagicMock(autospec=) + call 10/10 methods: 7.802 ms/call (2.2x faster)

./python benchmark/bench_inheritance.py
Sanity checks passed: invalid signatures raise TypeError under both autospec styles.

N=1000 iterations
Class        depth         own inherited | create_autospec  Mock(autospec=1)  speedup
-------------------------------------------------------------------------------------
Flat             1          10         0 |         16.430ms            1.123ms    14.6x
Deep2            2           5         5 |         16.072ms            1.281ms    12.5x
Deep4            4           1         9 |         16.785ms            1.230ms    13.6x
Wide+Deep        4           5        15 |         33.380ms            1.170ms    28.5x

[1] benchmark/bench_flat.py:

"""Benchmark: create_autospec vs Mock(autospec=...) on a flat 10-method class.

Run with:
    ./python benchmark/bench_flat.py
"""

import timeit
from typing import Any
from unittest import mock


class BigClass:
    def method_a(self, x: int, y: str) -> None: pass
    def method_b(self, a: Any, b: Any, c: Any = None) -> None: pass
    def method_c(self) -> None: pass
    def method_d(self, p: Any, q: Any, r: Any, s: int = 1, t: int = 2) -> None: pass
    def method_e(self, *args: Any, **kwargs: Any) -> None: pass
    def method_f(self, x: float) -> float: pass
    def method_g(self) -> None: pass
    def method_h(self, a: Any, b: Any) -> None: pass
    def method_i(self) -> None: pass
    def method_j(self, x: Any, y: Any, z: Any) -> None: pass


def call_valid(m: Any) -> None:
    m.method_a(1, "y")
    m.method_b(1, 2)
    m.method_c()
    m.method_d(1, 2, 3)
    m.method_e(1, 2, key="value")
    m.method_f(1.0)
    m.method_g()
    m.method_h(1, 2)
    m.method_i()
    m.method_j(1, 2, 3)


def sanity_check(m: Any) -> None:
    call_valid(m)

    try:
        m.method_a()
    except TypeError:
        pass
    else:
        raise AssertionError("method_a() with missing required arguments should have raised TypeError")

    try:
        m.method_j(1, 2)
    except TypeError:
        pass
    else:
        raise AssertionError("method_j() with too few arguments should have raised TypeError")


sanity_check(mock.create_autospec(BigClass))
sanity_check(mock.MagicMock(autospec=BigClass))

print("Sanity checks passed: invalid signatures raise TypeError under both autospec styles.\n")


N = 1000

t1 = timeit.timeit(lambda: call_valid(mock.create_autospec(BigClass)), number=N)


def autospec_call_1() -> None:
    m = mock.MagicMock(autospec=BigClass)
    m.method_a(1, "y")


def autospec_call_3() -> None:
    m = mock.MagicMock(autospec=BigClass)
    m.method_a(1, "y")
    m.method_b(1, 2)
    m.method_c()


def autospec_call_5() -> None:
    m = mock.MagicMock(autospec=BigClass)
    m.method_a(1, "y")
    m.method_b(1, 2)
    m.method_c()
    m.method_d(1, 2, 3)
    m.method_e(1, 2, key="value")


def autospec_call_10() -> None:
    m = mock.MagicMock(autospec=BigClass)
    call_valid(m)


t_1  = timeit.timeit(autospec_call_1,  number=N)
t_3  = timeit.timeit(autospec_call_3,  number=N)
t_5  = timeit.timeit(autospec_call_5,  number=N)
t_10 = timeit.timeit(autospec_call_10, number=N)


ms    = lambda t: f"{t / N * 1000:.3f} ms/call"
ratio = lambda t: f"{t1 / t:.1f}x faster" if t1 > t else f"{t / t1:.1f}x SLOWER"


print(f"N={N} iterations")
print(f"create_autospec(BigClass) + call 10/10 methods:  {ms(t1)}")
print(f"MagicMock(autospec=) + call  1/10 methods: {ms(t_1)}  ({ratio(t_1)})")
print(f"MagicMock(autospec=) + call  3/10 methods: {ms(t_3)}  ({ratio(t_3)})")
print(f"MagicMock(autospec=) + call  5/10 methods: {ms(t_5)}  ({ratio(t_5)})")
print(f"MagicMock(autospec=) + call 10/10 methods: {ms(t_10)} ({ratio(t_10)})")

[2] benchmark/bench_inheritance.py:

"""Benchmark: create_autospec vs Mock(autospec=...) across class hierarchies
of varying depth.

Run with:
    ./python benchmark/bench_inheritance.py
"""

import timeit
from unittest import mock


# Flat: 10 methods, no inheritance
class Flat:
    def m0(self) -> None: pass
    def m1(self) -> None: pass
    def m2(self) -> None: pass
    def m3(self) -> None: pass
    def m4(self) -> None: pass
    def m5(self) -> None: pass
    def m6(self) -> None: pass
    def m7(self) -> None: pass
    def m8(self) -> None: pass
    def m9(self) -> None: pass


# 2-level deep: 10 methods total (5 per level)
class Base2:
    def b0(self) -> None: pass
    def b1(self) -> None: pass
    def b2(self) -> None: pass
    def b3(self) -> None: pass
    def b4(self) -> None: pass


class Deep2(Base2):
    def d0(self) -> None: pass
    def d1(self) -> None: pass
    def d2(self) -> None: pass
    def d3(self) -> None: pass
    def d4(self) -> None: pass


# 4-level deep: 10 methods total (3+3+3+1)
class Base4A:
    def a0(self) -> None: pass
    def a1(self) -> None: pass
    def a2(self) -> None: pass


class Base4B(Base4A):
    def b0(self) -> None: pass
    def b1(self) -> None: pass
    def b2(self) -> None: pass


class Base4C(Base4B):
    def c0(self) -> None: pass
    def c1(self) -> None: pass
    def c2(self) -> None: pass


class Deep4(Base4C):
    def d0(self) -> None: pass


# Wide+deep: 4 levels, 5 methods per level = 20 methods total
class W0:
    def w0m0(self) -> None: pass
    def w0m1(self) -> None: pass
    def w0m2(self) -> None: pass
    def w0m3(self) -> None: pass
    def w0m4(self) -> None: pass


class W1(W0):
    def w1m0(self) -> None: pass
    def w1m1(self) -> None: pass
    def w1m2(self) -> None: pass
    def w1m3(self) -> None: pass
    def w1m4(self) -> None: pass


class W2(W1):
    def w2m0(self) -> None: pass
    def w2m1(self) -> None: pass
    def w2m2(self) -> None: pass
    def w2m3(self) -> None: pass
    def w2m4(self) -> None: pass


class W3(W2):
    def w3m0(self) -> None: pass
    def w3m1(self) -> None: pass
    def w3m2(self) -> None: pass
    def w3m3(self) -> None: pass
    def w3m4(self) -> None: pass


def sanity_check(cls: type) -> None:
    first_method = next(n for n in vars(cls) if not n.startswith("_"))

    for m in (mock.create_autospec(cls), mock.MagicMock(autospec=cls)):
        getattr(m, first_method)()  # valid call, no arguments, should not raise

        try:
            getattr(m, first_method)(1)  # extra unexpected positional argument
        except TypeError:
            pass
        else:
            raise AssertionError(
                f"{cls.__name__}.{first_method}(1) with an unexpected argument "
                "should have raised TypeError"
            )


for _cls in (Flat, Deep2, Deep4, W3):
    sanity_check(_cls)


print("Sanity checks passed: invalid signatures raise TypeError under both autospec styles.\n")


N = 1000


def bench(cls: type) -> tuple[float, float]:
    first_method = next(n for n in vars(cls) if not n.startswith("_"))

    t_eager = timeit.timeit(
        lambda: getattr(mock.create_autospec(cls), first_method)(), number=N
    )

    def autospec_1() -> None:
        m = mock.MagicMock(autospec=cls)
        getattr(m, first_method)()

    t_autospec1 = timeit.timeit(autospec_1, number=N)

    return t_eager / N * 1000, t_autospec1 / N * 1000


def methods_own(cls: type) -> int:
    return sum(1 for k, v in cls.__dict__.items() if callable(v) and not k.startswith("_"))


def methods_inherited(cls: type) -> int:
    return len([m for m in dir(cls) if not m.startswith("_") and m not in vars(cls) and callable(getattr(cls, m))])


header = "{:<12} {:>5} {:>11} {:>9} | {:>15} {:>17} {:>8}".format(
    "Class", "depth", "own", "inherited", "create_autospec", "Mock(autospec=1)", "speedup"
)

print(f"N={N} iterations")
print(header)
print("-" * len(header))
for name, cls in [("Flat", Flat), ("Deep2", Deep2), ("Deep4", Deep4), ("Wide+Deep", W3)]:
    depth = len(cls.__mro__) - 1
    own = methods_own(cls)
    inh = methods_inherited(cls)
    eager_ms, autospec_ms = bench(cls)

    row = "{:<12} {:>5} {:>11} {:>9} | {:>14.3f}ms {:>16.3f}ms {:>7.1f}x".format(
        name, depth, own, inh, eager_ms, autospec_ms, eager_ms / autospec_ms
    )
    print(row)

[claudiubelu]

/cc @ambv Hello, we talked a bit about Mock(autospec=Thing) recently.