Writing Unit Tests¶

Introduction¶

Like many software projects, Salt has two broad-based testing approaches -- integration testing and unit testing. While integration testing focuses on the interaction between components in a sandboxed environment, unit testing focuses on the singular implementation of individual functions.

Unit tests should be used specifically to test a function's logic. Unit tests rely on mocking external resources.

While unit tests are good for ensuring consistent results, they are most useful when they do not require more than a few mocks. Effort should be made to mock as many external resources as possible. This effort is encouraged, but not required. Sometimes the isolation provided by completely mocking the external dependencies is not worth the effort of mocking those dependencies.

In these cases, requiring an external library to be installed on the system before running the test file is a useful way to strike this balance. For example, the unit tests for the MySQL execution module require the presence of the MySQL python bindings on the system running the test file before proceeding to run the tests.

Overly detailed mocking can also result in decreased test readability and brittleness as the tests are more likely to fail when the code or its dependencies legitimately change. In these cases, it is better to add dependencies to the test runner dependency state.

Preparing to Write a Unit Test¶

This guide assumes that your Salt development environment is already configured and that you have a basic understanding of contributing to the Salt codebase. If you're unfamiliar with either of these topics, please refer to the Installing Salt for Development and the Contributing pages, respectively.

This documentation also assumes that you have an understanding of how to run Salt's test suite, including running the unit test subsection, running the unit tests without testing daemons to speed up development wait times, and running a unit test file, class, or individual test.

Best Practices¶

Unit tests should be written to the following specifications.

import pytest
import salt.modules.somemodule as somemodule


@pytest.fixture
def configure_loader_modules():
    """
    This fixture should return a dictionary which is what's going to be used to
    patch and mock Salt's loader
    """
    return {somemodule: {"__opts__": {"test": True}}}

import pytest
import salt.beacons.sensehat as sensehat
from tests.support.mock import MagicMock


@pytest.fixture
def configure_loader_modules():
    return {
        sensehat: {
            "__salt__": {
                "sensehat.get_humidity": MagicMock(return_value=80),
                "sensehat.get_temperature": MagicMock(return_value=30),
                "sensehat.get_pressure": MagicMock(return_value=1500),
            },
        }
    }


def test_non_list_config():
    config = {}

    ret = sensehat.validate(config)

    assert ret == (False, "Configuration for sensehat beacon must be a list.")


def test_empty_config():
    config = [{}]

    ret = sensehat.validate(config)

    assert ret == (False, "Configuration for sensehat beacon requires sensors.")


def test_sensehat_humidity_match():

    config = [{"sensors": {"humidity": "70%"}}]

    ret = sensehat.validate(config)
    assert ret == (True, "Valid beacon configuration")

    ret = sensehat.beacon(config)
    assert ret == [{"tag": "sensehat/humidity", "humidity": 80}]


def test_sensehat_temperature_match():

    config = [{"sensors": {"temperature": 20}}]

    ret = sensehat.validate(config)
    assert ret == (True, "Valid beacon configuration")

    ret = sensehat.beacon(config)
    assert ret == [{"tag": "sensehat/temperature", "temperature": 30}]


def test_sensehat_temperature_match_range():

    config = [{"sensors": {"temperature": [20, 29]}}]

    ret = sensehat.validate(config)
    assert ret == (True, "Valid beacon configuration")

    ret = sensehat.beacon(config)
    assert ret == [{"tag": "sensehat/temperature", "temperature": 30}]


def test_sensehat_pressure_match():

    config = [{"sensors": {"pressure": "1400"}}]

    ret = sensehat.validate(config)
    assert ret == (True, "Valid beacon configuration")

    ret = sensehat.beacon(config)
    assert ret == [{"tag": "sensehat/pressure", "pressure": 1500}]


def test_sensehat_no_match():

    config = [{"sensors": {"pressure": "1600"}}]

    ret = sensehat.validate(config)
    assert ret == (True, "Valid beacon configuration")

    ret = sensehat.beacon(config)
    assert ret == []

from tests.support.mock import patch, mock_open

import salt.modules.mymod as mymod


def test_something():
    fopen_mock = mock_open(read_data="foo\nbar\nbaz\n")
    with patch("salt.utils.files.fopen", fopen_mock):
        result = mymod.myfunc()
        assert result is True

import textwrap

from tests.support.mock import patch, mock_open

import salt.modules.mymod as mymod


def test_something():
    contents = {
        "/etc/foo.conf": textwrap.dedent(
            """\
            foo
            bar
            baz
            """
        ),
        "/etc/b*.conf": textwrap.dedent(
            """\
            one
            two
            three
            """
        ),
    }
    fopen_mock = mock_open(read_data=contents)
    with patch("salt.utils.files.fopen", fopen_mock):
        result = mymod.myfunc()
        assert result is True

mock_open(read_data="foo\n")
mock_open(read_data={"*": "foo\n"})

contents = OrderedDict()
contents["/etc/bar.conf"] = "foo\nbar\nbaz\n"
contents["/etc/b*.conf"] = IOError(errno.EACCES, "Permission denied")
contents["*"] = 'This is a fallback for files not beginning with "/etc/b"\n'
fopen_mock = mock_open(read_data=contents)

import errno

from tests.support.mock import patch, mock_open

import salt.modules.mymod as mymod


def test_something():
    exc = IOError(errno.EACCES, "Permission denied")
    fopen_mock = mock_open(read_data=exc)
    with patch("salt.utils.files.fopen", fopen_mock):
        mymod.myfunc()

import errno
import textwrap

from tests.support.mock import patch, mock_open

import salt.modules.mymod as mymod


def test_something():
    contents = {
        "/etc/foo.conf": [
            textwrap.dedent(
                """\
                foo
                bar
                """
            ),
            textwrap.dedent(
                """\
                foo
                bar
                baz
                """
            ),
        ],
        "/etc/b*.conf": [
            IOError(errno.ENOENT, "No such file or directory"),
            textwrap.dedent(
                """\
                one
                two
                three
                """
            ),
        ],
    }
    fopen_mock = mock_open(read_data=contents)
    with patch("salt.utils.files.fopen", fopen_mock):
        result = mymod.myfunc()
        assert result is True

fopen_mock = mock_open(read_data="foo\nbar\nbaz\n")
with patch("salt.utils.files.fopen", fopen_mock) as m_open:
    # do testing here
    ...
    ...

with patch("salt.utils.files.fopen", mock_open(read_data=contents)) as m_open:
    # Run the code you are unit testing
    mymod.myfunc()
    # Check that only the expected file was opened, and that it was opened
    # only once.
    assert m_open.call_count == 1
    assert list(m_open.filehandles) == ["/etc/foo.conf"]
    # "opens" will be a list of all the mocked filehandles opened
    opens = m_open.filehandles["/etc/foo.conf"]
    # Check that we wrote the expected lines ("expected" here is assumed to
    # be a list of strings)
    assert opens[0].write_calls == expected

with patch("salt.utils.files.fopen", mock_open(read_data=contents)) as m_open:
    # Run the code you are unit testing
    mymod.myfunc()
    # Check that .readlines() was called (remember, it's a Mock)
    m_open.filehandles["/etc/foo.conf"][0].readlines.assert_called()

with patch("salt.utils.files.fopen", mock_open(read_data=contents)) as m_open:
    # Run the code you are unit testing
    mymod.myfunc()
    # Check that we read the file and also wrote to it
    m_open.filehandles["/etc/foo.conf"][0].read.assert_called_once()
    m_open.filehandles["/etc/foo.conf"][1].writelines.assert_called_once()

import pytest

from tests.support.mock import MagicMock, patch, call

Evaluating Truth¶

A longer discussion on the types of assertions one can make can be found by reading PyTest's documentation on assertions.

Tests Using Mock Objects¶

In many cases, the purpose of a Salt module is to interact with some external system, whether it be to control a database, manipulate files on a filesystem or something else. In these varied cases, it's necessary to design a unit test which can test the function whilst replacing functions which might actually call out to external systems. One might think of this as "blocking the exits" for code under tests and redirecting the calls to external systems with our own code which produces known results during the duration of the test.

To achieve this behavior, Salt makes heavy use of the MagicMock package.

To understand how one might integrate Mock into writing a unit test for Salt, let's imagine a scenario in which we're testing an execution module that's designed to operate on a database. Furthermore, let's imagine two separate methods, here presented in pseduo-code in an imaginary execution module called 'db.py'.

def create_user(username):
    qry = "CREATE USER {0}".format(username)
    execute_query(qry)


def execute_query(qry):
    # Connect to a database and actually do the query...
    ...

Here, let's imagine that we want to create a unit test for the create_user function. In doing so, we want to avoid any calls out to an external system and so while we are running our unit tests, we want to replace the actual interaction with a database with a function that can capture the parameters sent to it and return pre-defined values. Therefore, our task is clear -- to write a unit test which tests the functionality of create_user while also replacing 'execute_query' with a mocked function.

To begin, we set up the skeleton of our test much like we did before, but with additional imports for MagicMock:

from salt.modules import db
from tests.support.mock import MagicMock, patch, call


def test_create_user():
    """
    Test creating a user
    """
    # First, we replace 'execute_query' with our own mock function
    with patch.object(db, "execute_query", MagicMock()) as db_exq:

        # Now that the exits are blocked, we can run the function under test.
        db.create_user("testuser")

        # We could now query our mock object to see which calls were made
        # to it.
        ## print db_exq.mock_calls

        # Construct a call object that simulates the way we expected
        # execute_query to have been called.
        expected_call = call("CREATE USER testuser")

        # Compare the expected call with the list of actual calls.  The
        # test will succeed or fail depending on the output of this
        # assertion.
        db_exq.assert_has_calls(expected_call)

Modifying __salt__ In Place¶

At times, it becomes necessary to make modifications to a module's view of functions in its own __salt__ dictionary. Luckily, this process is quite easy.

Below is an example that uses MagicMock's patch functionality to insert a function into __salt__ that's actually a MagicMock instance.

import pytest
import salt.modules.my_module as my_module


@pytest.fixture
def configure_loader_modules():
    """
    This fixture should return a dictionary which is what's going to be used to
    patch and mock Salt's loader
    """
    return {my_module: {}}


def show_patch(self):
    with patch.dict(my_module.__salt__, {"function.to_replace": MagicMock()}):
        # From this scope, carry on with testing, with a modified __salt__!
        ...

A Simple Example¶

Let's assume that we're testing a very basic function in an imaginary Salt execution module. Given a module called fib.py that has a function called calculate(num_of_results), which given a num_of_results, produces a list of sequential Fibonacci numbers of that length.

A unit test to test this function might be commonly placed in a file called tests/pytests/unit/modules/test_fib.py. The convention is to place unit tests for Salt execution modules in test/pytests/unit/modules/ and to name the tests module prefixed with test_*.py.

Tests are grouped around test cases, which are logically grouped sets of tests against a piece of functionality in the tested software. To return to our example, here's how we might write the skeleton for testing fib.py:

import salt.modules.fib as fib


def test_fib():
    """
    To create a unit test, we should prefix the name with `test_' so
    that it's recognized by the test runner.
    """
    fib_five = (0, 1, 1, 2, 3)
    assert fib.calculate(5) == fib_five

At this point, the test can now be run, either individually or as a part of a full run of the test runner. To ease development, a single test can be executed:

nox -e 'test-3(coverage=False)' -- -v tests/pytests/unit/modules/test_fib.py

This will report the status of the test: success, failure, or error. The -v flag increases output verbosity.

To review the results of a particular run, take a note of the log location given in the output for each test run:

...etc... --log-file=artifacts/logs/runtests-20210106103414.685791.log ...etc...

A More Complete Example¶

Consider the following function from salt/modules/linux_sysctl.py.

def get(name):
    """
    Return a single sysctl parameter for this minion

    CLI Example:

    .. code-block:: bash

        salt '*' sysctl.get net.ipv4.ip_forward
    """
    cmd = "sysctl -n {}".format(name)
    out = __salt__["cmd.run"](cmd, python_shell=False)
    return out

This function is very simple, comprising only four source lines of code and having only one return statement, so we know only one test is needed. There are also two inputs to the function, the name function argument and the call to __salt__['cmd.run'](), both of which need to be appropriately mocked.

Mocking a function parameter is straightforward, whereas mocking a function call will require, in this case, the use of MagicMock. For added isolation, we will also redefine the __salt__ dictionary such that it only contains 'cmd.run'.

import pytest
import salt.modules.linux_sysictl as linux_sysctl
from tests.support.mock import MagicMock, patch


@pytest.fixture
def configure_loader_modules():
    return {linux_sysctl: {}}


def test_get():
    """
    Tests the return of get function
    """
    mock_cmd = MagicMock(return_value=1)
    with patch.dict(linux_sysctl.__salt__, {"cmd.run": mock_cmd}):
        assert linux_sysctl.get("net.ipv4.ip_forward") == 1

Since get() has only one raise or return statement and that statement is a success condition, the test function is simply named test_get(). As described, the single function call parameter, name is mocked with net.ipv4.ip_forward and __salt__['cmd.run'] is replaced by a MagicMock function object. We are only interested in the return value of __salt__['cmd.run'], which MagicMock allows us by specifying via return_value=1. Finally, the test itself tests for equality between the return value of get() and the expected return of 1. This assertion is expected to succeed because get() will determine its return value from __salt__['cmd.run'], which we have mocked to return 1.

A Complex Example¶

Now consider the assign() function from the same salt/modules/linux_sysctl.py source file.

def assign(name, value):
    """
    Assign a single sysctl parameter for this minion

    CLI Example:

    .. code-block:: bash

        salt '*' sysctl.assign net.ipv4.ip_forward 1
    """
    value = str(value)

    tran_tab = name.translate("".maketrans("./", "/."))

    sysctl_file = "/proc/sys/{}".format(tran_tab)
    if not os.path.exists(sysctl_file):
        raise CommandExecutionError("sysctl {} does not exist".format(name))

    ret = {}
    cmd = 'sysctl -w {}="{}"'.format(name, value)
    data = __salt__["cmd.run_all"](cmd, python_shell=False)
    out = data["stdout"]
    err = data["stderr"]

    # Example:
    #    # sysctl -w net.ipv4.tcp_rmem="4096 87380 16777216"
    #    net.ipv4.tcp_rmem = 4096 87380 16777216
    regex = re.compile(r"^{}\s+=\s+{}$".format(re.escape(name), re.escape(value)))

    if not regex.match(out) or "Invalid argument" in str(err):
        if data["retcode"] != 0 and err:
            error = err
        else:
            error = out
        raise CommandExecutionError("sysctl -w failed: {}".format(error))
    new_name, new_value = out.split(" = ", 1)
    ret[new_name] = new_value
    return ret

This function contains two raise statements and one return statement, so we know that we will need (at least) three tests. It has two function arguments and many references to non-builtin functions. In the tests below you will see that MagicMock's patch() method may be used as a context manager or as a decorator. When patching the salt dunders however, please use the context manager approach.

There are three test functions, one for each raise and return statement in the source function. Each function is self-contained and contains all and only the mocks and data needed to test the raise or return statement it is concerned with.

import pytest
import salt.modules.linux_sysctl as linux_sysctl
from salt.exceptions import CommandExecutionError
from tests.support.mock import MagicMock, patch


@pytest.fixture
def configure_loader_modules():
    return {linux_sysctl: {}}


def test_assign_proc_sys_failed():
    """
    Tests if /proc/sys/<kernel-subsystem> exists or not
    """
    with patch("os.path.exists", MagicMock(return_value=False)):
        cmd = {
            "pid": 1337,
            "retcode": 0,
            "stderr": "",
            "stdout": "net.ipv4.ip_forward = 1",
        }
        mock_cmd = MagicMock(return_value=cmd)
        with patch.dict(linux_sysctl.__salt__, {"cmd.run_all": mock_cmd}):
            with pytest.raises(CommandExecutionError):
                linux_sysctl.assign("net.ipv4.ip_forward", 1)


def test_assign_cmd_failed():
    """
    Tests if the assignment was successful or not
    """
    with patch("os.path.exists", MagicMock(return_value=True)):
        cmd = {
            "pid": 1337,
            "retcode": 0,
            "stderr": 'sysctl: setting key "net.ipv4.ip_forward": Invalid argument',
            "stdout": "net.ipv4.ip_forward = backward",
        }
        mock_cmd = MagicMock(return_value=cmd)
        with patch.dict(linux_sysctl.__salt__, {"cmd.run_all": mock_cmd}):
            with pytest.raises(CommandExecutionError):
                linux_sysctl.assign("net.ipv4.ip_forward", "backward")


def test_assign_success():
    """
    Tests the return of successful assign function
    """
    with patch("os.path.exists", MagicMock(return_value=True)):
        cmd = {
            "pid": 1337,
            "retcode": 0,
            "stderr": "",
            "stdout": "net.ipv4.ip_forward = 1",
        }
        ret = {"net.ipv4.ip_forward": "1"}
        mock_cmd = MagicMock(return_value=cmd)
        with patch.dict(linux_sysctl.__salt__, {"cmd.run_all": mock_cmd}):
            assert linux_sysctl.assign("net.ipv4.ip_forward", 1) == ret