Add a multi-language mutation tester that writes .metrics/mutate for uml-viewer.

This commit is contained in:
Robert C. Martin
2026-09-30 10:40:54 -05:00
commit bf42bd22e7
24 changed files with 3201 additions and 0 deletions
+94
View File
@@ -0,0 +1,94 @@
import subprocess
import sys
from mutator.cli import parse_args, run
from mutator.runner import CommandResult
def test_lines_option_parses_positive_numbers():
options = parse_args(["--lines", "2, 4", "src/demo.py"])
assert options.lines == {2, 4}
def test_changed_selects_a_dirty_source_file(tmp_path, monkeypatch, capsys):
path = tmp_path / "src" / "demo.py"
path.parent.mkdir()
path.write_text("def place(x):\n return x > 0\n", encoding="utf-8")
subprocess.run(["git", "init"], cwd=tmp_path, check=True, capture_output=True)
subprocess.run(["git", "add", "src/demo.py"], cwd=tmp_path, check=True, capture_output=True)
monkeypatch.chdir(tmp_path)
code = run(["--scan", "--no-coverage", "--changed", "--root", str(tmp_path)])
captured = capsys.readouterr()
assert code == 0
assert "demo.py:2" in captured.out
def test_a_directory_argument_skips_a_named_test_file(tmp_path, capsys):
src = tmp_path / "pkg" / "app.py"
src.parent.mkdir()
src.write_text("def place(x):\n return x > 0\n", encoding="utf-8")
test = tmp_path / "pkg" / "test_app.py"
test.write_text("def test_place():\n assert True\n", encoding="utf-8")
code = run(
["--scan", "--no-coverage", "--root", str(tmp_path), str(tmp_path / "pkg"), str(test)]
)
captured = capsys.readouterr()
assert code == 0
assert "Skipping test file" in captured.err
assert "app.py:2" in captured.out
def test_help_and_conflicting_flags():
help_options = parse_args(["--help"])
assert help_options.exit_code == 0
assert "uml-viewer" in help_options.message
conflict = parse_args(["--mutate-all", "--since-last-run", "src/demo.py"])
assert conflict.exit_code == 1
assert run(["--help"]) == 0
def test_scan_lists_sites_without_writing_metrics(tmp_path, monkeypatch):
path = tmp_path / "src" / "demo.py"
path.parent.mkdir()
path.write_text("def place(x):\n return x > 0\n", encoding="utf-8")
monkeypatch.chdir(tmp_path)
code = run(["--scan", "--no-coverage", "--root", str(tmp_path), "src/demo.py"])
assert code == 0
assert not (tmp_path / ".metrics").exists()
def test_real_python_mutants_update_the_snapshot(tmp_path, capsys):
path = tmp_path / "src" / "demo.py"
path.parent.mkdir()
path.write_text(
"def add(a, b):\n"
" if a > 0:\n"
" return a + b\n"
" return 0\n",
encoding="utf-8",
)
command = (
f"{sys.executable} -c \"import sys; sys.path.insert(0, 'src'); "
"from demo import add; raise SystemExit(0 if add(2, 3) == 5 else 1)\""
)
code = run(
[
"--no-coverage",
"--mutate-all",
"--root",
str(tmp_path),
"--test-command",
command,
"src/demo.py",
]
)
captured = capsys.readouterr()
assert code == 3
assert "SURVIVED" in captured.out
assert "KILLED" in captured.out
snapshot = tmp_path / ".metrics" / "mutate" / "demo.edn"
assert snapshot.is_file()
text = snapshot.read_text(encoding="utf-8")
assert ":namespace \"demo\"" in text
assert ":id \"defn/add\"" in text
assert path.read_text(encoding="utf-8").startswith("def add")
+57
View File
@@ -0,0 +1,57 @@
from mutator.coverage import covered_lines
def test_jacoco_marks_lines_with_covered_instructions(tmp_path):
report = tmp_path / "target" / "site" / "jacoco" / "jacoco.xml"
report.parent.mkdir(parents=True)
report.write_text(
"""<?xml version="1.0"?>
<report name="demo">
<package name="demo">
<sourcefile name="Board.java">
<line nr="3" mi="0" ci="2" mb="0" cb="0"/>
<line nr="4" mi="4" ci="0" mb="1" cb="0"/>
<line nr="5" mi="0" ci="0" mb="0" cb="1"/>
</sourcefile>
</package>
</report>
""",
encoding="utf-8",
)
source = tmp_path / "src" / "demo" / "Board.java"
source.parent.mkdir(parents=True)
source.write_text("package demo;\nclass Board {}\n", encoding="utf-8")
assert covered_lines(tmp_path, source, "java") == {3, 5}
def test_go_profile_marks_hit_lines(tmp_path):
profile = tmp_path / "target" / "coverage" / "go" / "coverage.out"
profile.parent.mkdir(parents=True)
profile.write_text(
"mode: set\n"
"demo/widget.go:2.1,4.2 1 1\n"
"demo/widget.go:5.1,6.2 1 0\n",
encoding="utf-8",
)
source = tmp_path / "demo" / "widget.go"
source.parent.mkdir()
source.write_text("package demo\nfunc Run() {}\n", encoding="utf-8")
assert covered_lines(tmp_path, source, "go") == {2, 3, 4}
assert covered_lines(tmp_path, tmp_path / "other" / "missing.go", "go") is None
empty = tmp_path / "empty"
empty.mkdir()
assert covered_lines(empty, empty / "demo" / "widget.go", "go") is None
def test_lcov_marks_executed_lines(tmp_path):
report = tmp_path / "target" / "coverage" / "python" / "lcov.info"
report.parent.mkdir(parents=True)
report.write_text(
"SF:src/demo/app.py\nDA:1,3\nDA:2,0\nend_of_record\n",
encoding="utf-8",
)
source = tmp_path / "src" / "demo" / "app.py"
source.parent.mkdir(parents=True)
source.write_text("def place():\n return 1\n", encoding="utf-8")
assert covered_lines(tmp_path, source, "python") == {1}
assert covered_lines(tmp_path, source, "clojure") == {1}
+55
View File
@@ -0,0 +1,55 @@
from mutator.edn import dumps, keyword, loads
def test_round_trip_matches_the_snapshot_shape():
payload = {
"version": 1,
"tested-at": "2026-09-30T12:00:00-05:00",
"source": 'src/demo/say "hi".py',
"namespace": "demo.Board",
"outcomes": {
'["src/a.py","demo.Board","defn/place",1,2,">",">="]': keyword("killed"),
'["src/a.py","demo.Board","defn-/hide",3,4,"true","false"]': keyword("survived"),
},
"forms": [
{
"id": "defn/place",
"kind": "defn",
"file": "src/demo/Board.java",
"line": 4,
"end-line": 9,
"hash": "abc",
"killed": 1,
"survived": 0,
"uncovered": 2,
"sites": 3,
}
],
}
text = dumps(payload)
assert text.startswith("{")
assert ":namespace" in text
assert ":killed" in text
parsed = loads(text)
assert parsed["namespace"] == "demo.Board"
assert parsed["forms"][0]["id"] == "defn/place"
assert parsed["forms"][0]["end-line"] == 9
assert parsed["forms"][0]["killed"] == 1
assert set(parsed["outcomes"].values()) == {"killed", "survived"}
def test_empty_collections_and_nil():
assert loads("{:forms [] :outcomes {} :note nil}") == {
"forms": [],
"outcomes": {},
"note": None,
}
def test_atoms_cover_booleans_numbers_and_symbols():
assert loads("true") is True
assert loads("false") is False
assert loads("12") == 12
assert loads("-3") == -3
assert loads("1.50") == 1.5
assert loads("foo") == "foo"
+147
View File
@@ -0,0 +1,147 @@
from mutator.engine import mutate_file, restore_backups
from mutator.edn import loads
from mutator.metrics import snapshot_path
from mutator.runner import CommandResult
class FakeRunner:
def __init__(self):
self.verbose = False
self.calls = 0
def run(self, command, cwd, timeout):
self.calls += 1
text = (cwd / "src" / "demo.py").read_text(encoding="utf-8")
namespace: dict = {}
exec(text, namespace)
ok = namespace["add"](1, 2) == 3
return CommandResult(code=0 if ok else 1, timed_out=False, seconds=0.01, output="")
SOURCE = """def add(a, b):
if a > 0:
return a + b
return a + b
"""
def test_mutants_are_killed_or_kept_and_the_snapshot_is_differential(tmp_path):
path = tmp_path / "src" / "demo.py"
path.parent.mkdir()
path.write_text(SOURCE, encoding="utf-8")
original = path.read_bytes()
runner = FakeRunner()
first = mutate_file(
path,
tmp_path,
runner=runner,
covered_lines=None,
ignore_coverage=True,
mutate_all=True,
lines=None,
test_command="fake",
timeout_factor=10,
mutation_warning=50,
baselines={},
)
assert path.read_bytes() == original
assert first.baseline_failed is False
form = first.forms[0]
assert form.id == "defn/add"
assert form.namespace == "demo"
# `>` and the unexecuted `+` survive. The executed `+` is killed. `0` -> `1`
# still returns 3 for add(1, 2).
assert form.killed == 1
assert form.survived == 3
assert form.sites == 4
assert form.killed + form.survived + form.uncovered == form.sites
data = loads(snapshot_path(tmp_path, "demo").read_text(encoding="utf-8"))
assert data["namespace"] == "demo"
assert data["forms"][0]["id"] == "defn/add"
assert data["forms"][0]["killed"] == 1
assert data["forms"][0]["survived"] == 3
first_calls = runner.calls
second = mutate_file(
path,
tmp_path,
runner=runner,
covered_lines=None,
ignore_coverage=True,
mutate_all=False,
lines=None,
test_command="fake",
timeout_factor=10,
mutation_warning=50,
baselines={},
)
assert second.forms[0].killed == 1
assert second.forms[0].survived == 3
# Baseline plus the three survivors. The killed mutant stays in the snapshot.
assert runner.calls - first_calls == 4
def test_uncovered_sites_are_not_executed(tmp_path):
path = tmp_path / "src" / "demo.py"
path.parent.mkdir()
path.write_text(SOURCE, encoding="utf-8")
runner = FakeRunner()
result = mutate_file(
path,
tmp_path,
runner=runner,
covered_lines=set(),
ignore_coverage=False,
mutate_all=True,
lines=None,
test_command="fake",
timeout_factor=10,
mutation_warning=50,
baselines={},
)
assert runner.calls == 0
assert result.forms[0].uncovered == 4
assert result.forms[0].killed == 0
assert result.forms[0].sites == 4
def test_a_failed_baseline_does_not_rewrite_metrics(tmp_path):
path = tmp_path / "src" / "demo.py"
path.parent.mkdir()
path.write_text(SOURCE, encoding="utf-8")
class Red:
verbose = False
def run(self, command, cwd, timeout):
return CommandResult(code=1, timed_out=False, seconds=0.01, output="nope")
result = mutate_file(
path,
tmp_path,
runner=Red(),
covered_lines=None,
ignore_coverage=True,
mutate_all=True,
lines=None,
test_command="fake",
timeout_factor=10,
mutation_warning=50,
baselines={},
)
assert result.baseline_failed
assert not snapshot_path(tmp_path, "demo").exists()
def test_restore_backups_puts_an_interrupted_mutant_back(tmp_path):
source = tmp_path / "src" / "demo.py"
source.parent.mkdir()
source.write_text("original\n", encoding="utf-8")
backup = tmp_path / "target" / "mutator-backup" / "src" / "demo.py"
backup.parent.mkdir(parents=True)
backup.write_text("original\n", encoding="utf-8")
source.write_text("mutated\n", encoding="utf-8")
restored = restore_backups(tmp_path)
assert restored == [source]
assert source.read_text(encoding="utf-8") == "original\n"
assert not backup.exists()
+132
View File
@@ -0,0 +1,132 @@
from mutator.edn import dumps, loads
from mutator.metrics import load_history, snapshot_path, write_results
from mutator.model import FormResult
def _form(namespace, name, file_name, digest, killed=1, private=False):
form_id = f"defn-/{name}" if private else f"defn/{name}"
return FormResult(
id=form_id,
namespace=namespace,
name=name,
private=private,
file=file_name,
line=2,
end_line=4,
digest=digest,
killed=killed,
survived=0,
uncovered=1,
sites=2,
)
def _viewer_name(form_id):
"""The same split uml-viewer.application.overlay/form-name performs."""
if form_id.startswith("defn-/"):
return form_id[6:], True
if form_id.startswith("defn/"):
return form_id[5:], False
return None, False
def test_one_snapshot_per_namespace_preserves_the_other_file(tmp_path):
write_results(
tmp_path,
"a.go",
[_form("demo", "Run", "a.go", "hash-a")],
{},
)
write_results(
tmp_path,
"b.go",
[_form("demo", "Stop", "b.go", "hash-b", killed=4)],
{},
)
path = snapshot_path(tmp_path, "demo")
data = loads(path.read_text(encoding="utf-8"))
assert data["namespace"] == "demo"
names = []
for form in data["forms"]:
operation, private = _viewer_name(form["id"])
names.append((operation, private, form["killed"], form["uncovered"], form["sites"]))
assert ("Run", False, 1, 1, 2) in names
assert ("Stop", False, 4, 1, 2) in names
assert list(tmp_path.joinpath(".metrics", "mutate").rglob("*.edn")) == [path]
write_results(
tmp_path,
"a.go",
[_form("demo", "Run", "a.go", "hash-a", killed=9)],
{},
)
again = loads(path.read_text(encoding="utf-8"))
by_id = {form["id"]: form for form in again["forms"]}
assert by_id["defn/Run"]["killed"] == 9
assert by_id["defn/Stop"]["killed"] == 4
def test_private_form_id_and_history_round_trip(tmp_path):
outcome = '["hide.clj","demo.core","defn-/hide",8,12,"false","true"]'
write_results(
tmp_path,
"hide.clj",
[_form("demo.core", "hide", "hide.clj", "digest", private=True)],
{outcome: "killed"},
)
data = loads(snapshot_path(tmp_path, "demo.core").read_text(encoding="utf-8"))
assert _viewer_name(data["forms"][0]["id"]) == ("hide", True)
history = load_history(tmp_path, "hide.clj")
assert history.forms[("demo.core", "defn-/hide")].digest == "digest"
assert history.outcomes[outcome] == "killed"
def test_snapshot_paths_follow_the_namespace(tmp_path):
assert snapshot_path(tmp_path, "example.com/demo.Widget") == (
tmp_path / ".metrics" / "mutate" / "example.com" / "demo.Widget.edn"
)
assert snapshot_path(tmp_path, "crate::foo::Bar") == (
tmp_path / ".metrics" / "mutate" / "crate" / "foo" / "Bar.edn"
)
def test_a_non_canonical_snapshot_is_folded_into_the_namespace_file(tmp_path):
stale = tmp_path / ".metrics" / "mutate" / "legacy.edn"
stale.parent.mkdir(parents=True)
stale.write_text(
dumps(
{
"version": 1,
"namespace": "demo",
"source": "b.go",
"forms": [
{
"id": "defn/Stop",
"file": "b.go",
"line": 3,
"hash": "hash-b",
"killed": 4,
"survived": 0,
"uncovered": 1,
"sites": 2,
}
],
"outcomes": {},
}
)
+ "\n",
encoding="utf-8",
)
write_results(tmp_path, "a.go", [_form("demo", "Run", "a.go", "hash-a")], {})
assert not stale.exists()
data = loads(snapshot_path(tmp_path, "demo").read_text(encoding="utf-8"))
by_id = {form["id"]: form for form in data["forms"]}
assert by_id["defn/Stop"]["killed"] == 4
assert by_id["defn/Run"]["file"] == "a.go"
def test_dropping_a_file_removes_its_namespace_when_nothing_else_remains(tmp_path):
write_results(tmp_path, "a.go", [_form("demo", "Run", "a.go", "hash-a")], {})
write_results(tmp_path, "a.go", [], {})
assert not snapshot_path(tmp_path, "demo").exists()
+68
View File
@@ -0,0 +1,68 @@
from mutator.runner import _clojure_command, _python_command, nearest
def test_nearest_walks_up_to_the_marker(tmp_path):
root = tmp_path / "proj"
nested = root / "src" / "demo"
nested.mkdir(parents=True)
(root / "pyproject.toml").write_text("", encoding="utf-8")
source = nested / "app.py"
source.write_text("x = 1\n", encoding="utf-8")
assert nearest(source, "pyproject.toml", root) == root
assert nearest(nested, "pyproject.toml", root) == root
assert nearest(source, "missing.toml", root) is None
def test_clojure_commands(tmp_path):
bb = tmp_path / "bb-spec"
bb.mkdir()
(bb / "bb.edn").write_text('{:tasks {spec "spec"}}', encoding="utf-8")
assert _clojure_command(bb) == "bb spec --tag ~no-mutate"
bb_test = tmp_path / "bb-test"
bb_test.mkdir()
(bb_test / "bb.edn").write_text("{:tasks {test (clojure \"-M:test\")}}", encoding="utf-8")
assert _clojure_command(bb_test) == "bb test"
spec = tmp_path / "spec"
spec.mkdir()
(spec / "deps.edn").write_text(
"{:aliases {:spec {:extra-deps {speclj/speclj {}}}}}", encoding="utf-8"
)
assert _clojure_command(spec) == "clj -M:spec --tag ~no-mutate"
spec_only = tmp_path / "spec-only"
spec_only.mkdir()
(spec_only / "deps.edn").write_text("{:aliases {:spec {}}}", encoding="utf-8")
assert _clojure_command(spec_only) == "clj -M:spec"
plain = tmp_path / "plain"
plain.mkdir()
(plain / "deps.edn").write_text("{:deps {}}", encoding="utf-8")
assert _clojure_command(plain) == "clj -M:test"
def test_python_commands(tmp_path):
pytest_dir = tmp_path / "py"
pytest_dir.mkdir()
(pytest_dir / "pytest.ini").write_text("[pytest]\n", encoding="utf-8")
assert _python_command(pytest_dir).endswith("-m pytest")
unit = tmp_path / "unit"
unit.mkdir()
assert _python_command(unit).endswith("-m unittest discover")
cfg = tmp_path / "cfg"
cfg.mkdir()
(cfg / "setup.cfg").write_text("[tool:pytest]\n", encoding="utf-8")
assert _python_command(cfg).endswith("-m pytest")
project = tmp_path / "proj"
project.mkdir()
(project / "pyproject.toml").write_text("[project]\ndependencies=['pytest']\n", encoding="utf-8")
assert _python_command(project).endswith("-m pytest")
conf = tmp_path / "conf"
conf.mkdir()
(conf / "conftest.py").write_text("", encoding="utf-8")
assert _python_command(conf).endswith("-m pytest")
+165
View File
@@ -0,0 +1,165 @@
from pathlib import Path
from mutator.functions import form_id, is_private, project_functions, sites_in_file
def _sites(tmp_path: Path, relative: str, source: str):
path = tmp_path / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source, encoding="utf-8")
return sites_in_file(source, path, tmp_path, relative)
def test_clojure_rules_follow_clj_mutate_and_mark_private_defns(tmp_path):
source = """(ns demo.core)
(defn place [x]
(if (> x 0)
(+ x 1)
0))
(defn- hide []
false)
"""
sites = _sites(tmp_path, "src/demo/core.clj", source)
described = {(site.form_id, site.original, site.mutant) for site in sites}
assert ("defn/place", "if", "if-not") in described
assert ("defn/place", ">", ">=") in described
assert ("defn/place", "+", "-") in described
assert ("defn/place", "0", "1") in described
assert ("defn/place", "1", "0") in described
assert ("defn-/hide", "false", "true") in described
assert {site.namespace for site in sites} == {"demo.core"}
functions = project_functions(source, tmp_path / "src/demo/core.clj", tmp_path)
assert form_id("hide", True) == "defn-/hide"
assert is_private("clojure", "hide", source, 6, 7)
assert {fn.namespace for fn in functions} == {"demo.core"}
def test_clojure_ignores_strings_comments_and_quotes(tmp_path):
source = """(ns demo.core)
(defn place []
(comment (+ 1 0))
(quote (+ 1 0))
(str "true + 1"))
"""
sites = _sites(tmp_path, "src/demo/core.clj", source)
assert sites == []
def test_java_rules_follow_mutate4java(tmp_path):
source = """package demo;
class Board {
int place(int x) {
if (x > 0 && ready)
return x + 1;
return !ok ? -x : 0;
}
private int hide() { return 1; }
}
"""
sites = _sites(tmp_path, "src/demo/Board.java", source)
public = [site for site in sites if site.form_id == "defn/place"]
private = [site for site in sites if site.form_id == "defn-/hide"]
assert ("+", "-") in {(site.original, site.mutant) for site in public}
assert ("/", "*") not in {(site.original, site.mutant) for site in public}
assert ("*", "/") not in {(site.original, site.mutant) for site in public}
assert (">", ">=") in {(site.original, site.mutant) for site in public}
assert ("&&", "||") in {(site.original, site.mutant) for site in public}
assert ("!", "") in {(site.original, site.mutant) for site in public}
assert ("-", "") in {(site.original, site.mutant) for site in public}
assert ("0", "1") in {(site.original, site.mutant) for site in public}
assert private == [] or {site.original for site in private} == {"1"}
assert {site.namespace for site in sites} == {"demo.Board"}
def test_java_swaps_division_both_ways(tmp_path):
source = """package demo;
class Math {
int place(int a, int b) { return a * b + a / b; }
}
"""
sites = _sites(tmp_path, "src/demo/Math.java", source)
pairs = {(site.original, site.mutant) for site in sites}
assert ("*", "/") in pairs
assert ("/", "*") in pairs
assert ("+", "-") in pairs
def test_go_matches_mutate4go_and_splits_receivers(tmp_path):
source = """package demo
func Run(a int, b int) int {
if a > 0 && b == 1 {
return a * b
}
return a / b
}
type Widget struct{}
func (w Widget) Run() bool { return true }
"""
sites = _sites(tmp_path, "demo.go", source)
free = [site for site in sites if site.namespace == "demo"]
method = [site for site in sites if site.namespace == "demo.Widget"]
pairs = {(site.original, site.mutant) for site in free}
assert ("*", "/") in pairs
assert ("/", "*") not in pairs
assert ("&&", "||") in pairs
assert ("==", "!=") in pairs
assert (">", ">=") in pairs
assert ("0", "1") in pairs
assert ("1", "0") in pairs
assert {site.form_id for site in method} == {"defn/Run"}
assert ("true", "false") in {(site.original, site.mutant) for site in method}
assert {site.form_id for site in free} == {"defn/Run"}
def test_go_unexported_function_is_private(tmp_path):
source = """package demo
func hide() bool { return false }
"""
sites = _sites(tmp_path, "demo.go", source)
assert sites[0].form_id == "defn-/hide"
def test_python_typescript_and_rust_use_the_java_decisions(tmp_path):
python = _sites(
tmp_path,
"src/demo/app.py",
"def place(x):\n if x > 0 and ready:\n return x + 1\n if not ok:\n return 0\n",
)
typescript = _sites(
tmp_path,
"src/demo/app.ts",
"export function place(x: number) {\n if (x > 0 && ready) return x + 1;\n return true === false;\n}\n",
)
rust = _sites(
tmp_path,
"src/lib.rs",
"fn place(x: i32) -> i32 {\n if x > 0 && ready { return x + 1; }\n if !ok { return -x; }\n 0\n}\n",
)
assert ("and", "or") in {(site.original, site.mutant) for site in python}
assert ("not", "") in {(site.original, site.mutant) for site in python}
assert ("True", "False") not in {(site.original, site.mutant) for site in python}
assert {site.namespace for site in python} == {"demo.app"}
assert ("===", "!==") in {(site.original, site.mutant) for site in typescript}
assert ("&&", "||") in {(site.original, site.mutant) for site in typescript}
assert {site.namespace for site in typescript} == {"demo.app"}
assert ("&&", "||") in {(site.original, site.mutant) for site in rust}
assert ("!", "") in {(site.original, site.mutant) for site in rust}
assert ("-", "") in {(site.original, site.mutant) for site in rust}
def test_python_private_and_class_namespace(tmp_path):
source = """class Board:
def place(self, x):
return x > 0
def _hide(self):
return False
"""
sites = _sites(tmp_path, "src/demo/board.py", source)
assert {site.form_id for site in sites if site.name == "place"} == {"defn/place"}
assert {site.form_id for site in sites if site.name == "_hide"} == {"defn-/_hide"}
assert {site.namespace for site in sites} == {"demo.board.Board"}
def test_string_operators_are_not_sites(tmp_path):
source = 'def place():\n return "x > 0 and true"\n'
assert _sites(tmp_path, "src/demo/app.py", source) == []