diff --git a/src/dryer/astnorm.py b/src/dryer/astnorm.py index 80b5f0c..f185a2c 100644 --- a/src/dryer/astnorm.py +++ b/src/dryer/astnorm.py @@ -189,32 +189,39 @@ def _is_literal(node) -> bool: return True -def _normalize_call(node): +def _named_before_args(node): incoming = [] - args = None for child in node.named_children: if child.type in _ARG_LISTS: - args = child - break + return incoming, child incoming.append(child) - type_args = [child for child in incoming if child.type in _TYPE_ARGS] - rest = [child for child in incoming if child.type not in _TYPE_ARGS] + return incoming, None + + +def _without_type_args(nodes): + type_args = [child for child in nodes if child.type in _TYPE_ARGS] + rest = [child for child in nodes if child.type not in _TYPE_ARGS] + return rest, type_args + + +def _extend_normalized(out, nodes) -> None: + for node in nodes: + norm = _normalize(node, False) + if norm is not None: + out.append(norm) + + +def _normalize_call(node): + incoming, args = _named_before_args(node) + rest, type_args = _without_type_args(incoming) out = [K(node.type)] if rest: *receivers, callee = rest - for receiver in receivers: - norm = _normalize(receiver, False) - if norm is not None: - out.append(norm) + _extend_normalized(out, receivers) out.append(_normalize_callee(callee)) - for type_node in type_args: - norm = _normalize(type_node, False) - if norm is not None: - out.append(norm) + _extend_normalized(out, type_args) if args is not None: - norm = _normalize(args, False) - if norm is not None: - out.append(norm) + _extend_normalized(out, [args]) return out @@ -244,20 +251,29 @@ def _normalize_path(node): return parts +def _normalized_name(name, head: bool): + """Keep a member's spelling when it is the thing being called.""" + + if name.type in _IDENTIFIERS: + if head: + return [K("symbol"), _text(name)] + return K("symbol") + if head: + return _normalize_callee(name) + return _normalize(name, False) + + def _normalize_attr(node, head: bool): named = list(node.named_children) - parts = [K(node.type)] if not named: - return parts + return [K(node.type)] *objects, name = named + parts = [K(node.type)] for obj in objects: norm = _normalize(obj, False) if norm is not None: parts.append(norm) - if name.type in _IDENTIFIERS: - parts.append([K("symbol"), _text(name)] if head else K("symbol")) - else: - nested = _normalize_callee(name) if head else _normalize(name, False) - if nested is not None: - parts.append(nested) + nested = _normalized_name(name, head) + if nested is not None: + parts.append(nested) return parts diff --git a/src/dryer/cli.py b/src/dryer/cli.py index d391ee5..4460f0e 100644 --- a/src/dryer/cli.py +++ b/src/dryer/cli.py @@ -159,14 +159,13 @@ def parse_args(argv: list[str] | None = None) -> Options: return options +def _git_status(root: Path): + args = ["git", "status", "--porcelain"] + return subprocess.run(args, cwd=root, check=False, capture_output=True, text=True) + + def _changed_files(root: Path) -> list[Path]: - result = subprocess.run( - ["git", "status", "--porcelain"], - cwd=root, - check=False, - capture_output=True, - text=True, - ) + result = _git_status(root) if result.returncode != 0: print(result.stderr.strip() or "git status failed", file=sys.stderr) return [] @@ -196,12 +195,14 @@ def _positionals(root: Path, args: list[str]) -> tuple[list[Path], list[str]]: return existing, filters +def _tracked_source(path: Path) -> bool: + if language_of(path) is None: + return False + return not is_test_file(path) + + def _changed_source(root: Path) -> list[Path]: - return [ - path - for path in _changed_files(root) - if language_of(path) is not None and not is_test_file(path) - ] + return [path for path in _changed_files(root) if _tracked_source(path)] def _explicit_files(existing: list[Path]) -> list[Path]: diff --git a/src/dryer/clojure_read.py b/src/dryer/clojure_read.py index 6816a8d..532b19e 100644 --- a/src/dryer/clojure_read.py +++ b/src/dryer/clojure_read.py @@ -218,24 +218,15 @@ class Reader: form = self.read_form() if not isinstance(form, Coll) or form.kind != "list": raise ReadError(self.line, "reader conditional body must be a list") - chosen = None - index = 0 - items = form.items - while index + 1 < len(items): - feature = items[index] - expr = items[index + 1] - index += 2 - name = feature.name if isinstance(feature, Kw) else None - if name in {"clj", "default"}: - chosen = expr - break + chosen = _chosen_branch(form.items) if chosen is None: return None if not splicing: return chosen - if isinstance(chosen, Coll) and chosen.kind in {"list", "vector"}: - return Splice(list(chosen.items)) - raise ReadError(self.line, "splicing reader conditional needs a collection") + spliced = _spliced(chosen) + if spliced is None: + raise ReadError(self.line, "splicing reader conditional needs a collection") + return spliced def read_form(self): self.skip_ws() @@ -283,15 +274,47 @@ class Reader: return Sym(token, line) +def _branch_name(feature): + if isinstance(feature, Kw): + return feature.name + return None + + +def _chosen_branch(items): + index = 0 + while index + 1 < len(items): + expr = items[index + 1] + if _branch_name(items[index]) in {"clj", "default"}: + return expr + index += 2 + return None + + +def _spliced(chosen): + if isinstance(chosen, Coll) and chosen.kind in {"list", "vector"}: + return Splice(list(chosen.items)) + return None + + +def _strip_bom(text: str) -> str: + if text.startswith("\ufeff"): + return text[1:] + return text + + +def _skip_shebang(reader: Reader, text: str) -> None: + if not text.startswith("#!"): + return + while reader.peek() not in (None, "\n"): + reader.get() + + def read_source(text: str) -> tuple[list, str | None]: """Top-level forms, plus a warning if a later form could not be read.""" - if text.startswith("\ufeff"): - text = text[1:] + text = _strip_bom(text) reader = Reader(text) - if text.startswith("#!"): - while not reader.eof() and reader.peek() != "\n": - reader.get() + _skip_shebang(reader, text) forms: list = [] while True: reader.skip_ws() @@ -324,24 +347,26 @@ def max_line(form) -> int: return best +def _normalize_list(form): + if not form.items: + return [K("list"), K("literal")] + head_form, *args = form.items + return [K("list"), normalize(head_form, True), *[normalize(arg, False) for arg in args]] + + def normalize(form, head: bool = False): """dry4clj's `normalize-form`. Collection heads are normalized in full.""" if isinstance(form, Coll): if form.kind == "list": - if not form.items: - return [K("list"), K("literal")] - head_form, *args = form.items - return [K("list"), normalize(head_form, True), *[normalize(arg, False) for arg in args]] + return _normalize_list(form) if form.kind == "vector": return [K("vector"), *[normalize(item, False) for item in form.items]] if form.kind == "set": return [K("set"), *[normalize(item, False) for item in form.items]] if form.kind == "map": - return [ - K("map"), - *[[normalize(key, False), normalize(value, False)] for key, value in form.pairs], - ] + pairs = [[normalize(key, False), normalize(value, False)] for key, value in form.pairs] + return [K("map"), *pairs] return [K("literal")] if isinstance(form, Sym): if head: diff --git a/src/dryer/discover.py b/src/dryer/discover.py index 193b16e..6004797 100644 --- a/src/dryer/discover.py +++ b/src/dryer/discover.py @@ -79,6 +79,10 @@ def is_test_file(path: str | Path) -> bool: return any(part in TEST_DIRS for part in file_path.parts) +def _skipped_dir(name: str) -> bool: + return name in SKIP_DIRS or name in TEST_DIRS + + def _walk(root: Path) -> list[Path]: if not root.exists(): return [] @@ -86,9 +90,7 @@ def _walk(root: Path) -> list[Path]: return [root] found: list[Path] = [] for dirpath, dirnames, filenames in os.walk(root): - dirnames[:] = [ - name for name in dirnames if name not in SKIP_DIRS and name not in TEST_DIRS - ] + dirnames[:] = [name for name in dirnames if not _skipped_dir(name)] for name in filenames: path = Path(dirpath) / name if language_of(path) is None or is_test_file(path): diff --git a/src/dryer/report.py b/src/dryer/report.py index 8bc7a6a..25bf7a6 100644 --- a/src/dryer/report.py +++ b/src/dryer/report.py @@ -34,32 +34,32 @@ def _edn_float(value: float) -> str: return text +def _span_edn(span) -> str: + return ( + f"{{:file {_edn_string(span.file)}, " + f":start-line {span.start_line}, :end-line {span.end_line}}}" + ) + + +def _candidate_edn(candidate: Duplicate) -> str: + return "\n".join( + [ + f" {{:score {_edn_float(candidate.score)}", + f" :language {_edn_string(candidate.language)}", + f" :left {_span_edn(candidate.left)}", + f" :right {_span_edn(candidate.right)}", + f" :left-nodes {candidate.left_nodes}", + f" :right-nodes {candidate.right_nodes}}}", + ] + ) + + def render_edn(candidates: list[Duplicate]) -> str: """EDN map `{:candidates [...]}` with dry4clj's keys, plus `:language`.""" if not candidates: return "{:candidates []}\n" - rows = [] - for candidate in candidates: - left = candidate.left - right = candidate.right - rows.append( - " {:score " - + _edn_float(candidate.score) - + "\n :language " - + _edn_string(candidate.language) - + "\n :left {:file " - + _edn_string(left.file) - + f", :start-line {left.start_line}, :end-line {left.end_line}" - + "}" - + "\n :right {:file " - + _edn_string(right.file) - + f", :start-line {right.start_line}, :end-line {right.end_line}" - + "}" - + f"\n :left-nodes {candidate.left_nodes}" - + f"\n :right-nodes {candidate.right_nodes}" - + "}" - ) + rows = [_candidate_edn(candidate) for candidate in candidates] return "{:candidates [\n" + "\n".join(rows) + "\n]}\n" diff --git a/src/dryer/scan.py b/src/dryer/scan.py index 8ce54b0..f8996e2 100644 --- a/src/dryer/scan.py +++ b/src/dryer/scan.py @@ -41,6 +41,26 @@ def scan_files( return entries, warnings +def _same_span(left: Entry, right: Entry) -> bool: + if left.file != right.file: + return False + if left.start_line != right.start_line: + return False + return left.end_line == right.end_line + + +def _duplicate_key(item: Duplicate): + score = -item.score + return ( + score, + item.language, + item.left.file, + item.left.start_line, + item.right.file, + item.right.start_line, + ) + + def find_duplicates(entries: list[Entry], threshold: float) -> list[Duplicate]: """Pair forms of the same language. Each unordered pair is reported once.""" @@ -52,11 +72,7 @@ def find_duplicates(entries: list[Entry], threshold: float) -> list[Duplicate]: for language, group in by_language.items(): for index, left in enumerate(group): for right in group[index + 1 :]: - if ( - left.file == right.file - and left.start_line == right.start_line - and left.end_line == right.end_line - ): + if _same_span(left, right): continue score = jaccard(left.fingerprints, right.fingerprints) if score < threshold: @@ -71,14 +87,5 @@ def find_duplicates(entries: list[Entry], threshold: float) -> list[Duplicate]: right_nodes=right.nodes, ) ) - found.sort( - key=lambda item: ( - -item.score, - item.language, - item.left.file, - item.left.start_line, - item.right.file, - item.right.start_line, - ) - ) + found.sort(key=_duplicate_key) return found diff --git a/tests/test_cli.py b/tests/test_cli.py index 7d56b44..fd2f8e5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,7 +1,9 @@ from pathlib import Path -from dryer.cli import main, parse_args, run -from dryer.discover import language_of +import pytest + +from dryer.cli import _changed_files, _count, _tracked_source, main, parse_args, run, select_files +from dryer.discover import is_test_file, iter_source_files, language_of def write_source(root: Path, name: str, text: str) -> None: @@ -135,3 +137,187 @@ def test_main_exits(monkeypatch): monkeypatch.setattr("dryer.cli.sys.exit", lambda code: codes.append(code)) main(["--help"]) assert codes == [0] + + +def test_names_that_are_tests(): + names = [ + "pkg/foo_test.go", + "src/app_test.clj", + "src/app_test.cljc", + "src/app_test.cljs", + "src/app_test.cljd", + "src/app_test.bb", + "ui/a.test.ts", + "ui/a.spec.ts", + "ui/a.test.tsx", + "ui/a.spec.tsx", + "ui/a.test.mts", + "ui/a.spec.mts", + "tests/conftest.py", + "pkg/foo_test.py", + "pkg/test_foo.py", + "tests/app.py", + ] + for name in names: + assert is_test_file(name), name + assert not is_test_file("src/app.py") + + +class _Git: + def __init__(self, returncode, stdout, stderr=""): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +def test_changed_files_reads_porcelain(tmp_path, monkeypatch): + stdout = ' M src/a.py\nR old.py -> src/b.py\n?? "src/c d.py"\n\nX\n' + monkeypatch.setattr( + "dryer.cli.subprocess.run", + lambda *args, **kwargs: _Git(0, stdout), + ) + assert _changed_files(tmp_path) == [ + (tmp_path / "src/a.py").resolve(), + (tmp_path / "src/b.py").resolve(), + (tmp_path / "src/c d.py").resolve(), + ] + + +def test_changed_files_reports_git_failure(tmp_path, monkeypatch, capsys): + monkeypatch.setattr( + "dryer.cli.subprocess.run", + lambda *args, **kwargs: _Git(1, "", "fatal: not a git repository\n"), + ) + assert _changed_files(tmp_path) == [] + assert capsys.readouterr().err == "fatal: not a git repository\n" + + monkeypatch.setattr( + "dryer.cli.subprocess.run", + lambda *args, **kwargs: _Git(1, "", " \n"), + ) + assert _changed_files(tmp_path) == [] + assert capsys.readouterr().err == "git status failed\n" + + +def test_changed_limits_the_report(tmp_path, monkeypatch, capsys): + body = "def alpha(xs):\n ys = filter(xs, odd)\n zs = map(ys, inc)\n return list(zs)\n" + other = "def beta(items):\n kept = filter(items, even)\n out = map(kept, dec)\n return list(out)\n" + write_source(tmp_path, "src/a.py", body) + write_source(tmp_path, "src/b.py", other) + write_source(tmp_path, "src/c.py", body) + write_source(tmp_path, "tests/test_a.py", other) + stdout = " M src/a.py\n M src/b.py\n M tests/test_a.py\n ?? README.md\n" + monkeypatch.setattr( + "dryer.cli.subprocess.run", + lambda *args, **kwargs: _Git(0, stdout), + ) + assert run(["--root", str(tmp_path), "--changed", "--min-lines", "3", "--min-nodes", "1"]) == 0 + out = capsys.readouterr().out + assert "src/a.py" in out + assert "src/b.py" in out + assert "src/c.py" not in out + assert "test_a.py" not in out + + +def test_an_option_without_a_value_is_a_usage_error(capsys): + assert run(["--threshold"]) == 1 + assert "--threshold requires a value" in capsys.readouterr().err + + +def test_an_empty_value_is_rejected(capsys): + assert run(["file.py", "--min-lines", ""]) == 1 + assert "--min-lines requires a value" in capsys.readouterr().err + + +def test_a_flag_is_not_a_value(capsys): + assert run(["--min-lines", "--min-nodes", "1"]) == 1 + assert "--min-lines requires a value" in capsys.readouterr().err + + +def test_zero_is_a_valid_count(): + options = parse_args(["--min-lines", "0", "--min-nodes", "0"]) + assert options.action == "scan" + assert options.min_lines == 0 + assert options.min_nodes == 0 + + +def test_a_negative_count_is_rejected(): + with pytest.raises(ValueError, match="non-negative"): + _count("-1", "--min-lines") + + +def test_a_count_must_be_an_integer(capsys): + assert run(["--min-lines", "nope"]) == 1 + assert "--min-lines requires an integer" in capsys.readouterr().err + + +def test_a_threshold_must_be_a_number(capsys): + assert run(["--threshold", "nope"]) == 1 + assert "--threshold requires a number" in capsys.readouterr().err + + +def test_parse_reads_the_process_arguments(monkeypatch): + monkeypatch.setattr("dryer.cli.sys.argv", ["dryer", "only-this"]) + assert parse_args().positionals == ["only-this"] + + +def test_text_format_wins_when_it_comes_last(tmp_path, capsys): + write_source(tmp_path, "one.py", "def alpha(xs):\n return xs\n") + assert run(["--root", str(tmp_path), "--edn", "--text", "--min-lines", "1", "--min-nodes", "1"]) == 0 + assert capsys.readouterr().out == "No duplicate candidates found.\n" + + +def test_source_root_limits_the_walk(tmp_path): + write_source(tmp_path, "src/a.py", "x = 1\n") + write_source(tmp_path, "extra/b.py", "x = 1\n") + options = parse_args(["--root", str(tmp_path), "--source-root", "src"]) + files = [path.relative_to(tmp_path).as_posix() for path in select_files(options)] + assert files == ["src/a.py"] + + +def test_git_status_asks_for_text_without_checking(tmp_path, monkeypatch): + seen = {} + + def fake_run(args, **kwargs): + seen["args"] = args + seen["kwargs"] = kwargs + return _Git(0, " M src/a.py\n") + + monkeypatch.setattr("dryer.cli.subprocess.run", fake_run) + assert _changed_files(tmp_path) == [(tmp_path / "src/a.py").resolve()] + assert seen["args"] == ["git", "status", "--porcelain"] + assert seen["kwargs"]["check"] is False + assert seen["kwargs"]["capture_output"] is True + assert seen["kwargs"]["text"] is True + assert seen["kwargs"]["cwd"] == tmp_path + + +def test_a_status_line_of_four_characters_is_a_path(tmp_path, monkeypatch): + monkeypatch.setattr( + "dryer.cli.subprocess.run", + lambda *args, **kwargs: _Git(0, " M a\n"), + ) + assert _changed_files(tmp_path) == [(tmp_path / "a").resolve()] + + +def test_changed_source_skips_unknown_and_test_files(tmp_path): + assert _tracked_source(tmp_path / "README.md") is False + assert _tracked_source(tmp_path / "tests" / "test_a.py") is False + assert _tracked_source(tmp_path / "src" / "a.py") is True + + +def test_test_files_and_skipped_directories_are_left_out(tmp_path): + write_source(tmp_path, "src/app.py", "x = 1\n") + write_source(tmp_path, "src/foo_test.py", "x = 1\n") + write_source(tmp_path, "target/app.py", "x = 1\n") + write_source(tmp_path, "tests/app.py", "x = 1\n") + relative = [path.relative_to(tmp_path).as_posix() for path in iter_source_files([tmp_path])] + assert relative == ["src/app.py"] + + +def test_a_second_report_replaces_the_snapshot(tmp_path, capsys): + write_source(tmp_path, "one.py", "def alpha(xs):\n return xs\n") + args = ["--root", str(tmp_path), "--min-lines", "1", "--min-nodes", "1"] + assert run(args) == 0 + assert run(args) == 0 + assert (tmp_path / ".metrics" / "dry.edn").read_text(encoding="utf-8") == "{:candidates []}\n" diff --git a/tests/test_clojure.py b/tests/test_clojure.py index 4ba1937..89effff 100644 --- a/tests/test_clojure.py +++ b/tests/test_clojure.py @@ -1,8 +1,10 @@ from pathlib import Path -from dryer.report import format_text -from dryer.scan import find_duplicates, scan_files +from dryer.clojure_read import Coll, Sym, is_candidate_form, normalize, read_source from dryer.model import Duplicate, Span +from dryer.report import format_text, render_edn +from dryer.scan import find_duplicates, scan_files +from dryer.shape import K, node_count, pr def write_source(root: Path, name: str, text: str) -> None: @@ -23,6 +25,27 @@ def duplicates(root: Path, **options): return find_duplicates(entries, options.get("threshold", 0.82)) +def test_edn_report_matches_the_text_fields(): + candidate = Duplicate( + 0.875, + "clojure", + Span("a.clj", 10, 14), + Span("b.clj", 20, 24), + 30, + 31, + ) + assert render_edn([candidate]) == ( + "{:candidates [\n" + " {:score 0.875\n" + ' :language "clojure"\n' + ' :left {:file "a.clj", :start-line 10, :end-line 14}\n' + ' :right {:file "b.clj", :start-line 20, :end-line 24}\n' + " :left-nodes 30\n" + " :right-nodes 31}\n" + "]}\n" + ) + + def test_text_report_matches_dry4clj(): candidate = Duplicate( 0.875, @@ -185,3 +208,142 @@ def test_unreadable_tail_does_not_drop_an_earlier_form(tmp_path): assert warnings == ["left.clj:6: unterminated list"] found = find_duplicates(entries, 0.80) assert len(found) == 1 + + +def test_reads_escaped_strings_and_characters(): + forms, warning = read_source('(defn alpha [x]\n (str "a\\\\nb" \\newline \\a))\n') + assert warning is None + assert [form.items[0].name for form in forms] == ["defn"] + + +def test_reports_a_broken_string_or_character(): + _, warning = read_source('(defn alpha [x]\n "abc)\n') + assert warning == "2: unterminated string" + _, warning = read_source('(defn alpha [x] "abc\\') + assert warning == "1: unterminated string" + _, warning = read_source("(defn alpha [x] \\") + assert warning == "1: incomplete character" + + +def test_bom_and_shebang_are_not_forms(): + forms, warning = read_source("\ufeff#!/usr/bin/env bb\n(defn alpha [x]\n (inc x))\n") + assert warning is None + assert [form.items[0].name for form in forms] == ["defn"] + + +def test_reader_conditional_keeps_clj_and_splices_a_collection(): + forms, warning = read_source("#?(:cljs (inc x) :default (dec x))\n") + assert warning is None + assert forms[0].items[0].name == "dec" + + forms, warning = read_source("#?((inc x) (dec x))\n") + assert warning is None + assert forms == [] + + forms, warning = read_source("#?[:clj (inc x)]\n") + assert warning == "1: reader conditional body must be a list" + + forms, warning = read_source("(defn alpha []\n [#?@(:clj [:a :b])])\n") + assert warning is None + assert [item.name for item in forms[0].items[3].items] == ["a", "b"] + + forms, warning = read_source("(defn alpha [x]\n (inc x))\n#?@(:clj {:a 1})\n") + assert warning == "3: splicing reader conditional needs a collection" + assert [form.items[0].name for form in forms] == ["defn"] + + +def test_empty_list_normalizes_to_a_literal_list(): + forms, warning = read_source("(defn alpha [x]\n ())\n") + assert warning is None + assert "[:list :literal]" in pr(normalize(forms[0])) + + +def test_reader_macros_wrap_the_following_form(): + forms, warning = read_source("'x\n@x\n`x\n~x\n~@x\n") + assert warning is None + assert [form.items[0].name for form in forms] == [ + "quote", + "deref", + "syntax-quote", + "unquote", + "unquote-splicing", + ] + assert forms[0].items[1].name == "x" + assert forms[4].items[1].name == "x" + forms, warning = read_source("'#_x\n") + assert warning is None + assert [item.name for item in forms[0].items] == ["quote"] + + +def test_a_character_name_is_consumed(): + forms, warning = read_source("(defn alpha []\n (str \\newline))\n") + assert warning is None + assert [type(item).__name__ for item in forms[0].items[3].items] == ["Sym", "Lit"] + + +def test_a_bare_colon_is_an_empty_keyword(): + forms, warning = read_source(":\n") + assert warning is None + assert forms[0].name == "" + forms, warning = read_source("#!only") + assert warning is None + assert forms == [] + + +def test_namespaced_maps_keep_the_map(): + for source in ("#:foo{:a 1}\n", "#::{:a 1}\n", "#:{:a 1}\n"): + forms, warning = read_source(source) + assert warning is None, source + assert forms[0].kind == "map" + assert forms[0].pairs[0][0].name == "a" + + +def test_map_pairs_keep_source_order(): + forms, warning = read_source("{:a 1 :b 2}\n") + assert warning is None + assert [pair[0].name for pair in forms[0].pairs] == ["a", "b"] + left, warning = read_source("{foo 1}\n") + right, _ = read_source("{bar 2}\n") + assert warning is None + assert "foo" not in pr(normalize(left[0])) + assert pr(normalize(left[0])) == pr(normalize(right[0])) + paired, warning = read_source("{:a 1\n :b 2}\n") + assert warning is None + assert [value.line for _key, value in paired[0].pairs] == [1, 2] + symbols, warning = read_source("{foo bar}\n") + assert warning is None + printed = pr(normalize(symbols[0])) + assert "foo" not in printed + assert "bar" not in printed + + +def test_a_trailing_feature_without_an_expression_is_ignored(): + forms, warning = read_source("#?(:clj (inc x) :extra)\n") + assert warning is None + assert forms[0].items[0].name == "inc" + forms, warning = read_source("#?(:cljs (dec x) :extra)\n") + assert warning is None + assert forms == [] + + +def test_names_inside_a_set_are_erased(): + forms, warning = read_source("#{foo bar}\n") + assert warning is None + printed = pr(normalize(forms[0])) + assert printed.startswith("[:set ") + assert "foo" not in printed + + +def test_only_a_non_empty_list_that_is_not_ns_is_a_candidate(): + assert is_candidate_form(Coll("vector", 1, items=[Sym("a", 1)])) is False + assert is_candidate_form(Coll("list", 1, items=[])) is False + assert is_candidate_form(Sym("a", 1)) is False + assert is_candidate_form(Coll("list", 1, items=[Sym("ns", 1), Sym("demo", 1)])) is False + assert is_candidate_form(Coll("list", 1, items=[Sym("defn", 1)])) is True + assert pr(normalize(Coll("bag", 1))) == "[:literal]" + + +def test_node_count_and_keyword_printing(): + assert repr(K("symbol")) == ":symbol" + assert node_count(K("symbol")) == 1 + assert node_count([K("list"), K("symbol"), K("literal")]) == 4 diff --git a/tests/test_languages.py b/tests/test_languages.py index 6e31f5f..6e1bcf5 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -1,6 +1,9 @@ from pathlib import Path -from dryer.scan import find_duplicates, scan_files +from dryer.extract import entries_in_source +from dryer.model import Entry +from dryer.scan import _keep, find_duplicates, scan_files +from dryer.treesitter import descendants, end_line, parse def write_source(root: Path, name: str, text: str) -> None: @@ -231,3 +234,368 @@ class Right { found = find_duplicates(entries, threshold=0.5) assert len(found) == 1 assert found[0].score < 1.0 + + +def _suffix(language: str) -> str: + return {"python": "py", "go": "go", "rust": "rs", "java": "java", "typescript": "ts"}[language] + + +def test_callee_names_stay_and_member_names_do_not(tmp_path): + samples = { + "left.py": """\ +def alpha(xs): + ys = xs.filter(odd) + return (len)(xs.total) +""", + "right.py": """\ +def beta(items): + kept = items.filter(even) + return (len)(items.count) +""", + "left.go": """\ +package demo + +func alpha(xs []int) { + fmt.Println(xs) +} +""", + "right.go": """\ +package demo + +func beta(items []int) { + log.Println(items) +} +""", + "left.rs": """\ +pub fn alpha(xs: Vec) -> Vec { + let ys = Vec::new(); + let zs = HashMap::::new(); + xs.iter() +} +""", + "right.rs": """\ +pub fn beta(items: Vec) -> Vec { + let kept = Vec::new(); + let rows = HashMap::::new(); + items.iter() +} +""", + "left.java": """\ +class Left { + int alpha(List xs) { + List ys = new ArrayList(); + ys.add(xs); + return xs.total; + } +} +""", + "right.java": """\ +class Right { + int beta(List items) { + List kept = new ArrayList(); + kept.add(items); + return items.count; + } +} +""", + "left.ts": """\ +export function alpha(xs: number[]): number { + return foo(xs.bar); +} +""", + "right.ts": """\ +export function beta(items: number[]): number { + return foo(items.baz); +} +""", + } + for name, source in samples.items(): + write_source(tmp_path, name, source) + found = pairs(tmp_path) + by_language = {} + for item in found: + by_language.setdefault(item.language, []).append(item) + assert set(by_language) == {"python", "go", "rust", "java", "typescript"} + for language, group in by_language.items(): + assert [(item.left.file, item.right.file, item.score) for item in group] == [ + (f"left.{_suffix(language)}", f"right.{_suffix(language)}", 1.0) + ], language + + +def test_a_different_callee_name_lowers_the_score(tmp_path): + write_source(tmp_path, "a.py", "def alpha(xs):\n return xs.filter(odd)\n") + write_source(tmp_path, "b.py", "def beta(items):\n return items.select(even)\n") + write_source(tmp_path, "a.rs", "fn alpha() {\n Vec::new()\n}\n") + write_source(tmp_path, "b.rs", "fn beta() {\n Vec::with_capacity(1)\n}\n") + files = sorted(path for path in tmp_path.rglob("*") if path.is_file()) + entries, warnings = scan_files(files, tmp_path, min_lines=1, min_nodes=1) + assert warnings == [] + found = find_duplicates(entries, threshold=0.0) + assert len(found) == 2 + assert all(item.score < 1.0 for item in found) + + +def forms(language: str, source: str, file: str) -> list[Entry]: + entries, warning = entries_in_source(language, source, file, file) + assert warning is None + return entries + + +def spans(language: str, source: str, file: str) -> list[tuple[int, int]]: + return [(entry.start_line, entry.end_line) for entry in forms(language, source, file)] + + +def test_a_parenthesized_callee_matches_the_bare_call(): + wrapped = forms("python", "def alpha(xs):\n return (len)(xs)\n", "a.py") + bare = forms("python", "def beta(items):\n return len(items)\n", "b.py") + assert any("len" in item for item in wrapped[0].fingerprints) + found = find_duplicates(wrapped + bare, threshold=0.99) + assert len(found) == 1 + assert found[0].score == 1.0 + + +def test_a_string_and_a_number_normalize_to_the_same_literal(): + string = forms("python", 'def alpha(xs):\n return "xs"\n', "a.py") + number = forms("python", "def beta(items):\n return 1\n", "b.py") + assert any(item == ":literal" for item in number[0].fingerprints) + found = find_duplicates(string + number, threshold=0.99) + assert len(found) == 1 + assert found[0].score == 1.0 + + +def test_an_interpolated_string_is_not_a_plain_literal(): + plain = forms("python", 'def alpha(xs):\n return "xs"\n', "a.py") + interpolated = forms("python", 'def beta(items):\n return f"{items}"\n', "b.py") + found = find_duplicates(plain + interpolated, threshold=0.0) + assert len(found) == 1 + assert found[0].score < 1.0 + + +def test_constructed_names_stay_in_the_fingerprint(): + rust = forms( + "rust", + """\ +pub fn alpha() { + let ys = Vec::new(); + let zs = HashMap::::new(); +} +""", + "a.rs", + ) + java = forms( + "java", + """\ +class Left { + int alpha() { + return new ArrayList(); + } +} +""", + "a.java", + ) + rust_names = " ".join(rust[0].fingerprints) + java_names = " ".join(java[0].fingerprints) + assert "Vec" in rust_names + assert "new" in rust_names + assert "HashMap" in rust_names + assert "ArrayList" in java_names + + +def test_a_class_contributes_its_method_only(): + assert spans( + "python", + "class Board:\n def alpha(self, xs):\n return xs\n", + "a.py", + ) == [(2, 3)] + + +def test_java_keeps_the_method_that_has_its_own_body(): + assert spans( + "java", + """\ +abstract class Left { + abstract int declared(int[] xs); + int alpha(int[] xs) { + class Inner { + int run(int[] ys) { + return ys.length; + } + } + return xs.length; + } +} +""", + "a.java", + ) == [(3, 10)] + + +def test_a_go_function_needs_a_body(): + assert spans( + "go", + """\ +package demo + +func declared(xs []int) + +func alpha(xs []int) int { + return len(xs) +} +""", + "a.go", + ) == [(5, 7)] + + +def test_a_closure_in_a_let_stays_inside_the_function(): + assert spans( + "rust", + """\ +fn alpha() { + let f = || { let x = 1; }; +} +""", + "a.rs", + ) == [(1, 3)] + + +def test_a_method_nested_in_a_function_is_not_its_own_candidate(): + assert spans( + "typescript", + """\ +export function outer(xs: number[]): number[] { + class Hidden { + run(ys: number[]): number[] { + return ys; + } + } + return xs; +} +""", + "a.ts", + ) == [(1, 8)] + + +def test_rust_keeps_helpers_and_skips_tests_and_signatures(): + assert spans( + "rust", + """\ +fn declared(); + +fn alpha(xs: Vec) -> usize { + xs.len() +} + +mod helpers { + fn beta(xs: Vec) -> usize { + xs.len() + } +} + +mod tests { + fn gamma(xs: Vec) -> usize { + xs.len() + } +} +""", + "a.rs", + ) == [(3, 5), (8, 10)] + + +def test_typescript_keeps_methods_and_skips_nested_functions(): + assert spans( + "typescript", + """\ +export class Box { + run(xs: number[]): number[] { + return xs; + } +} +export function outer(xs: number[]): number[] { + function inner(ys: number[]): number[] { + return ys; + } + const cb = (zs: number[]) => zs; + return inner(xs); +} +xs.map((item: number) => item + 1); +""", + "a.ts", + ) == [(2, 4), (6, 12)] + + +def test_a_private_field_name_is_not_part_of_the_structure(): + entries = forms( + "typescript", + """\ +export class Box { + value(): number { + return this.#total; + } + run(xs: number[]): number[] { + return this.#hidden(xs); + } +} +""", + "a.ts", + ) + assert [(entry.start_line, entry.end_line) for entry in entries] == [(2, 4), (5, 7)] + printed = " ".join(item for entry in entries for item in entry.fingerprints) + assert "#total" not in printed + assert "#hidden" not in printed + + +def test_end_line_uses_the_last_occupied_row(): + source = "def alpha(xs):\n return xs\n" + _data, tree = parse(source, "python") + function = next(node for node in descendants(tree.root_node) if node.type == "function_definition") + assert end_line(tree.root_node) == 2 + assert end_line(function) == 2 + assert spans("python", source, "a.py") == [(1, 2)] + + +def _entry(file: str, start: int, end: int, names: set[str], language: str = "python", nodes: int = 20): + return Entry(language, file, start, end, nodes, frozenset(names)) + + +def test_the_same_span_is_not_a_pair(): + assert find_duplicates([ + _entry("a.py", 1, 4, {"q"}), + _entry("a.py", 1, 4, {"q"}), + ], 0.0) == [] + assert len(find_duplicates([ + _entry("a.py", 1, 4, {"q"}), + _entry("b.py", 1, 4, {"q"}), + ], 0.5)) == 1 + assert len(find_duplicates([ + _entry("a.py", 1, 5, {"q"}), + _entry("a.py", 3, 5, {"q"}), + ], 0.5)) == 1 + assert len(find_duplicates([ + _entry("a.py", 1, 4, {"q"}), + _entry("a.py", 1, 8, {"q"}), + ], 0.5)) == 1 + + +def test_a_score_equal_to_the_threshold_is_kept(): + left = _entry("a.rs", 1, 4, {"x", "y"}, language="rust") + right = _entry("b.rs", 1, 4, {"x", "z"}, language="rust") + assert len(find_duplicates([left, right], 1 / 3)) == 1 + assert find_duplicates([left, right], 1 / 3 + 0.01) == [] + + +def test_the_higher_score_is_reported_first(): + exact = [ + _entry("a.py", 1, 4, {"only"}), + _entry("b.py", 1, 4, {"only"}), + ] + partial = [ + _entry("a.rs", 1, 4, {"x", "y"}, language="rust"), + _entry("b.rs", 10, 14, {"x", "z"}, language="rust"), + ] + found = find_duplicates(partial + exact, 0.3) + assert [item.score for item in found] == [1.0, 1 / 3] + + +def test_keep_uses_the_line_and_node_minimums(): + assert _keep(_entry("a.py", 1, 4, {"q"}, nodes=20), 4, 20) is True + assert _keep(_entry("a.py", 1, 3, {"q"}, nodes=20), 4, 20) is False + assert _keep(_entry("a.py", 1, 4, {"q"}, nodes=19), 4, 20) is False