Add a multi-language duplicate finder that writes .metrics/dry.edn.

tests/test_cli.py

@@ -0,0 +1,137 @@
+from pathlib import Path
+
+from dryer.cli import main, parse_args, run
+from dryer.discover import language_of
+
+
+def write_source(root: Path, name: str, text: str) -> None:
+    path = root / name
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+
+
+def test_language_detection():
+    assert language_of("src/app/core.clj") == "clojure"
+    assert language_of("src/app/core.cljd") == "clojure"
+    assert language_of("Widget.java") == "java"
+    assert language_of("board.go") == "go"
+    assert language_of("ui/view.tsx") == "typescript"
+    assert language_of("src/lib.rs") == "rust"
+    assert language_of("src/dryer/cli.py") == "python"
+    assert language_of("types.d.ts") is None
+    assert language_of("notes.md") is None
+
+
+def test_help_does_not_scan(capsys):
+    assert run(["--help"]) == 0
+    out = capsys.readouterr().out
+    assert "Usage: dryer" in out
+    assert "--threshold" in out
+
+
+def test_unknown_format(capsys):
+    assert run(["--format", "csv"]) == 2
+    err = capsys.readouterr().err
+    assert err == "Unknown format: csv\n"
+
+
+def test_unknown_option(capsys):
+    assert run(["--nope"]) == 1
+    assert "Unknown option: --nope" in capsys.readouterr().err
+
+
+def test_parse_defaults_and_flags():
+    options = parse_args(
+        ["--threshold", "0.9", "--min-lines", "5", "--min-nodes", "30", "--edn", "spec"]
+    )
+    assert options.threshold == 0.9
+    assert options.min_lines == 5
+    assert options.min_nodes == 30
+    assert options.format == "edn"
+    assert options.positionals == ["spec"]
+
+
+def test_project_report_and_snapshot(tmp_path, capsys):
+    write_source(
+        tmp_path,
+        "src/left.py",
+        "def alpha(xs):\n    ys = filter(xs, odd)\n    zs = map(ys, inc)\n    return list(zs)\n",
+    )
+    write_source(
+        tmp_path,
+        "src/right.py",
+        "def beta(items):\n    kept = filter(items, even)\n    out = map(kept, dec)\n    return list(out)\n",
+    )
+    write_source(
+        tmp_path,
+        "tests/test_left.py",
+        "def alpha(xs):\n    ys = filter(xs, odd)\n    zs = map(ys, inc)\n    return list(zs)\n",
+    )
+    code = run(["--root", str(tmp_path), "--min-lines", "3", "--min-nodes", "1"])
+    captured = capsys.readouterr()
+    assert code == 0
+    assert "DUPLICATE score=1.00" in captured.out
+    assert "src/left.py:1-" in captured.out
+    assert "src/right.py:1-" in captured.out
+    assert "test_left.py" not in captured.out
+    text = (tmp_path / ".metrics" / "dry.edn").read_text(encoding="utf-8")
+    assert ':language "python"' in text
+    assert ':file "src/left.py"' in text
+    assert "Wrote" in captured.err
+
+
+def test_edn_stdout(tmp_path, capsys):
+    write_source(tmp_path, "one.py", "def alpha(xs):\n    return xs\n")
+    assert run(["--root", str(tmp_path), "--edn", "--min-lines", "1", "--min-nodes", "1"]) == 0
+    out = capsys.readouterr().out
+    assert out == "{:candidates []}\n"
+
+
+def test_empty_project(tmp_path, capsys):
+    assert run(["--root", str(tmp_path)]) == 0
+    assert capsys.readouterr().out == "No source files to analyze.\n"
+
+
+def test_path_filter(tmp_path, capsys):
+    body_a = "def alpha(xs):\n    ys = filter(xs, odd)\n    zs = map(ys, inc)\n    return list(zs)\n"
+    body_b = "def beta(items):\n    kept = filter(items, even)\n    out = map(kept, dec)\n    return list(out)\n"
+    write_source(tmp_path, "src/board/a.py", body_a)
+    write_source(tmp_path, "src/board/b.py", body_b)
+    write_source(tmp_path, "src/other/a.py", body_a)
+    write_source(tmp_path, "src/other/b.py", body_b)
+    assert run(["--root", str(tmp_path), "--min-lines", "3", "--min-nodes", "1", "board"]) == 0
+    out = capsys.readouterr().out
+    assert "src/board/a.py" in out
+    assert "src/other/a.py" not in out
+
+
+def test_explicit_test_file_is_included(tmp_path, capsys):
+    body = "def alpha(xs):\n    ys = filter(xs, odd)\n    zs = map(ys, inc)\n    return list(zs)\n"
+    other = "def beta(items):\n    kept = filter(items, even)\n    out = map(kept, dec)\n    return list(out)\n"
+    write_source(tmp_path, "src/a.py", body)
+    write_source(tmp_path, "tests/test_a.py", other)
+    assert (
+        run(
+            [
+                "--root",
+                str(tmp_path),
+                "--min-lines",
+                "3",
+                "--min-nodes",
+                "1",
+                "src/a.py",
+                "tests/test_a.py",
+            ]
+        )
+        == 0
+    )
+    out = capsys.readouterr().out
+    assert "src/a.py" in out
+    assert "tests/test_a.py" in out
+
+
+def test_main_exits(monkeypatch):
+    codes = []
+    monkeypatch.setattr("dryer.cli.sys.exit", lambda code: codes.append(code))
+    main(["--help"])
+    assert codes == [0]

tests/test_clojure.py

@@ -0,0 +1,187 @@
+from pathlib import Path
+
+from dryer.report import format_text
+from dryer.scan import find_duplicates, scan_files
+from dryer.model import Duplicate, Span
+
+
+def write_source(root: Path, name: str, text: str) -> None:
+    path = root / name
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+
+
+def duplicates(root: Path, **options):
+    files = sorted(path for path in root.rglob("*") if path.is_file())
+    entries, warnings = scan_files(
+        files,
+        root,
+        options.get("min_lines", 4),
+        options.get("min_nodes", 20),
+    )
+    assert warnings == []
+    return find_duplicates(entries, options.get("threshold", 0.82))
+
+
+def test_text_report_matches_dry4clj():
+    candidate = Duplicate(
+        0.875,
+        "clojure",
+        Span("a.clj", 10, 14),
+        Span("b.clj", 20, 24),
+        30,
+        31,
+    )
+    assert (
+        format_text([candidate])
+        == "DUPLICATE score=0.88\n  a.clj:10-14\n  b.clj:20-24\n"
+    )
+    assert format_text([]) == "No duplicate candidates found.\n"
+
+
+def test_reports_structural_duplicates_with_line_ranges(tmp_path):
+    write_source(
+        tmp_path,
+        "left.cljc",
+        "(ns sample.left)\n\n(defn alpha [xs]\n  (let [ys (filter odd? xs)]\n    (map inc ys)))\n",
+    )
+    write_source(
+        tmp_path,
+        "right.cljc",
+        "(ns sample.right)\n\n(defn beta [items]\n  (let [kept (filter even? items)]\n    (map dec kept)))\n",
+    )
+    found = duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=8)
+    assert len(found) == 1
+    assert found[0].score == 1.0
+    assert found[0].language == "clojure"
+    assert found[0].left.file == "left.cljc"
+    assert (found[0].left.start_line, found[0].left.end_line) == (3, 5)
+    assert found[0].right.file == "right.cljc"
+    assert (found[0].right.start_line, found[0].right.end_line) == (3, 5)
+
+
+def test_matches_maps_sets_and_keyword_calls(tmp_path):
+    write_source(
+        tmp_path,
+        "left.cljc",
+        "(ns sample.left)\n\n(defn gamma [m]\n  (when (#{:a :b} (:kind m))\n    {:left (:a m) :right (:b m)}))\n",
+    )
+    write_source(
+        tmp_path,
+        "right.cljc",
+        "(ns sample.right)\n\n(defn delta [row]\n  (when (#{:c :d} (:kind row))\n    {:left (:c row) :right (:d row)}))\n",
+    )
+    found = duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=8)
+    assert len(found) == 1
+    assert found[0].score == 1.0
+
+
+def test_reads_cljc_reader_conditionals(tmp_path):
+    write_source(
+        tmp_path,
+        "left.cljc",
+        "(ns sample.left)\n\n(defn alpha [x]\n  #?(:clj (when (pos? x)\n            (inc x))))\n",
+    )
+    write_source(
+        tmp_path,
+        "right.cljc",
+        "(ns sample.right)\n\n(defn beta [y]\n  #?(:clj (when (pos? y)\n            (inc y))))\n",
+    )
+    found = duplicates(tmp_path, threshold=0.50, min_lines=1, min_nodes=1)
+    assert len(found) == 1
+    assert found[0].score == 1.0
+
+
+def test_filters_forms_shorter_than_the_minimum_line_count(tmp_path):
+    write_source(tmp_path, "one.clj", "(ns one)\n(defn a [x] (+ x 1))\n")
+    write_source(tmp_path, "two.clj", "(ns two)\n(defn b [y] (+ y 2))\n")
+    assert duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=1) == []
+
+
+def test_metadata_comments_and_discards_are_not_structure(tmp_path):
+    write_source(
+        tmp_path,
+        "left.clj",
+        "(ns left)\n\n(defn alpha [xs] ; keep\n  #_(println xs)\n  (let [ys (filter odd? xs)]\n    (map inc ys)))\n",
+    )
+    write_source(
+        tmp_path,
+        "right.clj",
+        "(ns right)\n\n(defn ^String beta [items]\n  (let [kept (filter even? items)]\n    (map dec kept)))\n",
+    )
+    found = duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=8)
+    assert len(found) == 1
+    assert found[0].score == 1.0
+
+
+def test_extra_binding_scores_below_one(tmp_path):
+    write_source(
+        tmp_path,
+        "invoice.clj",
+        """(ns billing.invoice)
+
+(defn invoice-summary [orders]
+  (let [paid (filter paid? orders), domestic (filter domestic? paid)
+        sorted (sort-by :date domestic), amounts (map :amount sorted)
+        taxes (map tax amounts), ids (map :id sorted)
+        customers (map :customer sorted), regions (group-by :region sorted)
+        flagged (filter flagged? sorted)]
+    {:count (count sorted)
+     :first-id (first ids), :last-id (last ids)
+     :customers (set customers), :regions (keys regions)
+     :flagged (count flagged), :total (reduce + 0 amounts)
+     :tax (reduce + 0 taxes)}))
+""",
+    )
+    write_source(
+        tmp_path,
+        "receipt.clj",
+        """(ns billing.receipt)
+
+(defn receipt-summary [rows]
+  (let [closed (filter closed? rows), local (filter local? closed)
+        ordered (sort-by :date local), amounts (map :amount ordered)
+        taxable (filter taxable? ordered), taxes (map tax amounts)
+        ids (map :id ordered), customers (map :customer ordered)
+        regions (group-by :region ordered)
+        flagged (filter flagged? ordered)]
+    {:count (count ordered)
+     :first-id (first ids), :last-id (last ids)
+     :customers (set customers), :regions (keys regions)
+     :flagged (count flagged), :total (reduce + 0 amounts)
+     :tax (reduce + 0 taxes)}))
+""",
+    )
+    found = duplicates(tmp_path)
+    assert len(found) == 1
+    assert 0.85 <= found[0].score < 1.0
+
+
+def test_three_copies_make_three_pairs(tmp_path):
+    body = "(let [ys (filter odd? xs)]\n    (map inc ys))"
+    write_source(
+        tmp_path,
+        "same.clj",
+        f"(ns same)\n\n(defn a [xs]\n  {body})\n\n(defn b [xs]\n  {body})\n\n(defn c [xs]\n  {body})\n",
+    )
+    found = duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=8)
+    assert len(found) == 3
+    assert {item.score for item in found} == {1.0}
+
+
+def test_unreadable_tail_does_not_drop_an_earlier_form(tmp_path):
+    write_source(
+        tmp_path,
+        "left.clj",
+        "(ns left)\n\n(defn alpha [xs]\n  (let [ys (filter odd? xs)]\n    (map inc ys)))\n(defn broken [x]\n",
+    )
+    write_source(
+        tmp_path,
+        "right.clj",
+        "(ns right)\n\n(defn beta [items]\n  (let [kept (filter even? items)]\n    (map dec kept)))\n",
+    )
+    files = sorted(tmp_path.glob("*.clj"))
+    entries, warnings = scan_files(files, tmp_path, 3, 8)
+    assert warnings == ["left.clj:6: unterminated list"]
+    found = find_duplicates(entries, 0.80)
+    assert len(found) == 1

tests/test_languages.py

@@ -0,0 +1,233 @@
+from pathlib import Path
+
+from dryer.scan import find_duplicates, scan_files
+
+
+def write_source(root: Path, name: str, text: str) -> None:
+    path = root / name
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+
+
+def pairs(root: Path):
+    files = sorted(path for path in root.rglob("*") if path.is_file())
+    entries, warnings = scan_files(files, root, min_lines=1, min_nodes=1)
+    assert warnings == []
+    return find_duplicates(entries, threshold=0.99)
+
+
+JAVA_LEFT = """\
+class Left {
+  int alpha(int[] xs) {
+    int[] ys = filter(xs, odd);
+    return map(ys, inc);
+  }
+}
+"""
+
+JAVA_RIGHT = """\
+class Right {
+  int beta(int[] items) {
+    int[] kept = filter(items, even);
+    return map(kept, dec);
+  }
+}
+"""
+
+GO_LEFT = """\
+package demo
+
+func (b *Board) alpha(xs []int) []int {
+	ys := filter(xs, odd)
+	return mapFn(ys, inc)
+}
+"""
+
+GO_RIGHT = """\
+package demo
+
+func (p *Panel) beta(items []int) []int {
+	kept := filter(items, even)
+	return mapFn(kept, dec)
+}
+"""
+
+PY_LEFT = """\
+class Board:
+    def alpha(self, xs):
+        ys = filter(xs, odd)
+        return map(ys, inc)
+"""
+
+PY_RIGHT = """\
+class Panel:
+    def beta(self, items):
+        kept = filter(items, even)
+        return map(kept, dec)
+"""
+
+TS_LEFT = """\
+export function alpha(xs: number[]): number[] {
+  const ys = filter(xs, odd);
+  return map(ys, inc);
+}
+"""
+
+TS_RIGHT = """\
+export function beta(items: number[]): number[] {
+  const kept = filter(items, even);
+  return map(kept, dec);
+}
+"""
+
+TSX_LEFT = """\
+export const alpha = (xs: number[]): number[] => {
+  const ys = filter(xs, odd);
+  return map(ys, inc);
+};
+"""
+
+TSX_RIGHT = """\
+export const beta = (items: number[]): number[] => {
+  const kept = filter(items, even);
+  return map(kept, dec);
+};
+"""
+
+RUST_LEFT = """\
+pub fn alpha(xs: Vec<i32>) -> Vec<i32> {
+    let ys = filter(xs, odd);
+    map(ys, inc)
+}
+"""
+
+RUST_RIGHT = """\
+pub fn beta(items: Vec<i32>) -> Vec<i32> {
+    let kept = filter(items, even);
+    map(kept, dec)
+}
+"""
+
+
+def test_each_language_matches_renamed_locals(tmp_path):
+    samples = {
+        "left.java": JAVA_LEFT,
+        "right.java": JAVA_RIGHT,
+        "left.go": GO_LEFT,
+        "right.go": GO_RIGHT,
+        "left.py": PY_LEFT,
+        "right.py": PY_RIGHT,
+        "left.ts": TS_LEFT,
+        "right.ts": TS_RIGHT,
+        "left.tsx": TSX_LEFT,
+        "right.tsx": TSX_RIGHT,
+        "left.rs": RUST_LEFT,
+        "right.rs": RUST_RIGHT,
+    }
+    for name, source in samples.items():
+        write_source(tmp_path, name, source)
+    found = pairs(tmp_path)
+    by_language = {}
+    for item in found:
+        by_language.setdefault(item.language, []).append(item)
+    assert set(by_language) == {"java", "go", "python", "typescript", "rust"}
+    assert [(item.left.file, item.right.file) for item in by_language["typescript"]] == [
+        ("left.ts", "right.ts"),
+        ("left.tsx", "right.tsx"),
+    ]
+    for language, group in by_language.items():
+        assert all(item.score == 1.0 for item in group), language
+
+
+def test_different_callee_is_not_the_same_structure(tmp_path):
+    write_source(
+        tmp_path,
+        "a.py",
+        "def alpha(xs):\n    return filter(xs, odd)\n",
+    )
+    write_source(
+        tmp_path,
+        "b.py",
+        "def beta(xs):\n    return select(xs, odd)\n",
+    )
+    files = sorted(tmp_path.glob("*.py"))
+    entries, warnings = scan_files(files, tmp_path, min_lines=1, min_nodes=1)
+    assert warnings == []
+    found = find_duplicates(entries, threshold=0.0)
+    assert len(found) == 1
+    assert found[0].score < 1.0
+
+
+def test_nested_functions_stay_inside_the_enclosing_function(tmp_path):
+    write_source(
+        tmp_path,
+        "a.py",
+        """\
+def alpha(xs):
+    def inner(ys):
+        return filter(ys, odd)
+    return inner(xs)
+""",
+    )
+    write_source(
+        tmp_path,
+        "b.py",
+        """\
+def beta(items):
+    def inner(kept):
+        return filter(kept, even)
+    return inner(items)
+""",
+    )
+    found = pairs(tmp_path)
+    assert len(found) == 1
+    assert found[0].score == 1.0
+    assert found[0].left.start_line == 1
+
+
+def test_rust_tests_module_is_not_production_code(tmp_path):
+    write_source(
+        tmp_path,
+        "a.rs",
+        """\
+pub fn alpha(xs: Vec<i32>) -> Vec<i32> {
+    let ys = filter(xs, odd);
+    map(ys, inc)
+}
+
+mod tests {
+    pub fn helper(xs: Vec<i32>) -> Vec<i32> {
+        let ys = filter(xs, odd);
+        map(ys, inc)
+    }
+}
+""",
+    )
+    write_source(tmp_path, "b.rs", RUST_RIGHT)
+    found = pairs(tmp_path)
+    assert len(found) == 1
+    assert found[0].left.file == "a.rs"
+    assert found[0].left.start_line == 1
+
+
+def test_an_extra_statement_lowers_the_score(tmp_path):
+    write_source(tmp_path, "a.java", JAVA_LEFT)
+    write_source(
+        tmp_path,
+        "b.java",
+        """\
+class Right {
+  int beta(int[] items) {
+    int[] kept = filter(items, even);
+    int[] extra = filter(kept, even);
+    return map(extra, dec);
+  }
+}
+""",
+    )
+    files = sorted(tmp_path.glob("*.java"))
+    entries, warnings = scan_files(files, tmp_path, min_lines=1, min_nodes=1)
+    assert warnings == []
+    found = find_duplicates(entries, threshold=0.5)
+    assert len(found) == 1
+    assert found[0].score < 1.0