From fcf865759dafabcd3082426e05fa29f47522da16 Mon Sep 17 00:00:00 2001 From: "Robert C. Martin" Date: Wed, 30 Sep 2026 14:37:25 -0500 Subject: [PATCH] Add a multi-language duplicate finder that writes .metrics/dry.edn. --- .gitignore | 8 + README.md | 115 ++++++++++++ dryer | 9 + pyproject.toml | 27 +++ src/dryer/__init__.py | 3 + src/dryer/__main__.py | 4 + src/dryer/astnorm.py | 263 +++++++++++++++++++++++++++ src/dryer/cli.py | 270 ++++++++++++++++++++++++++++ src/dryer/clojure_read.py | 361 ++++++++++++++++++++++++++++++++++++++ src/dryer/discover.py | 105 +++++++++++ src/dryer/extract.py | 197 +++++++++++++++++++++ src/dryer/model.py | 34 ++++ src/dryer/report.py | 74 ++++++++ src/dryer/scan.py | 84 +++++++++ src/dryer/shape.py | 59 +++++++ src/dryer/treesitter.py | 47 +++++ tests/test_cli.py | 137 +++++++++++++++ tests/test_clojure.py | 187 ++++++++++++++++++++ tests/test_languages.py | 233 ++++++++++++++++++++++++ 19 files changed, 2217 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100755 dryer create mode 100644 pyproject.toml create mode 100644 src/dryer/__init__.py create mode 100644 src/dryer/__main__.py create mode 100644 src/dryer/astnorm.py create mode 100644 src/dryer/cli.py create mode 100644 src/dryer/clojure_read.py create mode 100644 src/dryer/discover.py create mode 100644 src/dryer/extract.py create mode 100644 src/dryer/model.py create mode 100644 src/dryer/report.py create mode 100644 src/dryer/scan.py create mode 100644 src/dryer/shape.py create mode 100644 src/dryer/treesitter.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_clojure.py create mode 100644 tests/test_languages.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a588dbd --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.venv/ +__pycache__/ +.pytest_cache/ +*.egg-info/ +.metrics/ +target/ +dist/ +build/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..fae6e64 --- /dev/null +++ b/README.md @@ -0,0 +1,115 @@ +# dryer + +dryer finds candidate duplicate code in Clojure, Java, Go, TypeScript, Rust, and Python. One run detects the language of each source file, normalizes that language's forms, and reports pairs whose structure is close enough to review. + +Clojure follows [dry4clj](https://github.com/unclebob/dry4clj). The other languages use the same idea on the functions and methods [crapper](https://github.com/unclebob/crapper) and [mutator](https://github.com/unclebob/mutator) already treat as one unit. A form is only compared with forms written in the same language. + +The score is Jaccard similarity of structural fingerprints: + +```text +score = shared fingerprints / all fingerprints seen in either form +``` + +A score of `1.0` means the normalized structures have the same fingerprint set. The default `--threshold 0.82` reports candidates close enough to be worth review. + +These two functions score `1.0`. The names, locals, predicates, and mapped functions differ, and those incidental symbols normalize away. A default run still skips them: they are three lines, and the default minimum is four. + +```clojure +(defn alpha [xs] + (let [ys (filter odd? xs)] + (map inc ys))) + +(defn beta [items] + (let [kept (filter even? items)] + (map dec kept))) +``` + +The same shape in the other languages scores `1.0` too: the called function or method stays, and the locals do not. + +```java +int alpha(int[] xs) { + int[] ys = filter(xs, odd); + return map(ys, inc); +} +``` + +```python +def beta(items): + kept = filter(items, even) + return map(kept, dec) +``` + +What stays is the call or method name, the operator, and the shape of the tree. What drops out is the local name, the field name, the literal, and the keyword. A Clojure keyword call `(:kind m)` and `(:id row)` is the same shape. So is `order.total` and `row.amount`. + +## Run + +```bash +./dryer +``` + +The first run creates `.venv` and installs the tool. From a project root it walks the tree, skips `test`, `spec`, `vendor`, `node_modules`, and `target`, and writes two results: + +- a report on stdout, closest match first +- `.metrics/dry.edn`, replaced on every successful analysis + +```bash +./dryer src/demo src/ui # these files and trees +./dryer --changed # git additions and edits +./dryer billing # path fragment +./dryer --threshold 0.9 src +./dryer --edn --min-lines 3 src +``` + +```text +--threshold N Minimum structural similarity score, default 0.82 +--min-lines N Minimum source lines in a candidate form, default 4 +--min-nodes N Minimum normalized syntax nodes, default 20 +--format F text or edn, default text +--edn Same as --format edn +--text Same as --format text +``` + +Normalized syntax nodes are the structural pieces left after incidental names and literals are replaced. `--min-nodes` drops forms smaller than that before comparison. + +Default text output for the invoice and receipt from dry4clj. They differ by one extra binding: + +```text +DUPLICATE score=0.89 + src/billing/invoice.clj:3-13 + src/billing/receipt.clj:3-14 +``` + +`.metrics/dry.edn` is the same report dry4clj prints as EDN, with the language added: + +```clojure +{:candidates + [{:score 0.890909090909 + :language "clojure" + :left {:file "src/billing/invoice.clj", :start-line 3, :end-line 13} + :right {:file "src/billing/receipt.clj", :start-line 3, :end-line 14} + :left-nodes 158 + :right-nodes 166}]} +``` + +## What each language compares + +| Language | Candidate | Left out | +| --- | --- | --- | +| Clojure | every top-level form except `ns` | reader-conditional branches other than `:clj` | +| Java | methods with a body | constructors, abstract methods, methods inside anonymous or local classes | +| Go | functions and methods with a body | function literals stay inside the enclosing function | +| TypeScript | functions, methods, and top-level arrow functions | callbacks nested inside another function | +| Rust | functions and methods with a body | functions inside `mod tests`, and closures | +| Python | functions and methods | functions nested inside another function | + +Clojure reader conditionals keep the `:clj` branch, as dry4clj does. Syntax-quote is kept as a `syntax-quote` form rather than expanded. + +A missing or unreadable Clojure form is reported on stderr. Forms read before it are still compared. The run exits `0` after a report, `1` on a usage error, and `2` on an unknown `--format`. + +## Development + +```bash +python3 -m venv .venv +.venv/bin/pip install -e '.[dev]' +.venv/bin/pytest +``` diff --git a/dryer b/dryer new file mode 100755 index 0000000..16756c0 --- /dev/null +++ b/dryer @@ -0,0 +1,9 @@ +#!/bin/sh +# Run the duplication finder, creating a local virtualenv on first use. +ROOT=$(CDPATH= cd -- "$(dirname "$0")" && pwd) +PY="$ROOT/.venv/bin/python" +if [ ! -x "$PY" ]; then + python3 -m venv "$ROOT/.venv" + "$ROOT/.venv/bin/pip" install -e "$ROOT" +fi +exec "$PY" -m dryer "$@" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..463ca67 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "dryer" +version = "0.1.0" +description = "Structural duplicate finder for Clojure, Java, Go, TypeScript, Rust, and Python." +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "tree-sitter>=0.24", + "tree-sitter-language-pack>=0.7", +] + +[project.scripts] +dryer = "dryer.cli:main" + +[project.optional-dependencies] +dev = ["pytest>=8"] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/src/dryer/__init__.py b/src/dryer/__init__.py new file mode 100644 index 0000000..3591c90 --- /dev/null +++ b/src/dryer/__init__.py @@ -0,0 +1,3 @@ +"""Find structural duplicates in Clojure, Java, Go, TypeScript, Rust, and Python.""" + +__version__ = "0.1.0" diff --git a/src/dryer/__main__.py b/src/dryer/__main__.py new file mode 100644 index 0000000..cb40ee0 --- /dev/null +++ b/src/dryer/__main__.py @@ -0,0 +1,4 @@ +from dryer.cli import main + +if __name__ == "__main__": + main() diff --git a/src/dryer/astnorm.py b/src/dryer/astnorm.py new file mode 100644 index 0000000..80b5f0c --- /dev/null +++ b/src/dryer/astnorm.py @@ -0,0 +1,263 @@ +"""Normalize a tree-sitter tree the way dry4clj normalizes a Clojure form. + +The name in call position stays: `filter`, `Println`, `map`, an operator, the +type constructed by `new`. Local names, field names, type names in argument +position, and literals become generic markers. Two functions that call the +same operations in the same shape therefore share a fingerprint set even when +their locals differ. +""" + +from __future__ import annotations + +from dryer.shape import K +from dryer.treesitter import node_text + +_SKIP = { + "comment", + "line_comment", + "block_comment", + "documentation_comment", + "doc_comment", +} + +_IDENTIFIERS = { + "identifier", + "type_identifier", + "field_identifier", + "property_identifier", + "shorthand_property_identifier", + "shorthand_property_identifier_pattern", + "package_identifier", +} + +_LITERALS = { + "string", + "string_literal", + "interpreted_string_literal", + "raw_string_literal", + "char_literal", + "rune_literal", + "character_literal", + "number", + "integer", + "float", + "decimal_integer_literal", + "hex_integer_literal", + "octal_integer_literal", + "binary_integer_literal", + "decimal_floating_point_literal", + "hex_floating_point_literal", + "int_literal", + "float_literal", + "imaginary_literal", + "integer_literal", + "true", + "false", + "null", + "nil", + "none", + "undefined", + "null_literal", + "boolean_literal", + "template_string", + "regex", + "regex_literal", +} + +_OPERATORS = { + "+", + "-", + "*", + "/", + "%", + "**", + "==", + "!=", + "<", + ">", + "<=", + ">=", + "===", + "!==", + "&&", + "||", + "&", + "|", + "^", + "<<", + ">>", + ">>>", + "+=", + "-=", + "*=", + "/=", + "%=", + "&=", + "|=", + "^=", + "<<=", + ">>=", + ">>>=", + "!", + "~", + "++", + "--", + "and", + "or", + "not", + "??", + "?.", + "=", + ":=", + "=>", + "?", +} + +_CALLS = { + "call", + "call_expression", + "method_invocation", + "object_creation_expression", + "new_expression", +} + +_ATTRIBUTES = { + "attribute", + "member_expression", + "selector_expression", + "field_access", + "field_expression", +} + +_SCOPED = { + "scoped_identifier", + "scoped_type_identifier", +} + +_ARG_LISTS = {"argument_list", "arguments"} +_TYPE_ARGS = {"type_arguments", "type_parameters"} + +_DATA: bytes = b"" + + +def normalize(node, data: bytes): + """Normalized tree for `node`, or None when the node is only punctuation.""" + + global _DATA + _DATA = data + return _normalize(node) + + +def _text(node) -> str: + return node_text(_DATA, node) + + +def _normalize(node, head: bool = False): + if node.type in _SKIP: + return None + if node.type in _OPERATORS: + return [K("symbol"), node.type] + if not node.is_named: + return None + if node.type in _IDENTIFIERS: + if head: + return [K("symbol"), _text(node)] + return K("symbol") + if _is_literal(node): + return K("literal") + if node.type == "parenthesized_expression": + named = node.named_children + if len(named) == 1: + return _normalize(named[0], head) + if node.type in _CALLS: + return _normalize_call(node) + if node.type in _ATTRIBUTES: + return _normalize_attr(node, head) + children = [] + for child in node.children: + norm = _normalize(child, False) + if norm is not None: + children.append(norm) + return [K(node.type), *children] + + +def _is_literal(node) -> bool: + if node.type not in _LITERALS: + return False + if node.type == "string" and any(child.type == "interpolation" for child in node.children): + return False + return True + + +def _normalize_call(node): + incoming = [] + args = None + for child in node.named_children: + if child.type in _ARG_LISTS: + args = child + break + incoming.append(child) + type_args = [child for child in incoming if child.type in _TYPE_ARGS] + rest = [child for child in incoming if child.type not in _TYPE_ARGS] + out = [K(node.type)] + if rest: + *receivers, callee = rest + for receiver in receivers: + norm = _normalize(receiver, False) + if norm is not None: + out.append(norm) + out.append(_normalize_callee(callee)) + for type_node in type_args: + norm = _normalize(type_node, False) + if norm is not None: + out.append(norm) + if args is not None: + norm = _normalize(args, False) + if norm is not None: + out.append(norm) + return out + + +def _normalize_callee(node): + if node.type in _IDENTIFIERS: + return [K("symbol"), _text(node)] + if node.type in _ATTRIBUTES: + return _normalize_attr(node, True) + if node.type in _SCOPED or node.type == "generic_type": + return _normalize_path(node) + return _normalize(node, True) + + +def _normalize_path(node): + """A callee path keeps each name in the path. Type arguments stay structural.""" + + parts = [K(node.type)] + for child in node.named_children: + if child.type in _IDENTIFIERS: + parts.append([K("symbol"), _text(child)]) + elif child.type in _SCOPED or child.type == "generic_type": + parts.append(_normalize_path(child)) + else: + norm = _normalize(child, False) + if norm is not None: + parts.append(norm) + return parts + + +def _normalize_attr(node, head: bool): + named = list(node.named_children) + parts = [K(node.type)] + if not named: + return parts + *objects, name = named + for obj in objects: + norm = _normalize(obj, False) + if norm is not None: + parts.append(norm) + if name.type in _IDENTIFIERS: + parts.append([K("symbol"), _text(name)] if head else K("symbol")) + else: + nested = _normalize_callee(name) if head else _normalize(name, False) + if nested is not None: + parts.append(nested) + return parts diff --git a/src/dryer/cli.py b/src/dryer/cli.py new file mode 100644 index 0000000..d391ee5 --- /dev/null +++ b/src/dryer/cli.py @@ -0,0 +1,270 @@ +"""Command line for the multi-language duplication finder.""" + +from __future__ import annotations + +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +from dryer.discover import is_test_file, iter_source_files, language_of +from dryer.report import format_text, render_edn, write_metrics +from dryer.scan import find_duplicates, scan_files + +HELP = """\ +Usage: dryer [options] [path-or-filter ...] + +Find candidate duplicate code in Clojure, Java, Go, TypeScript, Rust, and +Python. Each form is normalized — call and method names, operators, and +tree shape stay; local names, field names, and literals do not — and compared +with the other forms of the same language. The score is Jaccard similarity of +those structural fingerprints, the same score as dry4clj. + +Prints a report and writes .metrics/dry.edn. + +Languages: Clojure (.clj .cljc .cljs .cljd .bb), Java (.java), Go (.go), +TypeScript (.ts .tsx .mts .cts), Rust (.rs), Python (.py). + +Options: + -h, --help Print this help and exit. + --root Project root. Metrics are written here. + Default: the current directory. + -s, --source-root Walk this tree instead of the project root. + May be repeated. + --changed Compare added and modified source files from + git status. + --threshold N Minimum similarity score, default 0.82. + --min-lines N Minimum source lines in a candidate form, + default 4. + --min-nodes N Minimum normalized syntax nodes, default 20. + --format F text or edn, default text. + --edn Same as --format edn. + --text Same as --format text. + +Arguments: + path File or directory to compare. Test paths are included when + you name them explicitly. + filter When the argument is not a path, only source files whose + path contains this text are compared. + +With no paths, source files under the project root are compared. Directories +named test, tests, spec, specs, vendor, node_modules, and target are skipped, +as are *_test.go, *.spec.ts, test_*.py, and *_test.py files. + +Clojure compares every top-level form except ns. The other languages compare +functions and methods. A form is only compared with forms in the same language. + +Exit codes: + 0 the report was written + 1 usage error + 2 unknown output format +""" + + +@dataclass +class Options: + action: str + message: str = "" + exit_code: int = 0 + project_root: Path = field(default_factory=lambda: Path(".")) + source_roots: list[str] = field(default_factory=list) + positionals: list[str] = field(default_factory=list) + threshold: float = 0.82 + min_lines: int = 4 + min_nodes: int = 20 + format: str = "text" + changed: bool = False + + +def _take(args: list[str], index: int, option: str) -> str: + if index + 1 >= len(args) or not args[index + 1] or args[index + 1].startswith("-"): + raise ValueError(f"{option} requires a value") + return args[index + 1] + + +def _number(value: str, option: str) -> float: + try: + return float(value) + except ValueError as exc: + raise ValueError(f"{option} requires a number") from exc + + +def _count(value: str, option: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise ValueError(f"{option} requires an integer") from exc + if parsed < 0: + raise ValueError(f"{option} requires a non-negative integer") + return parsed + + +def parse_args(argv: list[str] | None = None) -> Options: + args = list(sys.argv[1:] if argv is None else argv) + if any(arg in {"-h", "--help"} for arg in args): + return Options(action="help", message=HELP, exit_code=0) + options = Options(action="scan") + index = 0 + try: + while index < len(args): + arg = args[index] + if arg in {"-s", "--source-root"}: + options.source_roots.append(_take(args, index, arg)) + index += 2 + continue + if arg == "--root": + options.project_root = Path(_take(args, index, arg)) + index += 2 + continue + if arg == "--threshold": + options.threshold = _number(_take(args, index, arg), arg) + index += 2 + continue + if arg == "--min-lines": + options.min_lines = _count(_take(args, index, arg), arg) + index += 2 + continue + if arg == "--min-nodes": + options.min_nodes = _count(_take(args, index, arg), arg) + index += 2 + continue + if arg == "--format": + options.format = _take(args, index, arg) + index += 2 + continue + if arg == "--edn": + options.format = "edn" + index += 1 + continue + if arg == "--text": + options.format = "text" + index += 1 + continue + if arg == "--changed": + options.changed = True + index += 1 + continue + if arg.startswith("-"): + raise ValueError(f"Unknown option: {arg}") + options.positionals.append(arg) + index += 1 + except ValueError as exc: + return Options(action="help", message=f"{exc}\n\n{HELP}", exit_code=1) + if options.format not in {"text", "edn"}: + return Options( + action="format", + message=f"Unknown format: {options.format}\n", + exit_code=2, + ) + return options + + +def _changed_files(root: Path) -> list[Path]: + result = subprocess.run( + ["git", "status", "--porcelain"], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(result.stderr.strip() or "git status failed", file=sys.stderr) + return [] + found: list[Path] = [] + for line in result.stdout.splitlines(): + if len(line) < 4: + continue + path_text = line[3:].strip() + if " -> " in path_text: + path_text = path_text.split(" -> ", 1)[1] + path_text = path_text.strip('"') + found.append((root / path_text).resolve()) + return found + + +def _positionals(root: Path, args: list[str]) -> tuple[list[Path], list[str]]: + existing: list[Path] = [] + filters: list[str] = [] + for arg in args: + candidate = Path(arg) + if not candidate.is_absolute(): + candidate = root / arg + if candidate.exists(): + existing.append(candidate.resolve()) + else: + filters.append(arg) + return existing, filters + + +def _changed_source(root: Path) -> list[Path]: + return [ + path + for path in _changed_files(root) + if language_of(path) is not None and not is_test_file(path) + ] + + +def _explicit_files(existing: list[Path]) -> list[Path]: + files: list[Path] = [] + for path in existing: + if path.is_dir(): + files.extend(iter_source_files([path])) + elif language_of(path) is not None: + files.append(path) + return files + + +def _selected(options: Options, root: Path, existing: list[Path]) -> list[Path]: + if options.changed: + return _changed_source(root) + if options.source_roots: + return iter_source_files([(root / path).resolve() for path in options.source_roots]) + if existing: + return _explicit_files(existing) + return iter_source_files([root]) + + +def _apply_filters(files: list[Path], filters: list[str]) -> list[Path]: + if not filters: + return files + return [path for path in files if any(item in path.as_posix() for item in filters)] + + +def select_files(options: Options) -> list[Path]: + root = options.project_root.resolve() + existing, filters = _positionals(root, options.positionals) + files = _apply_filters(_selected(options, root, existing), filters) + return sorted({path.resolve() for path in files}, key=lambda path: path.as_posix()) + + +def run(argv: list[str] | None = None) -> int: + options = parse_args(argv) + if options.action == "help": + stream = sys.stdout if options.exit_code == 0 else sys.stderr + print(options.message, file=stream, end="" if options.message.endswith("\n") else "\n") + return options.exit_code + if options.action == "format": + print(options.message, file=sys.stderr, end="") + return options.exit_code + + root = options.project_root.resolve() + files = select_files(options) + if not files: + print("No source files to analyze.") + return 0 + + entries, warnings = scan_files(files, root, options.min_lines, options.min_nodes) + for warning in warnings: + print(f"dryer: {warning}", file=sys.stderr) + candidates = find_duplicates(entries, options.threshold) + metrics = write_metrics(candidates, root) + if options.format == "edn": + print(render_edn(candidates), end="") + else: + print(format_text(candidates), end="") + print(f"Wrote {metrics}", file=sys.stderr) + return 0 + + +def main(argv: list[str] | None = None) -> None: + sys.exit(run(argv)) diff --git a/src/dryer/clojure_read.py b/src/dryer/clojure_read.py new file mode 100644 index 0000000..6816a8d --- /dev/null +++ b/src/dryer/clojure_read.py @@ -0,0 +1,361 @@ +"""Read Clojure source into forms and normalize them the way dry4clj does. + +Names in argument position, keywords, and literals become generic markers. +The symbol at the head of a list keeps its spelling, so `filter` and `map` +stay distinct while the locals passed to them do not. + +Reader conditionals keep the `:clj` branch, which is what dry4clj selects. +Syntax-quote is kept as a `syntax-quote` list instead of being expanded into +`seq` and `concat` forms. +""" + +from __future__ import annotations + +import re + +from dryer.shape import K + +_WHITESPACE = set(" \t\n\r,") +_TERMINATORS = set("()[]{}\";'@^`~\\,#:") +_NUMBER = re.compile( + r"^[+-]?(?:" + r"\d+/\d+" + r"|\d+\.\d+(?:[eE][+-]?\d+)?" + r"|\.\d+(?:[eE][+-]?\d+)?" + r"|\d+(?:[eE][+-]?\d+)?" + r"|0[xX][0-9A-Fa-f]+" + r")[MN]?$" +) + + +class ReadError(Exception): + def __init__(self, line: int, message: str): + super().__init__(message) + self.line = line + + +class Sym: + def __init__(self, name: str, line: int): + self.name = name + self.line = line + + +class Kw: + def __init__(self, name: str, line: int): + self.name = name + self.line = line + + +class Lit: + def __init__(self, line: int): + self.line = line + + +class Coll: + def __init__(self, kind: str, line: int, items=None, pairs=None): + self.kind = kind + self.line = line + self.items = [] if items is None else items + self.pairs = [] if pairs is None else pairs + + +class Splice: + def __init__(self, items: list): + self.items = items + + +class Reader: + def __init__(self, text: str): + self.text = text + self.n = len(text) + self.i = 0 + self.line = 1 + + def peek(self) -> str | None: + if self.i >= self.n: + return None + return self.text[self.i] + + def eof(self) -> bool: + return self.i >= self.n + + def get(self) -> str: + ch = self.text[self.i] + self.i += 1 + if ch == "\n": + self.line += 1 + return ch + + def skip_ws(self) -> None: + while not self.eof(): + ch = self.peek() + if ch in _WHITESPACE: + self.get() + continue + if ch == ";": + while not self.eof() and self.peek() != "\n": + self.get() + continue + break + + def read_token(self) -> str: + start = self.i + line = self.line + while not self.eof(): + ch = self.peek() + if ch in _WHITESPACE or ch in _TERMINATORS: + break + self.get() + if self.i == start: + raise ReadError(line, "expected a symbol") + return self.text[start:self.i] + + def read_string(self) -> Lit: + line = self.line + self.get() + while not self.eof(): + ch = self.get() + if ch == "\\": + if self.eof(): + break + self.get() + continue + if ch == '"': + return Lit(line) + raise ReadError(line, "unterminated string") + + def read_char(self) -> Lit: + line = self.line + self.get() + if self.eof(): + raise ReadError(line, "incomplete character") + self.get() + while not self.eof(): + ch = self.peek() + if ch in _WHITESPACE or ch in _TERMINATORS: + break + self.get() + return Lit(line) + + def read_keyword(self) -> Kw: + line = self.line + self.get() + if self.peek() == ":": + self.get() + if self.eof() or self.peek() in _WHITESPACE or self.peek() in _TERMINATORS: + return Kw("", line) + return Kw(self.read_token(), line) + + def read_collection(self, end: str, kind: str) -> Coll: + line = self.line + self.get() + items: list = [] + while True: + self.skip_ws() + if self.eof(): + raise ReadError(line, f"unterminated {kind}") + if self.peek() == end: + self.get() + break + form = self.read_form() + if form is None: + continue + if isinstance(form, Splice): + items.extend(form.items) + continue + items.append(form) + if kind == "map": + if len(items) % 2: + raise ReadError(line, "map literal has an odd number of forms") + pairs = [(items[i], items[i + 1]) for i in range(0, len(items), 2)] + return Coll("map", line, pairs=pairs) + return Coll(kind, line, items=items) + + def _wrap(self, name: str, line: int) -> Coll: + inner = self.read_form() + items = [Sym(name, line)] + if inner is not None and not isinstance(inner, Splice): + items.append(inner) + return Coll("list", line, items=items) + + def read_dispatch(self): + line = self.line + self.get() + ch = self.peek() + if ch is None: + raise ReadError(line, "incomplete dispatch") + if ch == "_": + self.get() + self.read_form() + return None + if ch == "{": + return self.read_collection("}", "set") + if ch == "(": + inner = self.read_collection(")", "list") + return Coll("list", line, items=[Sym("fn*", line), *inner.items]) + if ch == '"': + self.read_string() + return Lit(line) + if ch == "?": + self.get() + splicing = False + if self.peek() == "@": + self.get() + splicing = True + return self.read_cond(splicing) + if ch == ":": + self.get() + if self.peek() == ":": + self.get() + elif self.peek() not in (None, "{") and self.peek() not in _WHITESPACE: + self.read_token() + return self.read_form() + self.read_token() + self.read_form() + return Lit(line) + + def read_cond(self, splicing: bool): + form = self.read_form() + if not isinstance(form, Coll) or form.kind != "list": + raise ReadError(self.line, "reader conditional body must be a list") + chosen = None + index = 0 + items = form.items + while index + 1 < len(items): + feature = items[index] + expr = items[index + 1] + index += 2 + name = feature.name if isinstance(feature, Kw) else None + if name in {"clj", "default"}: + chosen = expr + break + if chosen is None: + return None + if not splicing: + return chosen + if isinstance(chosen, Coll) and chosen.kind in {"list", "vector"}: + return Splice(list(chosen.items)) + raise ReadError(self.line, "splicing reader conditional needs a collection") + + def read_form(self): + self.skip_ws() + if self.eof(): + raise ReadError(self.line, "unexpected end of file") + ch = self.peek() + line = self.line + if ch == "(": + return self.read_collection(")", "list") + if ch == "[": + return self.read_collection("]", "vector") + if ch == "{": + return self.read_collection("}", "map") + if ch == '"': + return self.read_string() + if ch == "\\": + return self.read_char() + if ch == ":": + return self.read_keyword() + if ch == "'": + self.get() + return self._wrap("quote", line) + if ch == "@": + self.get() + return self._wrap("deref", line) + if ch == "`": + self.get() + return self._wrap("syntax-quote", line) + if ch == "~": + self.get() + name = "unquote" + if self.peek() == "@": + self.get() + name = "unquote-splicing" + return self._wrap(name, line) + if ch == "^": + self.get() + self.read_form() + return self.read_form() + if ch == "#": + return self.read_dispatch() + token = self.read_token() + if _NUMBER.match(token): + return Lit(line) + return Sym(token, line) + + +def read_source(text: str) -> tuple[list, str | None]: + """Top-level forms, plus a warning if a later form could not be read.""" + + if text.startswith("\ufeff"): + text = text[1:] + reader = Reader(text) + if text.startswith("#!"): + while not reader.eof() and reader.peek() != "\n": + reader.get() + forms: list = [] + while True: + reader.skip_ws() + if reader.eof(): + return forms, None + try: + form = reader.read_form() + except ReadError as exc: + return forms, f"{exc.line}: {exc}" + if form is None or isinstance(form, Splice): + continue + forms.append(form) + + +def _children(form): + if isinstance(form, Coll): + if form.kind == "map": + for key, value in form.pairs: + yield key + yield value + else: + yield from form.items + + +def max_line(form) -> int: + best = getattr(form, "line", 1) or 1 + for child in _children(form): + if child is not None: + best = max(best, max_line(child)) + return best + + +def normalize(form, head: bool = False): + """dry4clj's `normalize-form`. Collection heads are normalized in full.""" + + if isinstance(form, Coll): + if form.kind == "list": + if not form.items: + return [K("list"), K("literal")] + head_form, *args = form.items + return [K("list"), normalize(head_form, True), *[normalize(arg, False) for arg in args]] + if form.kind == "vector": + return [K("vector"), *[normalize(item, False) for item in form.items]] + if form.kind == "set": + return [K("set"), *[normalize(item, False) for item in form.items]] + if form.kind == "map": + return [ + K("map"), + *[[normalize(key, False), normalize(value, False)] for key, value in form.pairs], + ] + return [K("literal")] + if isinstance(form, Sym): + if head: + return [K("symbol"), form.name] + return K("symbol") + if isinstance(form, Kw): + return K("keyword") + return K("literal") + + +def is_candidate_form(form) -> bool: + """A non-empty top-level list whose head is not `ns`.""" + + if not isinstance(form, Coll) or form.kind != "list" or not form.items: + return False + head = form.items[0] + return not (isinstance(head, Sym) and head.name == "ns") diff --git a/src/dryer/discover.py b/src/dryer/discover.py new file mode 100644 index 0000000..193b16e --- /dev/null +++ b/src/dryer/discover.py @@ -0,0 +1,105 @@ +"""Find source files and decide which language each one is written in. + +The extensions and the skipped directories match crapper and mutator, with +`.cljd` added because dry4clj reads it. +""" + +import os +from pathlib import Path + +EXTENSIONS = { + ".clj": "clojure", + ".cljc": "clojure", + ".cljs": "clojure", + ".cljd": "clojure", + ".bb": "clojure", + ".java": "java", + ".go": "go", + ".ts": "typescript", + ".tsx": "typescript", + ".mts": "typescript", + ".cts": "typescript", + ".rs": "rust", + ".py": "python", +} + +SKIP_DIRS = { + ".git", + ".hg", + ".svn", + ".idea", + ".venv", + "venv", + "node_modules", + "vendor", + "target", + "dist", + "build", + "out", + "coverage", + ".metrics", + "testdata", + "__pycache__", + ".clj-kondo", +} + +TEST_DIRS = {"test", "tests", "spec", "specs", "__tests__"} + + +def language_of(path: str | Path) -> str | None: + file_path = Path(path) + name = file_path.name + if name.endswith(".d.ts"): + return None + return EXTENSIONS.get(file_path.suffix) + + +def is_test_file(path: str | Path) -> bool: + file_path = Path(path) + name = file_path.name + if name.endswith("_test.go"): + return True + if name.endswith(("_test.clj", "_test.cljc", "_test.cljs", "_test.cljd", "_test.bb")): + return True + if name.endswith( + ( + ".test.ts", + ".spec.ts", + ".test.tsx", + ".spec.tsx", + ".test.mts", + ".spec.mts", + ) + ): + return True + if name == "conftest.py": + return True + if name.endswith("_test.py") or (name.startswith("test_") and name.endswith(".py")): + return True + return any(part in TEST_DIRS for part in file_path.parts) + + +def _walk(root: Path) -> list[Path]: + if not root.exists(): + return [] + if root.is_file(): + return [root] + found: list[Path] = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [ + name for name in dirnames if name not in SKIP_DIRS and name not in TEST_DIRS + ] + for name in filenames: + path = Path(dirpath) / name + if language_of(path) is None or is_test_file(path): + continue + found.append(path) + return found + + +def iter_source_files(roots: list[Path]) -> list[Path]: + files: set[Path] = set() + for root in roots: + for path in _walk(root): + files.add(path.resolve()) + return sorted(files, key=lambda path: path.as_posix()) diff --git a/src/dryer/extract.py b/src/dryer/extract.py new file mode 100644 index 0000000..66a6953 --- /dev/null +++ b/src/dryer/extract.py @@ -0,0 +1,197 @@ +"""Pick the forms each language compares. + +Clojure compares every top-level list except `ns`, matching dry4clj. The other +languages compare the functions and methods crapper scores: bodies, not +signatures alone, and not callbacks nested inside another function. Rust +functions inside `mod tests` are test code and are skipped. +""" + +from __future__ import annotations + +from dryer.astnorm import normalize +from dryer.clojure_read import is_candidate_form, max_line, normalize as normalize_clj +from dryer.clojure_read import read_source +from dryer.model import Entry +from dryer.shape import fingerprints, node_count +from dryer.treesitter import child_of_type, descendants, end_line, node_text, parse, start_line + +_TS_FUNCTION = { + "function_declaration", + "method_definition", + "arrow_function", + "function_expression", +} +_DEFINITION_PARENTS = { + "variable_declarator", + "public_field_definition", + "property_definition", +} + + +def _grammar(path: str) -> str: + if path.endswith(".tsx"): + return "tsx" + return "typescript" + + +def _has(node, *types: str) -> bool: + return child_of_type(node, *types) is not None + + +def _inside(node, types: set[str]) -> bool: + current = node.parent + while current is not None: + if current.type in types: + return True + current = current.parent + return False + + +def _python_nested(node) -> bool: + """A function inside another function. A class in between keeps the method.""" + + current = node.parent + while current is not None: + if current.type == "class_definition": + return False + if current.type == "function_definition": + return True + current = current.parent + return False + + +def _rust_in_tests(data: bytes, node) -> bool: + current = node.parent + while current is not None: + if current.type == "mod_item": + ident = child_of_type(current, "identifier") + if ident is not None and node_text(data, ident) == "tests": + return True + current = current.parent + return False + + +def _java_nodes(data: bytes, root) -> list: + found = [] + + def visit(node) -> None: + if node.type == "method_declaration": + nested = _inside( + node, + {"method_declaration", "constructor_declaration", "lambda_expression"}, + ) + if _has(node, "block") and not nested: + found.append(node) + return + for child in node.children: + visit(child) + + visit(root) + return found + + +def _go_nodes(data: bytes, root) -> list: + found = [] + for node in descendants(root): + if node.type in {"function_declaration", "method_declaration"} and _has(node, "block"): + found.append(node) + return found + + +def _python_nodes(data: bytes, root) -> list: + found = [] + for node in descendants(root): + if node.type == "function_definition" and not _python_nested(node): + found.append(node) + return found + + +def _typescript_nodes(data: bytes, root) -> list: + found = [] + for node in descendants(root): + if node.type == "function_declaration": + if _has(node, "statement_block") and not _inside(node, _TS_FUNCTION): + found.append(node) + elif node.type == "method_definition": + if _has(node, "statement_block") and not _inside(node, _TS_FUNCTION): + found.append(node) + elif node.type in {"arrow_function", "function_expression"}: + parent = node.parent + if parent is None or parent.type not in _DEFINITION_PARENTS: + continue + if _inside(parent, _TS_FUNCTION): + continue + found.append(parent) + return found + + +def _rust_nodes(data: bytes, root) -> list: + found = [] + for node in descendants(root): + if node.type != "function_item" or not _has(node, "block"): + continue + if node.parent is not None and node.parent.type == "block": + continue + if _rust_in_tests(data, node): + continue + found.append(node) + return found + + +_PICKERS = { + "java": ("java", _java_nodes), + "go": ("go", _go_nodes), + "python": ("python", _python_nodes), + "typescript": (_grammar, _typescript_nodes), + "rust": ("rust", _rust_nodes), +} + + +def _tree_entries(language: str, source: str, path: str, file: str) -> list[Entry]: + grammar_or_name, pick = _PICKERS[language] + grammar = grammar_or_name(path) if callable(grammar_or_name) else grammar_or_name + data, tree = parse(source, grammar) + entries = [] + for node in pick(data, tree.root_node): + normalized = normalize(node, data) + if normalized is None: + continue + entries.append( + Entry( + language=language, + file=file, + start_line=start_line(node), + end_line=end_line(node), + nodes=node_count(normalized), + fingerprints=fingerprints(normalized), + ) + ) + return entries + + +def _clojure_entries(source: str, file: str) -> tuple[list[Entry], str | None]: + forms, warning = read_source(source) + entries = [] + for form in forms: + if not is_candidate_form(form): + continue + normalized = normalize_clj(form) + entries.append( + Entry( + language="clojure", + file=file, + start_line=form.line, + end_line=max_line(form), + nodes=node_count(normalized), + fingerprints=fingerprints(normalized), + ) + ) + return entries, warning + + +def entries_in_source(language: str, source: str, path: str, file: str) -> tuple[list[Entry], str | None]: + if language == "clojure": + return _clojure_entries(source, file) + if language not in _PICKERS: + return [], None + return _tree_entries(language, source, path, file), None diff --git a/src/dryer/model.py b/src/dryer/model.py new file mode 100644 index 0000000..74ed888 --- /dev/null +++ b/src/dryer/model.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Span: + """Where one candidate form sits in a source file.""" + + file: str + start_line: int + end_line: int + + +@dataclass(frozen=True) +class Duplicate: + """One pair of forms whose normalized structure is similar enough to report.""" + + score: float + language: str + left: Span + right: Span + left_nodes: int + right_nodes: int + + +@dataclass(frozen=True) +class Entry: + """A normalized form ready to compare. Forms are only compared within one language.""" + + language: str + file: str + start_line: int + end_line: int + nodes: int + fingerprints: frozenset[str] diff --git a/src/dryer/report.py b/src/dryer/report.py new file mode 100644 index 0000000..8bc7a6a --- /dev/null +++ b/src/dryer/report.py @@ -0,0 +1,74 @@ +"""Text and EDN reports. The text layout matches dry4clj.""" + +from __future__ import annotations + +from pathlib import Path + +from dryer.model import Duplicate + + +def format_text(candidates: list[Duplicate]) -> str: + if not candidates: + return "No duplicate candidates found.\n" + blocks = [] + for candidate in candidates: + left = candidate.left + right = candidate.right + blocks.append( + f"DUPLICATE score={candidate.score:.2f}\n" + f" {left.file}:{left.start_line}-{left.end_line}\n" + f" {right.file}:{right.start_line}-{right.end_line}" + ) + return "\n\n".join(blocks) + "\n" + + +def _edn_string(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def _edn_float(value: float) -> str: + text = f"{value:.12f}".rstrip("0").rstrip(".") + if "." not in text: + text += ".0" + return text + + +def render_edn(candidates: list[Duplicate]) -> str: + """EDN map `{:candidates [...]}` with dry4clj's keys, plus `:language`.""" + + if not candidates: + return "{:candidates []}\n" + rows = [] + for candidate in candidates: + left = candidate.left + right = candidate.right + rows.append( + " {:score " + + _edn_float(candidate.score) + + "\n :language " + + _edn_string(candidate.language) + + "\n :left {:file " + + _edn_string(left.file) + + f", :start-line {left.start_line}, :end-line {left.end_line}" + + "}" + + "\n :right {:file " + + _edn_string(right.file) + + f", :start-line {right.start_line}, :end-line {right.end_line}" + + "}" + + f"\n :left-nodes {candidate.left_nodes}" + + f"\n :right-nodes {candidate.right_nodes}" + + "}" + ) + return "{:candidates [\n" + "\n".join(rows) + "\n]}\n" + + +def metrics_path(root: Path) -> Path: + return root / ".metrics" / "dry.edn" + + +def write_metrics(candidates: list[Duplicate], root: Path) -> Path: + path = metrics_path(root) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render_edn(candidates), encoding="utf-8") + return path diff --git a/src/dryer/scan.py b/src/dryer/scan.py new file mode 100644 index 0000000..8ce54b0 --- /dev/null +++ b/src/dryer/scan.py @@ -0,0 +1,84 @@ +"""Read source files and pair the forms that clear the similarity threshold.""" + +from __future__ import annotations + +from pathlib import Path + +from dryer.discover import language_of +from dryer.extract import entries_in_source +from dryer.model import Duplicate, Entry, Span +from dryer.shape import jaccard + + +def display_path(path: Path, root: Path) -> str: + resolved = path.resolve() + try: + return resolved.relative_to(root.resolve()).as_posix() + except ValueError: + return resolved.as_posix() + + +def _keep(entry: Entry, min_lines: int, min_nodes: int) -> bool: + lines = entry.end_line - entry.start_line + 1 + return lines >= min_lines and entry.nodes >= min_nodes + + +def scan_files( + files: list[Path], root: Path, min_lines: int, min_nodes: int +) -> tuple[list[Entry], list[str]]: + entries: list[Entry] = [] + warnings: list[str] = [] + for path in files: + language = language_of(path) + if language is None: + continue + source = path.read_text(encoding="utf-8", errors="replace") + file = display_path(path, root) + found, warning = entries_in_source(language, source, path.as_posix(), file) + if warning: + warnings.append(f"{file}:{warning}") + entries.extend(entry for entry in found if _keep(entry, min_lines, min_nodes)) + return entries, warnings + + +def find_duplicates(entries: list[Entry], threshold: float) -> list[Duplicate]: + """Pair forms of the same language. Each unordered pair is reported once.""" + + by_language: dict[str, list[Entry]] = {} + for entry in entries: + by_language.setdefault(entry.language, []).append(entry) + + found: list[Duplicate] = [] + for language, group in by_language.items(): + for index, left in enumerate(group): + for right in group[index + 1 :]: + if ( + left.file == right.file + and left.start_line == right.start_line + and left.end_line == right.end_line + ): + continue + score = jaccard(left.fingerprints, right.fingerprints) + if score < threshold: + continue + found.append( + Duplicate( + score=score, + language=language, + left=Span(left.file, left.start_line, left.end_line), + right=Span(right.file, right.start_line, right.end_line), + left_nodes=left.nodes, + right_nodes=right.nodes, + ) + ) + found.sort( + key=lambda item: ( + -item.score, + item.language, + item.left.file, + item.left.start_line, + item.right.file, + item.right.start_line, + ) + ) + return found diff --git a/src/dryer/shape.py b/src/dryer/shape.py new file mode 100644 index 0000000..0ab0c1d --- /dev/null +++ b/src/dryer/shape.py @@ -0,0 +1,59 @@ +"""Jaccard similarity over structural fingerprints. + +This is the dry4clj score. A normalized form is a tree of keywords, names, and +vectors. The fingerprint set contains the printed form of every subtree. The +score is the size of the shared set divided by the size of the union. +""" + +from __future__ import annotations + + +class K: + """A keyword in the normalized tree. Prints the way Clojure prints keywords.""" + + def __init__(self, name: str): + self.name = name + + def __repr__(self) -> str: + return ":" + self.name + + +def pr(node) -> str: + """Print a normalized node the way Clojure `pr-str` prints it.""" + + if isinstance(node, K): + return ":" + node.name + if isinstance(node, str): + escaped = node.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + if isinstance(node, list): + return "[" + " ".join(pr(child) for child in node) + "]" + raise TypeError(f"cannot print {node!r}") + + +def node_count(node) -> int: + """One plus the count of every nested node. Atoms count as one.""" + + if isinstance(node, list): + return 1 + sum(node_count(child) for child in node) + return 1 + + +def fingerprints(node) -> frozenset[str]: + found: set[str] = set() + + def walk(form) -> None: + found.add(pr(form)) + if isinstance(form, list): + for child in form: + walk(child) + + walk(node) + return frozenset(found) + + +def jaccard(left: frozenset[str], right: frozenset[str]) -> float: + union = left | right + if not union: + return 0.0 + return len(left & right) / len(union) diff --git a/src/dryer/treesitter.py b/src/dryer/treesitter.py new file mode 100644 index 0000000..4038af9 --- /dev/null +++ b/src/dryer/treesitter.py @@ -0,0 +1,47 @@ +"""Shared tree-sitter helpers.""" + +from functools import lru_cache + + +@lru_cache(maxsize=None) +def parser_for(language: str): + from tree_sitter_language_pack import download, get_parser + + download([language]) + return get_parser(language) + + +def parse(source: str, language: str): + data = source.encode("utf-8") + tree = parser_for(language).parse(data) + return data, tree + + +def node_text(data: bytes, node) -> str: + return data[node.start_byte : node.end_byte].decode("utf-8") + + +def start_line(node) -> int: + return node.start_point[0] + 1 + + +def end_line(node) -> int: + row, col = node.end_point + if col == 0: + return max(start_line(node), row) + return row + 1 + + +def descendants(node): + stack = [node] + while stack: + current = stack.pop() + yield current + stack.extend(reversed(current.children)) + + +def child_of_type(node, *types: str): + for child in node.children: + if child.type in types: + return child + return None diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..7d56b44 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,137 @@ +from pathlib import Path + +from dryer.cli import main, parse_args, run +from dryer.discover import language_of + + +def write_source(root: Path, name: str, text: str) -> None: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def test_language_detection(): + assert language_of("src/app/core.clj") == "clojure" + assert language_of("src/app/core.cljd") == "clojure" + assert language_of("Widget.java") == "java" + assert language_of("board.go") == "go" + assert language_of("ui/view.tsx") == "typescript" + assert language_of("src/lib.rs") == "rust" + assert language_of("src/dryer/cli.py") == "python" + assert language_of("types.d.ts") is None + assert language_of("notes.md") is None + + +def test_help_does_not_scan(capsys): + assert run(["--help"]) == 0 + out = capsys.readouterr().out + assert "Usage: dryer" in out + assert "--threshold" in out + + +def test_unknown_format(capsys): + assert run(["--format", "csv"]) == 2 + err = capsys.readouterr().err + assert err == "Unknown format: csv\n" + + +def test_unknown_option(capsys): + assert run(["--nope"]) == 1 + assert "Unknown option: --nope" in capsys.readouterr().err + + +def test_parse_defaults_and_flags(): + options = parse_args( + ["--threshold", "0.9", "--min-lines", "5", "--min-nodes", "30", "--edn", "spec"] + ) + assert options.threshold == 0.9 + assert options.min_lines == 5 + assert options.min_nodes == 30 + assert options.format == "edn" + assert options.positionals == ["spec"] + + +def test_project_report_and_snapshot(tmp_path, capsys): + write_source( + tmp_path, + "src/left.py", + "def alpha(xs):\n ys = filter(xs, odd)\n zs = map(ys, inc)\n return list(zs)\n", + ) + write_source( + tmp_path, + "src/right.py", + "def beta(items):\n kept = filter(items, even)\n out = map(kept, dec)\n return list(out)\n", + ) + write_source( + tmp_path, + "tests/test_left.py", + "def alpha(xs):\n ys = filter(xs, odd)\n zs = map(ys, inc)\n return list(zs)\n", + ) + code = run(["--root", str(tmp_path), "--min-lines", "3", "--min-nodes", "1"]) + captured = capsys.readouterr() + assert code == 0 + assert "DUPLICATE score=1.00" in captured.out + assert "src/left.py:1-" in captured.out + assert "src/right.py:1-" in captured.out + assert "test_left.py" not in captured.out + text = (tmp_path / ".metrics" / "dry.edn").read_text(encoding="utf-8") + assert ':language "python"' in text + assert ':file "src/left.py"' in text + assert "Wrote" in captured.err + + +def test_edn_stdout(tmp_path, capsys): + write_source(tmp_path, "one.py", "def alpha(xs):\n return xs\n") + assert run(["--root", str(tmp_path), "--edn", "--min-lines", "1", "--min-nodes", "1"]) == 0 + out = capsys.readouterr().out + assert out == "{:candidates []}\n" + + +def test_empty_project(tmp_path, capsys): + assert run(["--root", str(tmp_path)]) == 0 + assert capsys.readouterr().out == "No source files to analyze.\n" + + +def test_path_filter(tmp_path, capsys): + body_a = "def alpha(xs):\n ys = filter(xs, odd)\n zs = map(ys, inc)\n return list(zs)\n" + body_b = "def beta(items):\n kept = filter(items, even)\n out = map(kept, dec)\n return list(out)\n" + write_source(tmp_path, "src/board/a.py", body_a) + write_source(tmp_path, "src/board/b.py", body_b) + write_source(tmp_path, "src/other/a.py", body_a) + write_source(tmp_path, "src/other/b.py", body_b) + assert run(["--root", str(tmp_path), "--min-lines", "3", "--min-nodes", "1", "board"]) == 0 + out = capsys.readouterr().out + assert "src/board/a.py" in out + assert "src/other/a.py" not in out + + +def test_explicit_test_file_is_included(tmp_path, capsys): + body = "def alpha(xs):\n ys = filter(xs, odd)\n zs = map(ys, inc)\n return list(zs)\n" + other = "def beta(items):\n kept = filter(items, even)\n out = map(kept, dec)\n return list(out)\n" + write_source(tmp_path, "src/a.py", body) + write_source(tmp_path, "tests/test_a.py", other) + assert ( + run( + [ + "--root", + str(tmp_path), + "--min-lines", + "3", + "--min-nodes", + "1", + "src/a.py", + "tests/test_a.py", + ] + ) + == 0 + ) + out = capsys.readouterr().out + assert "src/a.py" in out + assert "tests/test_a.py" in out + + +def test_main_exits(monkeypatch): + codes = [] + monkeypatch.setattr("dryer.cli.sys.exit", lambda code: codes.append(code)) + main(["--help"]) + assert codes == [0] diff --git a/tests/test_clojure.py b/tests/test_clojure.py new file mode 100644 index 0000000..4ba1937 --- /dev/null +++ b/tests/test_clojure.py @@ -0,0 +1,187 @@ +from pathlib import Path + +from dryer.report import format_text +from dryer.scan import find_duplicates, scan_files +from dryer.model import Duplicate, Span + + +def write_source(root: Path, name: str, text: str) -> None: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def duplicates(root: Path, **options): + files = sorted(path for path in root.rglob("*") if path.is_file()) + entries, warnings = scan_files( + files, + root, + options.get("min_lines", 4), + options.get("min_nodes", 20), + ) + assert warnings == [] + return find_duplicates(entries, options.get("threshold", 0.82)) + + +def test_text_report_matches_dry4clj(): + candidate = Duplicate( + 0.875, + "clojure", + Span("a.clj", 10, 14), + Span("b.clj", 20, 24), + 30, + 31, + ) + assert ( + format_text([candidate]) + == "DUPLICATE score=0.88\n a.clj:10-14\n b.clj:20-24\n" + ) + assert format_text([]) == "No duplicate candidates found.\n" + + +def test_reports_structural_duplicates_with_line_ranges(tmp_path): + write_source( + tmp_path, + "left.cljc", + "(ns sample.left)\n\n(defn alpha [xs]\n (let [ys (filter odd? xs)]\n (map inc ys)))\n", + ) + write_source( + tmp_path, + "right.cljc", + "(ns sample.right)\n\n(defn beta [items]\n (let [kept (filter even? items)]\n (map dec kept)))\n", + ) + found = duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=8) + assert len(found) == 1 + assert found[0].score == 1.0 + assert found[0].language == "clojure" + assert found[0].left.file == "left.cljc" + assert (found[0].left.start_line, found[0].left.end_line) == (3, 5) + assert found[0].right.file == "right.cljc" + assert (found[0].right.start_line, found[0].right.end_line) == (3, 5) + + +def test_matches_maps_sets_and_keyword_calls(tmp_path): + write_source( + tmp_path, + "left.cljc", + "(ns sample.left)\n\n(defn gamma [m]\n (when (#{:a :b} (:kind m))\n {:left (:a m) :right (:b m)}))\n", + ) + write_source( + tmp_path, + "right.cljc", + "(ns sample.right)\n\n(defn delta [row]\n (when (#{:c :d} (:kind row))\n {:left (:c row) :right (:d row)}))\n", + ) + found = duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=8) + assert len(found) == 1 + assert found[0].score == 1.0 + + +def test_reads_cljc_reader_conditionals(tmp_path): + write_source( + tmp_path, + "left.cljc", + "(ns sample.left)\n\n(defn alpha [x]\n #?(:clj (when (pos? x)\n (inc x))))\n", + ) + write_source( + tmp_path, + "right.cljc", + "(ns sample.right)\n\n(defn beta [y]\n #?(:clj (when (pos? y)\n (inc y))))\n", + ) + found = duplicates(tmp_path, threshold=0.50, min_lines=1, min_nodes=1) + assert len(found) == 1 + assert found[0].score == 1.0 + + +def test_filters_forms_shorter_than_the_minimum_line_count(tmp_path): + write_source(tmp_path, "one.clj", "(ns one)\n(defn a [x] (+ x 1))\n") + write_source(tmp_path, "two.clj", "(ns two)\n(defn b [y] (+ y 2))\n") + assert duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=1) == [] + + +def test_metadata_comments_and_discards_are_not_structure(tmp_path): + write_source( + tmp_path, + "left.clj", + "(ns left)\n\n(defn alpha [xs] ; keep\n #_(println xs)\n (let [ys (filter odd? xs)]\n (map inc ys)))\n", + ) + write_source( + tmp_path, + "right.clj", + "(ns right)\n\n(defn ^String beta [items]\n (let [kept (filter even? items)]\n (map dec kept)))\n", + ) + found = duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=8) + assert len(found) == 1 + assert found[0].score == 1.0 + + +def test_extra_binding_scores_below_one(tmp_path): + write_source( + tmp_path, + "invoice.clj", + """(ns billing.invoice) + +(defn invoice-summary [orders] + (let [paid (filter paid? orders), domestic (filter domestic? paid) + sorted (sort-by :date domestic), amounts (map :amount sorted) + taxes (map tax amounts), ids (map :id sorted) + customers (map :customer sorted), regions (group-by :region sorted) + flagged (filter flagged? sorted)] + {:count (count sorted) + :first-id (first ids), :last-id (last ids) + :customers (set customers), :regions (keys regions) + :flagged (count flagged), :total (reduce + 0 amounts) + :tax (reduce + 0 taxes)})) +""", + ) + write_source( + tmp_path, + "receipt.clj", + """(ns billing.receipt) + +(defn receipt-summary [rows] + (let [closed (filter closed? rows), local (filter local? closed) + ordered (sort-by :date local), amounts (map :amount ordered) + taxable (filter taxable? ordered), taxes (map tax amounts) + ids (map :id ordered), customers (map :customer ordered) + regions (group-by :region ordered) + flagged (filter flagged? ordered)] + {:count (count ordered) + :first-id (first ids), :last-id (last ids) + :customers (set customers), :regions (keys regions) + :flagged (count flagged), :total (reduce + 0 amounts) + :tax (reduce + 0 taxes)})) +""", + ) + found = duplicates(tmp_path) + assert len(found) == 1 + assert 0.85 <= found[0].score < 1.0 + + +def test_three_copies_make_three_pairs(tmp_path): + body = "(let [ys (filter odd? xs)]\n (map inc ys))" + write_source( + tmp_path, + "same.clj", + f"(ns same)\n\n(defn a [xs]\n {body})\n\n(defn b [xs]\n {body})\n\n(defn c [xs]\n {body})\n", + ) + found = duplicates(tmp_path, threshold=0.80, min_lines=3, min_nodes=8) + assert len(found) == 3 + assert {item.score for item in found} == {1.0} + + +def test_unreadable_tail_does_not_drop_an_earlier_form(tmp_path): + write_source( + tmp_path, + "left.clj", + "(ns left)\n\n(defn alpha [xs]\n (let [ys (filter odd? xs)]\n (map inc ys)))\n(defn broken [x]\n", + ) + write_source( + tmp_path, + "right.clj", + "(ns right)\n\n(defn beta [items]\n (let [kept (filter even? items)]\n (map dec kept)))\n", + ) + files = sorted(tmp_path.glob("*.clj")) + entries, warnings = scan_files(files, tmp_path, 3, 8) + assert warnings == ["left.clj:6: unterminated list"] + found = find_duplicates(entries, 0.80) + assert len(found) == 1 diff --git a/tests/test_languages.py b/tests/test_languages.py new file mode 100644 index 0000000..6e31f5f --- /dev/null +++ b/tests/test_languages.py @@ -0,0 +1,233 @@ +from pathlib import Path + +from dryer.scan import find_duplicates, scan_files + + +def write_source(root: Path, name: str, text: str) -> None: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def pairs(root: Path): + files = sorted(path for path in root.rglob("*") if path.is_file()) + entries, warnings = scan_files(files, root, min_lines=1, min_nodes=1) + assert warnings == [] + return find_duplicates(entries, threshold=0.99) + + +JAVA_LEFT = """\ +class Left { + int alpha(int[] xs) { + int[] ys = filter(xs, odd); + return map(ys, inc); + } +} +""" + +JAVA_RIGHT = """\ +class Right { + int beta(int[] items) { + int[] kept = filter(items, even); + return map(kept, dec); + } +} +""" + +GO_LEFT = """\ +package demo + +func (b *Board) alpha(xs []int) []int { + ys := filter(xs, odd) + return mapFn(ys, inc) +} +""" + +GO_RIGHT = """\ +package demo + +func (p *Panel) beta(items []int) []int { + kept := filter(items, even) + return mapFn(kept, dec) +} +""" + +PY_LEFT = """\ +class Board: + def alpha(self, xs): + ys = filter(xs, odd) + return map(ys, inc) +""" + +PY_RIGHT = """\ +class Panel: + def beta(self, items): + kept = filter(items, even) + return map(kept, dec) +""" + +TS_LEFT = """\ +export function alpha(xs: number[]): number[] { + const ys = filter(xs, odd); + return map(ys, inc); +} +""" + +TS_RIGHT = """\ +export function beta(items: number[]): number[] { + const kept = filter(items, even); + return map(kept, dec); +} +""" + +TSX_LEFT = """\ +export const alpha = (xs: number[]): number[] => { + const ys = filter(xs, odd); + return map(ys, inc); +}; +""" + +TSX_RIGHT = """\ +export const beta = (items: number[]): number[] => { + const kept = filter(items, even); + return map(kept, dec); +}; +""" + +RUST_LEFT = """\ +pub fn alpha(xs: Vec) -> Vec { + let ys = filter(xs, odd); + map(ys, inc) +} +""" + +RUST_RIGHT = """\ +pub fn beta(items: Vec) -> Vec { + let kept = filter(items, even); + map(kept, dec) +} +""" + + +def test_each_language_matches_renamed_locals(tmp_path): + samples = { + "left.java": JAVA_LEFT, + "right.java": JAVA_RIGHT, + "left.go": GO_LEFT, + "right.go": GO_RIGHT, + "left.py": PY_LEFT, + "right.py": PY_RIGHT, + "left.ts": TS_LEFT, + "right.ts": TS_RIGHT, + "left.tsx": TSX_LEFT, + "right.tsx": TSX_RIGHT, + "left.rs": RUST_LEFT, + "right.rs": RUST_RIGHT, + } + for name, source in samples.items(): + write_source(tmp_path, name, source) + found = pairs(tmp_path) + by_language = {} + for item in found: + by_language.setdefault(item.language, []).append(item) + assert set(by_language) == {"java", "go", "python", "typescript", "rust"} + assert [(item.left.file, item.right.file) for item in by_language["typescript"]] == [ + ("left.ts", "right.ts"), + ("left.tsx", "right.tsx"), + ] + for language, group in by_language.items(): + assert all(item.score == 1.0 for item in group), language + + +def test_different_callee_is_not_the_same_structure(tmp_path): + write_source( + tmp_path, + "a.py", + "def alpha(xs):\n return filter(xs, odd)\n", + ) + write_source( + tmp_path, + "b.py", + "def beta(xs):\n return select(xs, odd)\n", + ) + files = sorted(tmp_path.glob("*.py")) + entries, warnings = scan_files(files, tmp_path, min_lines=1, min_nodes=1) + assert warnings == [] + found = find_duplicates(entries, threshold=0.0) + assert len(found) == 1 + assert found[0].score < 1.0 + + +def test_nested_functions_stay_inside_the_enclosing_function(tmp_path): + write_source( + tmp_path, + "a.py", + """\ +def alpha(xs): + def inner(ys): + return filter(ys, odd) + return inner(xs) +""", + ) + write_source( + tmp_path, + "b.py", + """\ +def beta(items): + def inner(kept): + return filter(kept, even) + return inner(items) +""", + ) + found = pairs(tmp_path) + assert len(found) == 1 + assert found[0].score == 1.0 + assert found[0].left.start_line == 1 + + +def test_rust_tests_module_is_not_production_code(tmp_path): + write_source( + tmp_path, + "a.rs", + """\ +pub fn alpha(xs: Vec) -> Vec { + let ys = filter(xs, odd); + map(ys, inc) +} + +mod tests { + pub fn helper(xs: Vec) -> Vec { + let ys = filter(xs, odd); + map(ys, inc) + } +} +""", + ) + write_source(tmp_path, "b.rs", RUST_RIGHT) + found = pairs(tmp_path) + assert len(found) == 1 + assert found[0].left.file == "a.rs" + assert found[0].left.start_line == 1 + + +def test_an_extra_statement_lowers_the_score(tmp_path): + write_source(tmp_path, "a.java", JAVA_LEFT) + write_source( + tmp_path, + "b.java", + """\ +class Right { + int beta(int[] items) { + int[] kept = filter(items, even); + int[] extra = filter(kept, even); + return map(extra, dec); + } +} +""", + ) + files = sorted(tmp_path.glob("*.java")) + entries, warnings = scan_files(files, tmp_path, min_lines=1, min_nodes=1) + assert warnings == [] + found = find_duplicates(entries, threshold=0.5) + assert len(found) == 1 + assert found[0].score < 1.0