From d569bff8a44fc30cf71eabdefd74ede575cd5510 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 22 Jun 2026 16:04:00 +0200 Subject: [PATCH 01/20] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20TypeScript=20sup?= =?UTF-8?q?port=20for=20discovery=20and=20analyse=20(#69)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/components/analyse.rst | 2 +- docs/source/components/configuration.rst | 7 +- docs/source/components/discover.rst | 10 ++ docs/source/development/change_log.rst | 6 +- pyproject.toml | 1 + src/sphinx_codelinks/analyse/utils.py | 13 +++ .../source_discover/config.py | 2 + tests/data/discover_fixtures.json | 44 +++++++-- tests/data/typescript/demo.ts | 17 ++++ tests/test_analyse.py | 24 ++++- tests/test_analyse_utils.py | 98 +++++++++++++++++++ tests/test_source_discover.py | 10 +- tests/test_src_trace.py | 2 +- 13 files changed, 222 insertions(+), 14 deletions(-) create mode 100644 tests/data/typescript/demo.ts diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 2b47c5a1..e00f1f1d 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -47,7 +47,7 @@ Limitations **Current Limitations:** -- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), Python (``#``), YAML (``#``) and Rust (``//``, ``/* */``, ``///``) comment styles are supported +- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), TypeScript (``//``, ``/* */``), Python (``#``), YAML (``#``) and Rust (``//``, ``/* */``, ``///``) comment styles are supported - **Single Comment Style**: Each analysis run processes only one comment style at a time Extraction Examples diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 0d354dd5..0f15c9de 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -271,7 +271,7 @@ Specifies the comment syntax style used in the source code files. This determine **Type:** ``str`` **Default:** ``"cpp"`` -**Supported values:** ``"cpp"``, ``"python"``, ``"cs"``, ``"yaml"``, ``"rust"`` +**Supported values:** ``"cpp"``, ``"python"``, ``"cs"``, ``"ts"``, ``"yaml"``, ``"rust"`` .. code-block:: toml @@ -304,6 +304,11 @@ Specifies the comment syntax style used in the source code files. This determine ``/* */`` (multi-line), ``///`` (XML doc comments) - ``.cs`` + * - TypeScript + - ``"ts"`` + - ``//`` (single-line), + ``/* */`` (multi-line) + - ``.ts``, ``.tsx`` * - YAML - ``"yaml"`` - ``#`` (single-line) diff --git a/docs/source/components/discover.rst b/docs/source/components/discover.rst index 33a2d526..8b78645d 100644 --- a/docs/source/components/discover.rst +++ b/docs/source/components/discover.rst @@ -38,3 +38,13 @@ Usage Examples include = [] exclude = ["tests/**", "setup.py"] comment_type = "python" + +**TypeScript Project:** + +.. code-block:: toml + + [source_discover] + src_dir = "./frontend" + include = ["**/*.ts", "**/*.tsx"] + exclude = ["**/*.test.ts", "**/*.spec.ts"] + comment_type = "ts" diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index e9f20e27..3847ddcf 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -7,6 +7,10 @@ Upcoming -------- - ⬆️ Support and test sphinx-needs v5-8 +- ✨ Added TypeScript comment type support for source discovery and analysis. + + TypeScript files can now be processed using ``comment_type = "ts"``. + Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. .. _`release:1.2.0`: @@ -31,7 +35,6 @@ New and Improved Warning messages now include more context to help diagnose parsing issues. - 📚 Added traceability page to the documentation. - - 📚 Added ``features.rst`` page documenting the full feature set with source tracing. Fixes @@ -41,7 +44,6 @@ Fixes Leading and trailing spaces in extracted marker content are now correctly stripped. - .. _`release:1.1.0`: 1.1.0 diff --git a/pyproject.toml b/pyproject.toml index 93cb917e..8dc6e873 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ # https://github.com/tree-sitter/py-tree-sitter/issues/386#issuecomment-3101430799 "tree-sitter~=0.25.1", "tree-sitter-c-sharp>=0.23.1", + "tree-sitter-typescript>=0.23.2", "tree-sitter-yaml>=0.7.1", "tree-sitter-rust>=0.23.0", ] diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 5a11fddb..b96cd5ca 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -19,6 +19,13 @@ # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP] CommentType.cpp: {"function_definition", "class_definition"}, CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"}, + CommentType.ts: { + "function_declaration", + "class_declaration", + "method_definition", + "lexical_declaration", + "variable_declaration", + }, CommentType.yaml: {"block_mapping_pair", "block_sequence_item", "document"}, # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST]; CommentType.rust: { @@ -55,6 +62,7 @@ """ CPP_QUERY = """(comment) @comment""" C_SHARP_QUERY = """(comment) @comment""" +TYPE_SCRIPT_QUERY = """(comment) @comment""" YAML_QUERY = """(comment) @comment""" RUST_QUERY = """ (line_comment) @comment @@ -94,6 +102,11 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: parsed_language = Language(tree_sitter_c_sharp.language()) query = Query(parsed_language, C_SHARP_QUERY) + elif comment_type == CommentType.ts: + import tree_sitter_typescript # noqa: PLC0415 + + parsed_language = Language(tree_sitter_typescript.language_typescript()) + query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: import tree_sitter_yaml # noqa: PLC0415 diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index 51ae3c08..45a51fd3 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -9,6 +9,7 @@ "cpp": ["c", "ci", "cpp", "cc", "cxx", "h", "hpp", "hxx", "hh", "ihl"], "python": ["py"], "cs": ["cs"], + "ts": ["ts", "tsx"], "yaml": ["yml", "yaml"], "rust": ["rs"], } @@ -18,6 +19,7 @@ class CommentType(str, Enum): python = "python" cpp = "cpp" cs = "cs" + ts = "ts" yaml = "yaml" # @Support Rust style comments, IMPL_RUST_1, impl, [FE_RUST]; rust = "rust" diff --git a/tests/data/discover_fixtures.json b/tests/data/discover_fixtures.json index 7ffd5d49..738c619c 100644 --- a/tests/data/discover_fixtures.json +++ b/tests/data/discover_fixtures.json @@ -75,7 +75,9 @@ "config": { "src_dir": "src", "include": [], - "exclude": ["**/build/**"], + "exclude": [ + "**/build/**" + ], "gitignore": false, "comment_type": "cpp" }, @@ -94,7 +96,9 @@ }, "config": { "src_dir": "src", - "include": ["**/*.cpp"], + "include": [ + "**/*.cpp" + ], "exclude": [], "gitignore": false, "comment_type": "cpp" @@ -115,8 +119,12 @@ }, "config": { "src_dir": "src", - "include": ["**/*.cpp"], - "exclude": ["**/test_*.cpp"], + "include": [ + "**/*.cpp" + ], + "exclude": [ + "**/test_*.cpp" + ], "gitignore": false, "comment_type": "cpp" }, @@ -280,7 +288,9 @@ "config": { "src_dir": "src", "include": [], - "exclude": ["**/test_*.cpp"], + "exclude": [ + "**/test_*.cpp" + ], "gitignore": true, "comment_type": "cpp" }, @@ -386,5 +396,27 @@ "expected": [ "src/Program.cs" ] + }, + { + "name": "typescript_comment_type", + "description": "TypeScript comment type discovers .ts and .tsx files", + "git_init": false, + "files": { + "src/main.ts": "// main", + "src/component.tsx": "// component", + "src/main.cpp": "// not ts", + "src/util.py": "# not ts" + }, + "config": { + "src_dir": "src", + "include": [], + "exclude": [], + "gitignore": false, + "comment_type": "ts" + }, + "expected": [ + "src/component.tsx", + "src/main.ts" + ] } -] +] \ No newline at end of file diff --git a/tests/data/typescript/demo.ts b/tests/data/typescript/demo.ts new file mode 100644 index 00000000..66aade0f --- /dev/null +++ b/tests/data/typescript/demo.ts @@ -0,0 +1,17 @@ +// regular comment +function testA() { + // @type,TS_REQ_002,TypeScript one-line test + return 1; +} + +/* regular block comment */ +const testB = () => { + return 2; +}; + +// another comment +class Demo { + methodA() { + return 3; + } +} diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 6e6c2a7f..500451c8 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -55,7 +55,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): @pytest.mark.parametrize( - "src_dir, src_paths , oneline_comment_style, result", + "src_dir, src_paths, comment_type, oneline_comment_style, result", [ ( TEST_DIR / "data" / "dcdc", @@ -65,6 +65,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", TEST_DIR / "data" / "dcdc" / "supercharge.cpp", ], + "cpp", ONELINE_COMMENT_STYLE, { "num_src_files": 4, @@ -79,6 +80,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): [ TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", ], + "cpp", ONELINE_COMMENT_STYLE, { "num_src_files": 1, @@ -94,6 +96,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): [ TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", ], + "cpp", ONELINE_COMMENT_STYLE_DEFAULT, { "num_src_files": 1, @@ -109,6 +112,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): [ TEST_DIR / "data" / "rust" / "demo.rs", ], + "rust", ONELINE_COMMENT_STYLE_DEFAULT, { "num_src_files": 1, @@ -118,10 +122,25 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_oneline_warnings": 0, }, ), + ( + TEST_DIR / "data" / "typescript", + [ + TEST_DIR / "data" / "typescript" / "demo.ts", + ], + "ts", + ONELINE_COMMENT_STYLE_DEFAULT, + { + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_comments": 4, + "num_oneline_warnings": 0, + }, + ), ], ) def test_analyse_oneline_needs( - tmp_path, src_dir, src_paths, oneline_comment_style, result + tmp_path, src_dir, src_paths, comment_type, oneline_comment_style, result ): src_analyse_config = SourceAnalyseConfig( src_files=src_paths, @@ -130,6 +149,7 @@ def test_analyse_oneline_needs( get_oneline_needs=True, get_rst=False, oneline_comment_style=oneline_comment_style, + comment_type=comment_type, ) src_analyse = SourceAnalyse(src_analyse_config) src_analyse.run() diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index 207b1f9a..c1a2dc83 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -10,6 +10,7 @@ import tree_sitter_cpp import tree_sitter_python import tree_sitter_rust +import tree_sitter_typescript import tree_sitter_yaml from sphinx_codelinks.analyse import utils @@ -57,6 +58,14 @@ def init_rust_tree_sitter() -> tuple[Parser, Query]: return parser, query +@pytest.fixture(scope="session") +def init_typescript_tree_sitter() -> tuple[Parser, Query]: + parsed_language = Language(tree_sitter_typescript.language_typescript()) + query = Query(parsed_language, utils.TYPE_SCRIPT_QUERY) + parser = Parser(parsed_language) + return parser, query + + @pytest.mark.parametrize( ("code", "result"), [ @@ -365,6 +374,41 @@ def test_find_associated_scope_rust(code, result, init_rust_tree_sitter): assert result in rust_def +@pytest.mark.parametrize( + ("code", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ( + b""" + class DummyClass { + // @req-id: need_001 + method1() { + } + } + """, + "method1()", + ), + ], +) +def test_find_associated_scope_typescript(code, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments = utils.extract_comments(code, parser, query) + node: TreeSitterNode | None = utils.find_associated_scope( + comments[0], CommentType.ts + ) + assert node + assert node.text + ts_def = node.text.decode("utf-8") + assert result in ts_def + + @pytest.mark.parametrize( ("code", "result"), [ @@ -519,6 +563,29 @@ def test_find_next_scope_csharp(code, result, init_csharp_tree_sitter): assert result in func_def +@pytest.mark.parametrize( + ("code", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ], +) +def test_find_next_scope_typescript(code, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments = utils.extract_comments(code, parser, query) + node: TreeSitterNode | None = utils.find_next_scope(comments[0], CommentType.ts) + assert node + assert node.text + func_def = node.text.decode("utf-8") + assert result in func_def + + @pytest.mark.parametrize( ("code", "result"), [ @@ -773,6 +840,37 @@ def test_csharp_comment(code, num_comments, result, init_csharp_tree_sitter): assert comments[0].text.decode("utf-8") == result +@pytest.mark.parametrize( + ("code", "num_comments", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + 1, + "// @req-id: need_001", + ), + ( + b""" + /* @req-id: need_001 */ + const value = 1; + """, + 1, + "/* @req-id: need_001 */", + ), + ], +) +def test_typescript_comment(code, num_comments, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments: list[TreeSitterNode] = utils.extract_comments(code, parser, query) + comments.sort(key=lambda x: x.start_point.row) + assert len(comments) == num_comments + assert comments[0].text + assert comments[0].text.decode("utf-8") == result + + @pytest.mark.parametrize( ("code", "num_comments", "result"), [ diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index 5a6f5c1f..30decf4c 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -49,7 +49,7 @@ "comment_type": "java", }, [ - "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'cs', 'python', 'rust', 'yaml']" + "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'cs', 'python', 'rust', 'ts', 'yaml']" ], ), ( @@ -99,6 +99,13 @@ def test_schema_negative(config, msgs): "gitignore": True, "comment_type": "python", }, + { + "src_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "comment_type": "ts", + }, { "src_dir": "/path/to/root", "follow_links": True, @@ -182,6 +189,7 @@ def create_source_files(tmp_path: Path) -> Path: [ ("cpp", len(COMMENT_FILETYPE["cpp"])), ("python", len(COMMENT_FILETYPE["python"])), + ("ts", len(COMMENT_FILETYPE["ts"])), ], ) def test_comment_filetype( diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py index 8e87a71e..2ecc3fd1 100644 --- a/tests/test_src_trace.py +++ b/tests/test_src_trace.py @@ -58,7 +58,7 @@ [ "Project 'dcdc' has the following errors:", "Schema validation error in field 'exclude': 123 is not of type 'string'", - "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'cs', 'python', 'rust', 'yaml']", + "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'cs', 'python', 'rust', 'ts', 'yaml']", "Schema validation error in field 'gitignore': '_true' is not of type 'boolean'", "Schema validation error in field 'include': 345 is not of type 'string'", "Schema validation error in field 'src_dir': ['../dcdc'] is not of type 'string'", From 4e9ef51d27534fffd4e67f29487388dd8f44feb8 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 22 Jun 2026 16:17:25 +0200 Subject: [PATCH 02/20] =?UTF-8?q?=E2=9C=A8=20Update=20changelog:=20Add=20T?= =?UTF-8?q?ypeScript=20comment=20type=20support=20for=20source=20discovery?= =?UTF-8?q?=20and=20analysis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/development/change_log.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index fe2e1384..4901b69a 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -3,6 +3,14 @@ Changelog ========= +Under development +----------------- + +- ✨ Added TypeScript comment type support for source discovery and analysis. + + TypeScript files can now be processed using ``comment_type = "ts"``. + Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. + .. _`release:1.3.0`: 1.3.0 @@ -23,11 +31,6 @@ New and Improved Comments in JSONC files are now parsed for need ID references and one-line need definitions. ``.json`` files are also checked when they begin with a comment (see jsonc.org). -- ✨ Added TypeScript comment type support for source discovery and analysis. - - TypeScript files can now be processed using ``comment_type = "ts"``. - Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. - - 👌 Replaced ``gitignore-parser`` with ``ignore-python`` for source discovery. This adds native nested ``.gitignore`` support, improves performance, and brings behavioral From 4d8509481f5a11e40de5b88f8e6205b423990c49 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 22 Jun 2026 16:18:13 +0200 Subject: [PATCH 03/20] =?UTF-8?q?=E2=9C=A8=20Update=20changelog:=20Add=20T?= =?UTF-8?q?ypeScript=20support=20details=20for=20source=20discovery=20and?= =?UTF-8?q?=20analysis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/development/change_log.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 4901b69a..cfb8246a 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -6,11 +6,17 @@ Changelog Under development ----------------- +New and Improved +................ + - ✨ Added TypeScript comment type support for source discovery and analysis. TypeScript files can now be processed using ``comment_type = "ts"``. Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. +Fixes +..... + .. _`release:1.3.0`: 1.3.0 From 28e12e1c07e31cc8f58fff3c91beaa77d10e5852 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 22 Jun 2026 16:28:21 +0200 Subject: [PATCH 04/20] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Resolve=20CI=20pre-?= =?UTF-8?q?commit=20and=20docs=20build=20failures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/components/configuration.rst | 10 +-- tests/data/discover_fixtures.json | 2 +- tests/test_analyse.py | 99 ++++++++++++------------ 3 files changed, 55 insertions(+), 56 deletions(-) diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index cdad9d8b..8e6d36d3 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -304,11 +304,11 @@ Specifies the comment syntax style used in the source code files. This determine ``/* */`` (multi-line), ``///`` (XML doc comments) - ``.cs`` - * - TypeScript - - ``"ts"`` - - ``//`` (single-line), - ``/* */`` (multi-line) - - ``.ts``, ``.tsx`` + * - TypeScript + - ``"ts"`` + - ``//`` (single-line), + ``/* */`` (multi-line) + - ``.ts``, ``.tsx`` * - YAML - ``"yaml"`` - ``#`` (single-line) diff --git a/tests/data/discover_fixtures.json b/tests/data/discover_fixtures.json index 738c619c..5959caba 100644 --- a/tests/data/discover_fixtures.json +++ b/tests/data/discover_fixtures.json @@ -419,4 +419,4 @@ "src/main.ts" ] } -] \ No newline at end of file +] diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 2973ed6e..fe7098a8 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -56,34 +56,34 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): @pytest.mark.parametrize( - "src_dir, src_paths, comment_type, oneline_comment_style, result", + "case", [ - ( - TEST_DIR / "data" / "dcdc", - [ + { + "src_dir": TEST_DIR / "data" / "dcdc", + "src_paths": [ TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", TEST_DIR / "data" / "dcdc" / "charge" / "demo_2.cpp", TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", TEST_DIR / "data" / "dcdc" / "supercharge.cpp", ], - "cpp", - ONELINE_COMMENT_STYLE, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE, + "result": { "num_src_files": 4, "num_uncached_files": 4, "num_cached_files": 0, "num_comments": 29, "num_oneline_warnings": 0, }, - ), - ( - TEST_DIR / "data" / "oneline_comment_basic", - [ + }, + { + "src_dir": TEST_DIR / "data" / "oneline_comment_basic", + "src_paths": [ TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", ], - "cpp", - ONELINE_COMMENT_STYLE, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, @@ -91,15 +91,15 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_oneline_warnings": 0, "warnings_path_exists": True, }, - ), - ( - TEST_DIR / "data" / "oneline_comment_default", - [ + }, + { + "src_dir": TEST_DIR / "data" / "oneline_comment_default", + "src_paths": [ TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", ], - "cpp", - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, @@ -107,69 +107,68 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_oneline_warnings": 1, "warnings_path_exists": True, }, - ), - ( - TEST_DIR / "data" / "rust", - [ + }, + { + "src_dir": TEST_DIR / "data" / "rust", + "src_paths": [ TEST_DIR / "data" / "rust" / "demo.rs", ], - "rust", - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": "rust", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 6, "num_oneline_warnings": 0, }, - ), - ( - TEST_DIR / "data" / "typescript", - [ + }, + { + "src_dir": TEST_DIR / "data" / "typescript", + "src_paths": [ TEST_DIR / "data" / "typescript" / "demo.ts", ], - "ts", - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": "ts", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, }, - ), - ( - TEST_DIR / "data" / "jsonc", - [ + }, + { + "src_dir": TEST_DIR / "data" / "jsonc", + "src_paths": [ TEST_DIR / "data" / "jsonc" / "demo.jsonc", ], - CommentType.jsonc, - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": CommentType.jsonc, + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, }, - ), + }, ], ) -def test_analyse_oneline_needs( - tmp_path, src_dir, src_paths, comment_type, oneline_comment_style, result -): +def test_analyse_oneline_needs(tmp_path, case): src_analyse_config = SourceAnalyseConfig( - src_files=src_paths, - src_dir=src_dir, + src_files=case["src_paths"], + src_dir=case["src_dir"], get_need_id_refs=False, get_oneline_needs=True, get_rst=False, - oneline_comment_style=oneline_comment_style, - comment_type=comment_type, + oneline_comment_style=case["oneline_comment_style"], + comment_type=case["comment_type"], ) src_analyse = SourceAnalyse(src_analyse_config) src_analyse.run() + result = case["result"] assert len(src_analyse.src_files) == result["num_src_files"] assert len(src_analyse.oneline_warnings) == result["num_oneline_warnings"] From 5e7d48beadf14a7f8ec9d3d0ffa5797f209cf4b8 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 13 Jul 2026 20:48:38 +0200 Subject: [PATCH 05/20] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20CLAUDE.md=20for?= =?UTF-8?q?=20project=20guidance=20and=20command=20usage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..4d3014e1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,94 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +This repository already has a detailed **AGENTS.md** at the repo root — read it first for +full architecture diagrams, event-handler tables, commit/PR conventions, and common-pattern +recipes (adding a language, a marker type, a CLI command, a config option). This file only +covers what's needed to get moving quickly. + +## What this project is + +sphinx-codelinks is a Sphinx extension providing fast source-code traceability for +Sphinx-Needs: it scans source files (C++, Python, C#, Rust, TypeScript, Go, YAML, JSON) for +marker comments via tree-sitter, and generates Sphinx-Needs items / RST that link +documentation back to exact source locations. + +## Commands + +All commands run through `tox` (uses `tox-uv`). + +```bash +# Run default test env (py312-sphinx8-needs5) +tox + +# List all test env combinations (py{312,313,314}-sphinx{7,8,9}-needs{5,6,7,8}) +tox -a + +# Run a specific env / file / test +tox -e py312-sphinx8-needs5 +tox -e py312-sphinx8-needs5 -- tests/test_analyse.py +tox -e py312-sphinx8-needs5 -- tests/test_analyse.py::test_function_name + +# Update syrupy snapshots +tox -e py312-sphinx8-needs5 -- --snapshot-update + +# Type check / lint / format +tox -e mypy +tox -e ruff-check +tox -e ruff-fmt +pre-commit run --all-files + +# Docs +tox -e docs-clean +tox -e docs-update +BUILDER=linkcheck tox -e docs-clean +tox -e docs-live + +# End-to-end demo (analyse -> write RST -> build docs) +tox -e demo +``` + +The CLI itself is installed as `codelinks` (`codelinks analyse `, +`codelinks write rst --outpath `). + +## Architecture + +Pipeline: **Source Files → Discovery → Parsing → Analysis → Results (JSON) → RST Generation** + +- `source_discover/` — finds source files by include/exclude patterns, respects `.gitignore`. +- `analyse/oneline_parser.py` — tree-sitter based parser extracting comment marker nodes. +- `analyse/projects.py` — per-language analyzers, registered in a `LANGUAGE_ANALYZERS` dict. +- `analyse/analyse.py` — orchestrates discovery + parsing + analysis into `analyse/models.py` + Pydantic result models. +- `needextend_write.py` — turns analysis JSON into RST with Sphinx-Needs `needextend` + directives. +- `config.py` — Pydantic v2 config models (`AnalyseConfig` etc.), loadable from TOML. +- `sphinx_extension/source_tracing.py` — the Sphinx extension `setup()`; wires into Sphinx + build events (`config-inited`, `builder-inited`, `env-before-read-docs`, + `html-collect-pages`, `html-page-context`, `build-finished`) to register sphinx-needs extra + options/types, generate standalone traced-source HTML pages, and inject CSS + (`sphinx_extension/ub_sct.css`). See AGENTS.md for the full event table and mermaid diagram. + +Adding a new language analyzer, marker type, CLI command, or config option each follow a +short recipe documented in AGENTS.md under "Common Patterns" — follow those rather than +inventing a new approach. + +## Code style + +- Ruff for lint/format (strict rule set incl. `S`, `PL`, `PTH`, `SIM`, `SLF`; see + `pyproject.toml` for per-file ignores). +- Mypy strict mode (`disallow_any_*`, `disallow_untyped_*`); relaxed for `tests/*` and + `sphinx_codelinks.*` via overrides in `pyproject.toml`. +- Full type annotations everywhere; Pydantic models (frozen where possible) for config/data. +- Sphinx-style docstrings (`:param:`, `:return:`, `:raises:`), no types in docstrings. +- Prefer pure functions and immutable data structures. + +## Testing + +- `pytest` with fixtures in `tests/conftest.py`; test data in `tests/data/`; Sphinx + integration tests use real minimal Sphinx projects in `tests/doc_test/`. +- `syrupy` for snapshot testing of complex outputs (JSON, doctrees) — use + `snapshot.assert_match()` and re-run with `--snapshot-update` when output intentionally + changes. +- Use `@pytest.mark.parametrize` for multi-language / multi-scenario tests. From b36c271396a42d26e9cd7bba5e04a023acc773d5 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 13 Jul 2026 20:57:42 +0200 Subject: [PATCH 06/20] =?UTF-8?q?=E2=9C=A8=20Add=20TypeScript=20support=20?= =?UTF-8?q?for=20TSX=20files=20and=20enhance=20related=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sphinx_codelinks/analyse/utils.py | 45 +++++++++++-- tests/data/typescript/demo.tsx | 4 ++ tests/test_analyse.py | 23 +++++++ tests/test_analyse_utils.py | 94 ++++++++++++++++++++++++++- 4 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 tests/data/typescript/demo.tsx diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 8fcef2e2..050a2c17 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -131,7 +131,10 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: elif comment_type == CommentType.ts: import tree_sitter_typescript # noqa: PLC0415 - parsed_language = Language(tree_sitter_typescript.language_typescript()) + # The TSX grammar is a strict superset of the TypeScript grammar (it also + # parses plain .ts fine), so use it for both to support .tsx files without + # needing a per-file grammar choice. + parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: import tree_sitter_yaml # noqa: PLC0415 @@ -181,6 +184,38 @@ def extract_comments( return captures.get("comment") +TS_FUNCTION_VALUE_TYPES = {"arrow_function", "function_expression"} + + +def _is_function_like_lexical_declaration(node: TreeSitterNode) -> bool: + """True if a TS lexical/variable declaration's declarator is a function. + + ``const``/``let``/``var`` declarations are only treated as scopes when they + assign a function or arrow function, so a leading comment doesn't bind to an + unrelated ``const`` that merely precedes the function it documents. + """ + for declarator in node.named_children: + if declarator.type != "variable_declarator": + continue + value = declarator.child_by_field_name("value") + if value is not None and value.type in TS_FUNCTION_VALUE_TYPES: + return True + return False + + +def _matches_scope( + node: TreeSitterNode, scope_types: set[str], comment_type: CommentType +) -> bool: + if node.type not in scope_types: + return False + if comment_type == CommentType.ts and node.type in { + "lexical_declaration", + "variable_declaration", + }: + return _is_function_like_lexical_declaration(node) + return True + + def find_enclosing_scope( node: TreeSitterNode, comment_type: CommentType = CommentType.cpp ) -> TreeSitterNode | None: @@ -188,7 +223,7 @@ def find_enclosing_scope( scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp]) current: TreeSitterNode = node while current: - if current.type in scope_types: + if _matches_scope(current, scope_types, comment_type): return current current: TreeSitterNode | None = current.parent # type: ignore[no-redef] # required for node traversal return None @@ -201,12 +236,12 @@ def find_next_scope( scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp]) current: TreeSitterNode = node while current: - if current.type in scope_types: + if _matches_scope(current, scope_types, comment_type): return current current: TreeSitterNode | None = current.next_named_sibling # type: ignore[no-redef] # required for node traversal - if current and current.type == "block": + if current and current.type in {"block", "export_statement"}: for child in current.named_children: - if child.type in scope_types: + if _matches_scope(child, scope_types, comment_type): return child return None diff --git a/tests/data/typescript/demo.tsx b/tests/data/typescript/demo.tsx new file mode 100644 index 00000000..a7e78969 --- /dev/null +++ b/tests/data/typescript/demo.tsx @@ -0,0 +1,4 @@ +// @type,TS_REQ_003,TypeScript JSX component test +export function Button() { + return ; +} diff --git a/tests/test_analyse.py b/tests/test_analyse.py index fe7098a8..cad5bfb2 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -74,6 +74,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 29, "num_oneline_warnings": 0, + "num_oneline_needs": 12, }, }, { @@ -89,6 +90,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 14, "num_oneline_warnings": 0, + "num_oneline_needs": 8, "warnings_path_exists": True, }, }, @@ -105,6 +107,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 5, "num_oneline_warnings": 1, + "num_oneline_needs": 4, "warnings_path_exists": True, }, }, @@ -121,6 +124,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 6, "num_oneline_warnings": 0, + "num_oneline_needs": 4, }, }, { @@ -136,6 +140,23 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, + "num_oneline_needs": 1, + }, + }, + { + "src_dir": TEST_DIR / "data" / "typescript", + "src_paths": [ + TEST_DIR / "data" / "typescript" / "demo.tsx", + ], + "comment_type": "ts", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_comments": 1, + "num_oneline_warnings": 0, + "num_oneline_needs": 1, }, }, { @@ -151,6 +172,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, + "num_oneline_needs": 3, }, }, ], @@ -171,6 +193,7 @@ def test_analyse_oneline_needs(tmp_path, case): result = case["result"] assert len(src_analyse.src_files) == result["num_src_files"] assert len(src_analyse.oneline_warnings) == result["num_oneline_warnings"] + assert len(src_analyse.oneline_needs) == result["num_oneline_needs"] cnt_comments = 0 for src_file in src_analyse.src_files: diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index 95f5b9b2..98410148 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -62,7 +62,9 @@ def init_rust_tree_sitter() -> tuple[Parser, Query]: @pytest.fixture(scope="session") def init_typescript_tree_sitter() -> tuple[Parser, Query]: - parsed_language = Language(tree_sitter_typescript.language_typescript()) + # TSX grammar is a superset of the TypeScript grammar (parses plain .ts too), + # matching what utils.init_tree_sitter uses for CommentType.ts. + parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, utils.TYPE_SCRIPT_QUERY) parser = Parser(parsed_language) return parser, query @@ -455,6 +457,64 @@ class DummyClass { """, "method1()", ), + # leading comment on an exported function must descend into + # export_statement, not resolve to no scope + ( + b""" + // @req-id: need_001 + export function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + # leading comment on an exported class + ( + b""" + // @req-id: need_001 + export class DummyClass { + } + """, + "class DummyClass", + ), + # leading comment on an exported const arrow function + ( + b""" + // @req-id: need_001 + export const dummyFunc1 = () => { + }; + """, + "dummyFunc1", + ), + # a plain (non-function) const between the comment and the function it + # documents must not steal the association + ( + b""" + // @req-id: need_001 + const helperFlag = true; + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + # arrow-function const still resolves to itself + ( + b""" + // @req-id: need_001 + const dummyFunc1 = () => { + }; + """, + "dummyFunc1", + ), + # JSX-returning component parses cleanly and resolves scope (.tsx content) + ( + b""" + // @req-id: need_001 + export function Button() { + return ; + } + """, + "function Button()", + ), ], ) def test_find_associated_scope_typescript(code, result, init_typescript_tree_sitter): @@ -634,6 +694,23 @@ def test_find_next_scope_csharp(code, result, init_csharp_tree_sitter): """, "function dummyFunc1()", ), + ( + b""" + // @req-id: need_001 + export function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ( + b""" + // @req-id: need_001 + const helperFlag = true; + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), ], ) def test_find_next_scope_typescript(code, result, init_typescript_tree_sitter): @@ -646,6 +723,21 @@ def test_find_next_scope_typescript(code, result, init_typescript_tree_sitter): assert result in func_def +def test_find_associated_scope_typescript_jsx_no_parse_error( + init_typescript_tree_sitter, +): + """A JSX-returning component must parse cleanly under the TSX grammar.""" + code = b""" + // @req-id: need_001 + export function Button() { + return ; + } + """ + parser, _ = init_typescript_tree_sitter + tree = parser.parse(code) + assert not tree.root_node.has_error + + @pytest.mark.parametrize( ("code", "result"), [ From 852e50c821e416b480a914d322e586300c23fecd Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Fri, 7 Aug 2026 16:13:37 +0200 Subject: [PATCH 07/20] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20Add=20TypeScript?= =?UTF-8?q?=20declarative=20extraction=20fixture=20cases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...[oneline-default_oneliner_typescript].json | 19 ++++++++++++++ ...ion_fixture[oneline-jsx_oneliner_tsx].json | 19 ++++++++++++++ tests/data/extraction/README.md | 2 +- tests/data/extraction/oneline.yaml | 25 +++++++++++++++++++ tests/test_extraction_fixtures.py | 4 +++ 5 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json new file mode 100644 index 00000000..c53a7fbc --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_TS", + "title": "Ts Title", + "type": "impl", + "links": { + "links": [ + "REQ_TS" + ] + }, + "metadata": {}, + "line": 1 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json new file mode 100644 index 00000000..b1d1f534 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_TSX", + "title": "Tsx Title", + "type": "impl", + "links": { + "links": [ + "REQ_TSX" + ] + }, + "metadata": {}, + "line": 4 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/README.md b/tests/data/extraction/README.md index dd845c70..2dd95f16 100644 --- a/tests/data/extraction/README.md +++ b/tests/data/extraction/README.md @@ -12,7 +12,7 @@ Each `*.yaml` file in this directory is a map of `case_name → case`: ```yaml default_oneliner_cpp: - lang: cpp # cpp | c | python | csharp | rust | yaml | go | jsonc | bash + lang: cpp # cpp | c | python | csharp | rust | yaml | go | jsonc | bash | typescript | tsx config: default # "default", or an inline config block (see below) source: | // @My Title, IMPL_1, impl, [REQ_1] diff --git a/tests/data/extraction/oneline.yaml b/tests/data/extraction/oneline.yaml index 9a423ae4..79762245 100644 --- a/tests/data/extraction/oneline.yaml +++ b/tests/data/extraction/oneline.yaml @@ -57,3 +57,28 @@ shebang_oneliner_bash: #!/bin/bash # @Bash Title, IMPL_BASH_SHEBANG, impl, [REQ_BASH] function greet { echo hi; } + +default_oneliner_typescript: + lang: typescript + config: default + source: | + // @Ts Title, IMPL_TS, impl, [REQ_TS] + function f(): void {} + +# a JSX comment ({/* ... */}) is an ordinary block-comment node in the TSX +# grammar; the marker inside one must extract like any /* */ comment. The +# marker sits on its own line: with the default newline end_sequence, a +# marker sharing a line with the closing */ swallows the */ into its last +# field (pre-existing engine behavior; deliberately not exercised by the +# shared fixtures) +jsx_oneliner_tsx: + lang: tsx + config: default + source: | + const App = () => ( +
+ {/* + @Tsx Title, IMPL_TSX, impl, [REQ_TSX] + */} +
+ ); diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 1a0e560b..a51e2e03 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -36,6 +36,10 @@ "go": (CommentType.go, "go"), "jsonc": (CommentType.jsonc, "jsonc"), "bash": (CommentType.bash, "sh"), + "typescript": (CommentType.ts, "ts"), + # `.tsx` runs the same TSX grammar; the distinct extension + JSX source + # exercises the superset-grammar decision end to end. + "tsx": (CommentType.ts, "tsx"), } From 0330c0b748e6dad4c8b077dfa681b0b34935dbef Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Fri, 7 Aug 2026 16:24:06 +0200 Subject: [PATCH 08/20] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20Trace=20TypeScript?= =?UTF-8?q?=20support=20(FE=5FTS=20feature=20and=20impl=20markers)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/components/features.rst | 26 +++++++++++++++++++ src/sphinx_codelinks/analyse/utils.py | 4 ++- .../source_discover/config.py | 1 + 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index f342bc4b..9edea7ac 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -266,6 +266,32 @@ Features .. fault:: Sphinx-codelinks hallucinates traceability objects in Bash :id: FAULT_BASH_2 +.. feature:: TypeScript Language Support + :id: FE_TS + + Support for defining traceability objects in TypeScript source files via one-line + comment annotations. + + The TypeScript language parser leverages tree-sitter to accurately identify and + extract comments from TypeScript sources, including single-line (``//``) and + multi-line (``/* */``) comment styles. All files are parsed with the TSX + grammar — a strict superset of the TypeScript grammar — so ``.tsx`` files + (including JSX comments such as ``{/* ... */}``) need no per-file grammar choice. + + Key capabilities: + + * Detection of inline and block comments + * Association of comments with function, class, and method declarations + * ``const``/``let``/``var`` declarations count as scopes only when they assign + a function or arrow function + * File extensions ``.ts`` and ``.tsx`` auto-discovered when ``comment_type = "ts"`` + + .. fault:: Traceability objects are not detected in TypeScript language + :id: FAULT_TS_1 + + .. fault:: Sphinx-codelinks hallucinates traceability objects in TypeScript + :id: FAULT_TS_2 + .. feature:: Preprocessor-Aware C/C++ Extraction :id: FE_PREPROC diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index fe4109a1..6c5af640 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -28,6 +28,7 @@ # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP] CommentType.cpp: {"function_definition", "class_definition"}, CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"}, + # @TypeScript Scope Node Types, IMPL_TS_2, impl, [FE_TS] CommentType.ts: { "function_declaration", "class_declaration", @@ -73,6 +74,7 @@ """ CPP_QUERY = """(comment) @comment""" C_SHARP_QUERY = """(comment) @comment""" +# @TypeScript comment query for tree-sitter, IMPL_TS_3, impl, [FE_TS] TYPE_SCRIPT_QUERY = """(comment) @comment""" YAML_QUERY = """(comment) @comment""" RUST_QUERY = """ @@ -115,7 +117,7 @@ def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: return False -# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH] +# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH, FE_TS] def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: if comment_type == CommentType.cpp: import tree_sitter_cpp # noqa: PLC0415 diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index 70fd8649..ee4f382d 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -27,6 +27,7 @@ class CommentType(str, Enum): python = "python" cpp = "cpp" cs = "cs" + # @Support TypeScript style comments, IMPL_TS_1, impl, [FE_TS]; ts = "ts" yaml = "yaml" # @Support Rust style comments, IMPL_RUST_1, impl, [FE_RUST]; From 2ceb5b4ea62b351e9101fa9d6662e2fe87c250c5 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Fri, 7 Aug 2026 16:51:19 +0200 Subject: [PATCH 09/20] =?UTF-8?q?=F0=9F=94=A7=20MAINTAIN:=20Clarify=20tsx?= =?UTF-8?q?=20fixture=20comment=20and=20complete=20README=20lang=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/data/extraction/README.md | 2 +- tests/test_extraction_fixtures.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/data/extraction/README.md b/tests/data/extraction/README.md index 2dd95f16..7df71687 100644 --- a/tests/data/extraction/README.md +++ b/tests/data/extraction/README.md @@ -12,7 +12,7 @@ Each `*.yaml` file in this directory is a map of `case_name → case`: ```yaml default_oneliner_cpp: - lang: cpp # cpp | c | python | csharp | rust | yaml | go | jsonc | bash | typescript | tsx + lang: cpp # cpp | c | cpp_header | python | csharp | rust | yaml | go | jsonc | bash | typescript | tsx config: default # "default", or an inline config block (see below) source: | // @My Title, IMPL_1, impl, [REQ_1] diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index a51e2e03..881d7532 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -37,8 +37,11 @@ "jsonc": (CommentType.jsonc, "jsonc"), "bash": (CommentType.bash, "sh"), "typescript": (CommentType.ts, "ts"), - # `.tsx` runs the same TSX grammar; the distinct extension + JSX source - # exercises the superset-grammar decision end to end. + # `.tsx` is documentary: extraction never reads the file suffix, and the + # plain-TS and TSX grammars lex comments identically (the TSX-grammar + # choice is pinned by test_analyse_utils.py's has_error check instead). + # What this case pins: a marker on its own line inside a multi-line JSX + # block comment anchors to that line. "tsx": (CommentType.ts, "tsx"), } From 279c935577dd43d6700479a2a5a64292b029ea16 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sun, 9 Aug 2026 20:21:40 +0200 Subject: [PATCH 10/20] =?UTF-8?q?=E2=9C=A8=20NEW:=20Widen=20ts=20comment?= =?UTF-8?q?=20type=20to=20full=20TS/JS=20family?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TSX grammar used for comment_type = "ts" is a strict superset of TypeScript, which is itself a superset of JavaScript, so it already parses .mts/.cts (TypeScript's own ESM/CJS variants) and the full .js/.jsx/.mjs/.cjs JavaScript family with no new grammar dependency. --- docs/source/components/analyse.rst | 2 +- docs/source/components/configuration.rst | 5 +++-- docs/source/components/features.rst | 16 +++++++++------- docs/source/development/change_log.rst | 6 ++++-- src/sphinx_codelinks/analyse/utils.py | 7 ++++--- src/sphinx_codelinks/source_discover/config.py | 7 ++++++- tests/data/discover_fixtures.json | 16 ++++++++++++++-- 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 228163b9..4a5dcec7 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -47,7 +47,7 @@ Limitations **Current Limitations:** -- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), TypeScript (``//``, ``/* */``), Python (``#``), YAML (``#``), Rust (``//``, ``/* */``, ``///``), Go (``//``, ``/* */``), JSONC (``//``, ``/* */``) and Bash (``#``) comment styles are supported +- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), TypeScript/JavaScript (``//``, ``/* */``), Python (``#``), YAML (``#``), Rust (``//``, ``/* */``, ``///``), Go (``//``, ``/* */``), JSONC (``//``, ``/* */``) and Bash (``#``) comment styles are supported - **Single Comment Style**: Each analysis run processes only one comment style at a time Extraction Examples diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 8f81f3d2..315d4398 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -304,11 +304,12 @@ Specifies the comment syntax style used in the source code files. This determine ``/* */`` (multi-line), ``///`` (XML doc comments) - ``.cs`` - * - TypeScript + * - TypeScript / JavaScript - ``"ts"`` - ``//`` (single-line), ``/* */`` (multi-line) - - ``.ts``, ``.tsx`` + - ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, ``.mjs`` + and ``.cjs`` * - YAML - ``"yaml"`` - ``#`` (single-line) diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index 9edea7ac..021e12e3 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -269,14 +269,15 @@ Features .. feature:: TypeScript Language Support :id: FE_TS - Support for defining traceability objects in TypeScript source files via one-line - comment annotations. + Support for defining traceability objects in TypeScript and JavaScript source + files via one-line comment annotations. The TypeScript language parser leverages tree-sitter to accurately identify and - extract comments from TypeScript sources, including single-line (``//``) and - multi-line (``/* */``) comment styles. All files are parsed with the TSX - grammar — a strict superset of the TypeScript grammar — so ``.tsx`` files - (including JSX comments such as ``{/* ... */}``) need no per-file grammar choice. + extract comments from TypeScript and JavaScript sources, including single-line + (``//``) and multi-line (``/* */``) comment styles. All files are parsed with + the TSX grammar — a strict superset of the TypeScript grammar, which is in turn + a superset of JavaScript — so ``.tsx`` files (including JSX comments such as + ``{/* ... */}``) and plain JavaScript sources need no per-file grammar choice. Key capabilities: @@ -284,7 +285,8 @@ Features * Association of comments with function, class, and method declarations * ``const``/``let``/``var`` declarations count as scopes only when they assign a function or arrow function - * File extensions ``.ts`` and ``.tsx`` auto-discovered when ``comment_type = "ts"`` + * File extensions ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, + ``.mjs`` and ``.cjs`` auto-discovered when ``comment_type = "ts"`` .. fault:: Traceability objects are not detected in TypeScript language :id: FAULT_TS_1 diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 4556b14d..342d6738 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -11,8 +11,10 @@ New and Improved - ✨ Added TypeScript comment type support for source discovery and analysis. - TypeScript files can now be processed using ``comment_type = "ts"``. - Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. + TypeScript and JavaScript files can now be processed using ``comment_type = "ts"``, + since the TSX grammar used to parse them is a superset of both languages. + Source discovery supports ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, + ``.jsx``, ``.mjs`` and ``.cjs`` extensions by default. .. _`release:1.4.0`: diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 6c5af640..3bb4591e 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -137,9 +137,10 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: elif comment_type == CommentType.ts: import tree_sitter_typescript # noqa: PLC0415 - # The TSX grammar is a strict superset of the TypeScript grammar (it also - # parses plain .ts fine), so use it for both to support .tsx files without - # needing a per-file grammar choice. + # The TSX grammar is a strict superset of the TypeScript grammar, which is + # itself a superset of JavaScript, so it also parses plain .ts and the + # whole JavaScript family fine. Use it for all of them to avoid needing a + # per-file grammar choice. parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index ee4f382d..aa3a2086 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -9,7 +9,12 @@ "cpp": ["c", "ci", "cpp", "cc", "cxx", "h", "hpp", "hxx", "hh", "ihl"], "python": ["py"], "cs": ["cs"], - "ts": ["ts", "tsx"], + # ".mts"/".cts" are TypeScript's own ESM/CJS module variants. ".js"/".jsx"/ + # ".mjs"/".cjs" are JavaScript, covered by the same comment type because the + # TSX grammar used to parse "ts" sources is a strict superset of the + # TypeScript grammar, which is itself a superset of JavaScript, so no + # separate grammar or comment_type value is needed. + "ts": ["ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs"], "yaml": ["yml", "yaml"], "rust": ["rs"], "go": ["go"], diff --git a/tests/data/discover_fixtures.json b/tests/data/discover_fixtures.json index 5959caba..14516ddf 100644 --- a/tests/data/discover_fixtures.json +++ b/tests/data/discover_fixtures.json @@ -399,11 +399,17 @@ }, { "name": "typescript_comment_type", - "description": "TypeScript comment type discovers .ts and .tsx files", + "description": "TypeScript comment type discovers the full TypeScript and JavaScript family", "git_init": false, "files": { "src/main.ts": "// main", "src/component.tsx": "// component", + "src/esm.mts": "// esm module", + "src/cjs.cts": "// cjs module", + "src/script.js": "// script", + "src/widget.jsx": "// widget", + "src/esm.mjs": "// esm js", + "src/legacy.cjs": "// legacy", "src/main.cpp": "// not ts", "src/util.py": "# not ts" }, @@ -415,8 +421,14 @@ "comment_type": "ts" }, "expected": [ + "src/cjs.cts", "src/component.tsx", - "src/main.ts" + "src/esm.mjs", + "src/esm.mts", + "src/legacy.cjs", + "src/main.ts", + "src/script.js", + "src/widget.jsx" ] } ] From 41fb7a9233ac70ac77003b8a673bad42700616a1 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Mon, 10 Aug 2026 14:26:22 +0200 Subject: [PATCH 11/20] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Select=20TS/TSX=20g?= =?UTF-8?q?rammar=20per=20file=20suffix,=20not=20TSX=20for=20all?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TSX grammar is not a strict superset of TypeScript: a legacy angle-bracket type assertion (`x`), valid in `.ts`/`.mts`/`.cts`, parses as JSX under the TSX grammar and swallows the rest of the file into one `jsx_text` node, silently dropping every marker below it. Pick the tree-sitter grammar per file from its suffix instead: the plain TypeScript grammar for `.ts`/`.mts`/`.cts`, and the TSX grammar (safe for JavaScript and JSX) for everything else in the family. The parser/query pair is now built lazily per grammar in a small cache, so it is still built at most once per grammar actually used, not once per file. --- docs/source/components/features.rst | 12 +++-- docs/source/development/change_log.rst | 10 ++-- src/sphinx_codelinks/analyse/analyse.py | 23 ++++++++- src/sphinx_codelinks/analyse/utils.py | 49 ++++++++++++++++--- .../source_discover/config.py | 10 ++-- tests/test_analyse_utils.py | 39 ++++++++++++++- tests/test_extraction_fixtures.py | 13 +++-- 7 files changed, 130 insertions(+), 26 deletions(-) diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index 021e12e3..ccfd31f1 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -274,10 +274,14 @@ Features The TypeScript language parser leverages tree-sitter to accurately identify and extract comments from TypeScript and JavaScript sources, including single-line - (``//``) and multi-line (``/* */``) comment styles. All files are parsed with - the TSX grammar — a strict superset of the TypeScript grammar, which is in turn - a superset of JavaScript — so ``.tsx`` files (including JSX comments such as - ``{/* ... */}``) and plain JavaScript sources need no per-file grammar choice. + (``//``) and multi-line (``/* */``) comment styles. The grammar is chosen per + file from its extension: ``.ts``, ``.mts``, and ``.cts`` — TypeScript's own + module variants — are parsed with the plain TypeScript grammar, since a legacy + angle-bracket type assertion (``x``) is valid there but is JSX syntax + under the TSX grammar. Every other extension (``.tsx``, ``.jsx``, ``.js``, + ``.mjs``, ``.cjs``) is parsed with the TSX grammar, which is safe for plain + JavaScript and additionally handles JSX (including JSX comments such as + ``{/* ... */}``) embedded in ``.tsx`` or ``.js`` sources. Key capabilities: diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 342d6738..b63bad0f 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -11,10 +11,12 @@ New and Improved - ✨ Added TypeScript comment type support for source discovery and analysis. - TypeScript and JavaScript files can now be processed using ``comment_type = "ts"``, - since the TSX grammar used to parse them is a superset of both languages. - Source discovery supports ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, - ``.jsx``, ``.mjs`` and ``.cjs`` extensions by default. + TypeScript and JavaScript files can now be processed using ``comment_type = "ts"``. + The tree-sitter grammar is chosen per file from its extension: ``.ts``, ``.mts``, + and ``.cts`` use the plain TypeScript grammar, and everything else (``.tsx``, + ``.jsx``, ``.js``, ``.mjs``, ``.cjs``) falls back to the TSX grammar. Source + discovery supports ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, + ``.mjs`` and ``.cjs`` extensions by default. .. _`release:1.4.0`: diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index c2c4f7a8..a26a7bfc 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -5,6 +5,7 @@ from typing import Any, TypedDict, cast from tree_sitter import Node as TreeSitterNode +from tree_sitter import Parser, Query from sphinx_codelinks.analyse import utils from sphinx_codelinks.analyse.models import ( @@ -97,9 +98,29 @@ def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]: # type: yield src_path, text.encode("utf-8") def create_src_objects(self) -> None: - parser, query = utils.init_tree_sitter(self.analyse_config.comment_type) + comment_type = self.analyse_config.comment_type + # One (parser, query) pair per distinct grammar actually needed, built + # lazily so a parser is never rebuilt per file. Every comment type + # except TypeScript uses a single grammar for the whole run; + # TypeScript alone varies its grammar per file (utils.ts_grammar_key) + # because a legacy TypeScript-only cast parses as JSX under the wrong + # grammar — see the CommentType.ts branch of utils.init_tree_sitter. + parser_cache: dict[str, tuple[Parser, Query]] = {} for src_path, src_string in self.get_src_strings(): + # `comment_type` is normally a CommentType member, but a few call + # sites carry it as a plain (possibly invalid) str instead — see + # SourceAnalyseConfig.comment_type — so key on `str(comment_type)` + # rather than `.value`, which only the enum has. + cache_key = ( + utils.ts_grammar_key(src_path) + if comment_type == CommentType.ts + else str(comment_type) + ) + if cache_key not in parser_cache: + parser_cache[cache_key] = utils.init_tree_sitter(comment_type, src_path) + parser, query = parser_cache[cache_key] + comments: list[TreeSitterNode] | None = utils.extract_comments( src_string, parser, query ) diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 3bb4591e..6ed904fe 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -101,6 +101,25 @@ "null", } +# TypeScript's own module variants. Legacy angle-bracket type assertions +# (``x``) are valid syntax only here, not under the TSX grammar (see the +# CommentType.ts branch of init_tree_sitter for what goes wrong otherwise), so +# these three suffixes get the plain TypeScript grammar and everything else in +# the JS/TS family falls back to TSX. +TS_STRICT_GRAMMAR_SUFFIXES = {".ts", ".mts", ".cts"} + + +def ts_grammar_key(src_path: Path) -> str: + """Return which tree-sitter-typescript grammar ``src_path`` needs. + + ``"typescript"`` for TypeScript's own module variants (``.ts``, ``.mts``, + ``.cts``); ``"tsx"`` for the rest of the JavaScript/TypeScript family + (``.tsx``, ``.jsx``, ``.js``, ``.mjs``, ``.cjs``). Used both to pick the + grammar in ``init_tree_sitter`` and, by callers that parse many files, to + cache one parser per grammar instead of rebuilding one per file. + """ + return "typescript" if src_path.suffix in TS_STRICT_GRAMMAR_SUFFIXES else "tsx" + def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: """Return True if file is likely text, False if binary.""" @@ -118,7 +137,17 @@ def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: # @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH, FE_TS] -def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: +def init_tree_sitter( + comment_type: CommentType, src_path: Path | None = None +) -> tuple[Parser, Query]: + """Build the (parser, query) pair for ``comment_type``. + + ``src_path`` only matters for ``CommentType.ts``, whose grammar varies by + file suffix (see ``ts_grammar_key``); every other comment type ignores it + and uses a single grammar. When ``src_path`` is omitted the TSX grammar is + assumed, which is the safe default for the whole JS/TS family except + TypeScript's own module variants. + """ if comment_type == CommentType.cpp: import tree_sitter_cpp # noqa: PLC0415 @@ -137,11 +166,19 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: elif comment_type == CommentType.ts: import tree_sitter_typescript # noqa: PLC0415 - # The TSX grammar is a strict superset of the TypeScript grammar, which is - # itself a superset of JavaScript, so it also parses plain .ts and the - # whole JavaScript family fine. Use it for all of them to avoid needing a - # per-file grammar choice. - parsed_language = Language(tree_sitter_typescript.language_tsx()) + # Legacy angle-bracket type assertions (``x``) are valid TypeScript + # syntax in .ts/.mts/.cts, but the same text is JSX syntax under the TSX + # grammar: ``x`` parses as a jsx_opening_element and swallows the + # rest of the file into a single jsx_text node, silently dropping every + # marker after it. So the plain TypeScript grammar is required for those + # three suffixes. The TSX grammar remains the fallback for the rest of + # the JS/TS family (.tsx, .jsx, .js, .mjs, .cjs): JavaScript has no such + # cast syntax, so TSX is safe there, and it additionally handles JSX + # embedded in plain .js. + if src_path is not None and ts_grammar_key(src_path) == "typescript": + parsed_language = Language(tree_sitter_typescript.language_typescript()) + else: + parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: import tree_sitter_yaml # noqa: PLC0415 diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index aa3a2086..ca14515d 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -10,10 +10,12 @@ "python": ["py"], "cs": ["cs"], # ".mts"/".cts" are TypeScript's own ESM/CJS module variants. ".js"/".jsx"/ - # ".mjs"/".cjs" are JavaScript, covered by the same comment type because the - # TSX grammar used to parse "ts" sources is a strict superset of the - # TypeScript grammar, which is itself a superset of JavaScript, so no - # separate grammar or comment_type value is needed. + # ".mjs"/".cjs" are JavaScript. All of these share the "ts" comment type: + # comment syntax is identical across the family, and the analyse stage + # picks the actual tree-sitter grammar per file from the suffix (the plain + # TypeScript grammar for ".ts"/".mts"/".cts", the TSX grammar for + # everything else — see utils.ts_grammar_key), so no separate + # comment_type value is needed here. "ts": ["ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs"], "yaml": ["yml", "yaml"], "rust": ["rs"], diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index 0a031929..1833acb1 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -63,8 +63,11 @@ def init_rust_tree_sitter() -> tuple[Parser, Query]: @pytest.fixture(scope="session") def init_typescript_tree_sitter() -> tuple[Parser, Query]: - # TSX grammar is a superset of the TypeScript grammar (parses plain .ts too), - # matching what utils.init_tree_sitter uses for CommentType.ts. + # The TSX grammar, matching what utils.init_tree_sitter picks for + # CommentType.ts when the file isn't one of TypeScript's own module + # variants (.ts/.mts/.cts) — see utils.ts_grammar_key. Fine for the plain + # TS fixtures below too, since none of them use a legacy angle-bracket + # cast (the one construct where the two grammars disagree). parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, utils.TYPE_SCRIPT_QUERY) parser = Parser(parsed_language) @@ -794,6 +797,38 @@ def test_find_associated_scope_typescript_jsx_no_parse_error( assert not tree.root_node.has_error +def test_typescript_ts_suffix_recovers_markers_around_angle_bracket_cast(): + """A ``.ts`` file must use the plain TypeScript grammar, not TSX. + + ``x`` is a legacy angle-bracket type assertion: valid TypeScript + syntax, but JSX syntax under the TSX grammar. There it parses as a + ``jsx_opening_element`` and swallows the rest of the file into a single + ``jsx_text`` node, silently dropping every marker after it (``has_error`` + is also set). ``init_tree_sitter`` must pick the plain TypeScript grammar + for a ``.ts`` path, via ``ts_grammar_key``, so markers both above and + below the cast all survive. + """ + code = b"""// @Top, IMPL_TOP +const v = x; +// @Bottom, IMPL_BOTTOM +const y = 2; +// @Third, IMPL_THIRD +""" + parser, query = utils.init_tree_sitter(CommentType.ts, Path("dummy.ts")) + tree = parser.parse(code) + assert not tree.root_node.has_error + + comments = utils.extract_comments(code, parser, query) + assert comments is not None + comments.sort(key=lambda node: node.start_point.row) + texts = [node.text.decode("utf-8") for node in comments if node.text] + assert texts == [ + "// @Top, IMPL_TOP", + "// @Bottom, IMPL_BOTTOM", + "// @Third, IMPL_THIRD", + ] + + @pytest.mark.parametrize( ("code", "result"), [ diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 881d7532..1e4a8066 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -37,11 +37,14 @@ "jsonc": (CommentType.jsonc, "jsonc"), "bash": (CommentType.bash, "sh"), "typescript": (CommentType.ts, "ts"), - # `.tsx` is documentary: extraction never reads the file suffix, and the - # plain-TS and TSX grammars lex comments identically (the TSX-grammar - # choice is pinned by test_analyse_utils.py's has_error check instead). - # What this case pins: a marker on its own line inside a multi-line JSX - # block comment anchors to that line. + # `.tsx` matters here: extraction now picks the tree-sitter grammar per + # file from the suffix (utils.ts_grammar_key), and `.tsx` is one of the + # suffixes that gets the TSX grammar rather than the plain TypeScript one + # (the `.ts`/`.mts`/`.cts` suffixes get the latter — see + # utils.init_tree_sitter). What this case pins: a marker on its own line + # inside a multi-line JSX block comment anchors to that line, which + # requires the source (an arrow function returning JSX) to parse cleanly + # under the TSX grammar in the first place. "tsx": (CommentType.ts, "tsx"), } From 0cc642df9150b971538a8ec3c2fd7a1b5f88874b Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 18 Aug 2026 12:05:05 +0200 Subject: [PATCH 12/20] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Capture=20legacy=20?= =?UTF-8?q?html=5Fcomment=20nodes=20in=20the=20ts=20extraction=20query?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TypeScript and TSX grammars emit a separate html_comment node kind for legacy comments, which are valid in the .js sources this comment type also covers. The extraction query only matched comment, so markers written that way were silently dropped with no warning. --- src/sphinx_codelinks/analyse/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 6ed904fe..8eb49757 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -75,7 +75,14 @@ CPP_QUERY = """(comment) @comment""" C_SHARP_QUERY = """(comment) @comment""" # @TypeScript comment query for tree-sitter, IMPL_TS_3, impl, [FE_TS] -TYPE_SCRIPT_QUERY = """(comment) @comment""" +# ``html_comment`` is a separate node kind the TypeScript/TSX grammars emit +# for legacy ```` comments, which are valid in the ``.js`` sources +# this comment type also covers. Without matching it, markers written in that +# style are silently dropped. +TYPE_SCRIPT_QUERY = """ + (comment) @comment + (html_comment) @comment +""" YAML_QUERY = """(comment) @comment""" RUST_QUERY = """ (line_comment) @comment From 4ba26b7843713b60ce7c89ca52bdd990cf92464a Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 18 Aug 2026 12:05:20 +0200 Subject: [PATCH 13/20] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Exclude=20generated?= =?UTF-8?q?/vendored=20output=20from=20source=20discovery=20by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With src_dir defaulting to "./" and comment_type "ts" also discovering .js/.jsx/.mjs/.cjs files, checked-in bundler/tsc output (dist/, build/, lib/, ...) was scanned as source alongside the .ts it was generated from, producing duplicate need ids for the same marker. exclude now defaults to a list of common generated-output and dependency directory globs (node_modules, dist, build, lib, out, coverage) via a named DEFAULT_EXCLUDE constant. Setting exclude explicitly in a project's configuration - including to [] - replaces this default outright, so existing projects that rely on scanning everything are unaffected as long as they already set exclude. Adds tests proving the default excludes generated output and that an explicit exclude still replaces it as before. --- .../source_discover/config.py | 30 +++++++++- tests/test_source_discover.py | 56 +++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index ca14515d..97a76682 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -30,6 +30,31 @@ } +# Default ``exclude`` glob patterns applied when a project's configuration does +# not set ``exclude`` explicitly. +# +# ``src_dir`` defaults to ``"./"`` and the ``ts`` comment type claims ``.js``/ +# ``.jsx``/``.mjs``/``.cjs`` in addition to TypeScript's own extensions, so a +# checked-in ``tsc``/bundler output directory (``dist/``, ``build/``, ``lib/``, +# ...) is otherwise scanned as source alongside the ``.ts`` it was generated +# from, producing duplicate need ids for the same marker. These directory +# names are common generated-output or dependency locations across the JS/TS +# ecosystem (and beyond), so excluding them by default avoids that duplication +# for most projects out of the box. +# +# Setting ``exclude`` explicitly in a project's configuration replaces this +# default outright (dataclass fields don't merge) — including setting it to +# ``[]`` to scan everything. +DEFAULT_EXCLUDE = [ + "**/node_modules/**", + "**/dist/**", + "**/build/**", + "**/lib/**", + "**/out/**", + "**/coverage/**", +] + + class CommentType(str, Enum): python = "python" cpp = "cpp" @@ -81,10 +106,11 @@ def field_names(cls) -> set[str]: """The root of the source directory.""" exclude: list[str] = field( - default_factory=list, + default_factory=lambda: list(DEFAULT_EXCLUDE), metadata={"schema": {"type": "array", "items": {"type": "string"}}}, ) - """The glob pattern to exclude files.""" + """The glob pattern to exclude files. Defaults to ``DEFAULT_EXCLUDE``; set + this explicitly (e.g. to ``[]``) to replace that default outright.""" include: list[str] = field( default_factory=list, diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index 751b726a..ca37bb18 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -7,6 +7,7 @@ from sphinx_codelinks.source_discover.config import ( COMMENT_FILETYPE, + DEFAULT_EXCLUDE, SourceDiscoverConfig, SourceDiscoverConfigType, ) @@ -217,6 +218,61 @@ def test_jsonc_discover_gate() -> None: assert "plain.json" not in discovered +def _make_generated_output_tree(tmp_path: Path) -> Path: + """Lay out a source file alongside checked-in generated output. + + Mirrors a ``tsc``/bundler output tree: ``src/app.ts`` is the real source, + while ``lib/app.js``, ``dist/app.js`` and ``node_modules/pkg/index.js`` + stand in for generated or vendored output that carries the same marker. + """ + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.ts").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + (tmp_path / "lib").mkdir() + (tmp_path / "lib" / "app.js").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + (tmp_path / "dist").mkdir() + (tmp_path / "dist" / "app.js").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + (tmp_path / "node_modules" / "pkg").mkdir(parents=True) + (tmp_path / "node_modules" / "pkg" / "index.js").write_text( + "// vendored\n", encoding="utf-8" + ) + return tmp_path + + +def test_default_exclude_skips_generated_output(tmp_path: Path) -> None: + """The default ``exclude`` keeps generated/vendored JS out of discovery.""" + src_dir = _make_generated_output_tree(tmp_path) + config = SourceDiscoverConfig(src_dir=src_dir, comment_type="ts", gitignore=False) + assert config.exclude == DEFAULT_EXCLUDE + + discover = SourceDiscover(config) + discovered = sorted(str(p.relative_to(src_dir)) for p in discover.source_paths) + assert discovered == [str(Path("src") / "app.ts")] + + +def test_explicit_exclude_replaces_default(tmp_path: Path) -> None: + """An explicit ``exclude`` (even ``[]``) fully replaces the default list.""" + src_dir = _make_generated_output_tree(tmp_path) + config = SourceDiscoverConfig( + src_dir=src_dir, comment_type="ts", gitignore=False, exclude=[] + ) + assert config.exclude == [] + + discover = SourceDiscover(config) + discovered = sorted(str(p.relative_to(src_dir)) for p in discover.source_paths) + assert discovered == [ + str(Path("dist") / "app.js"), + str(Path("lib") / "app.js"), + str(Path("node_modules") / "pkg" / "index.js"), + str(Path("src") / "app.ts"), + ] + + def test_follow_links(tmp_path: Path) -> None: """Test that follow_links controls whether symbolic links are followed.""" # Create a real directory with a source file From 22322cf0d109647eae0c7325908e31757e9b48ed Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 18 Aug 2026 12:05:32 +0200 Subject: [PATCH 14/20] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20Add=20html=5Fcomme?= =?UTF-8?q?nt=20declarative=20extraction=20fixture=20for=20.js?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Locks the html_comment fix: a marker in a .js source now extracts to exactly one need. Adds the "js" LANG_MAP entry (CommentType.ts, extension "js") the case needs, since only "typescript" and "tsx" existed before. --- ...ure[oneline-html_comment_oneliner_js].json | 19 +++++++++++++++++++ tests/data/extraction/oneline.yaml | 14 ++++++++++++++ tests/test_extraction_fixtures.py | 5 +++++ 3 files changed, 38 insertions(+) create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-html_comment_oneliner_js].json diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-html_comment_oneliner_js].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-html_comment_oneliner_js].json new file mode 100644 index 00000000..5ab935aa --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-html_comment_oneliner_js].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_HTML", + "title": "Html Title", + "type": "impl", + "links": { + "links": [ + "REQ_HTML" + ] + }, + "metadata": {}, + "line": 1 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/oneline.yaml b/tests/data/extraction/oneline.yaml index 79762245..ad9beba0 100644 --- a/tests/data/extraction/oneline.yaml +++ b/tests/data/extraction/oneline.yaml @@ -82,3 +82,17 @@ jsx_oneliner_tsx: */} ); + +# legacy HTML-style comments (`` sits on +# its own line — sharing the marker's line would swallow it into the last +# field, the same pre-existing engine behavior noted above for jsx_oneliner_tsx +html_comment_oneliner_js: + lang: js + config: default + source: | + + const x = 1; diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 1e4a8066..ba10779a 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -46,6 +46,11 @@ # requires the source (an arrow function returning JSX) to parse cleanly # under the TSX grammar in the first place. "tsx": (CommentType.ts, "tsx"), + # `.js` matters here: it is parsed with the TSX grammar (see + # utils.ts_grammar_key), which also emits legacy ```` + # ``html_comment`` nodes as a separate node kind from ``comment`` — this + # case pins that the query captures both. + "js": (CommentType.ts, "js"), } From f16b61c8748bbd2fb8c19d2c048bb189befd5a2a Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 18 Aug 2026 12:05:46 +0200 Subject: [PATCH 15/20] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20Document=20the=20n?= =?UTF-8?q?ew=20exclude=20default=20and=20two=20JSDoc=20caveats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - exclude: documents the new default generated-output/dependency glob list and that setting it explicitly (including to []) replaces it outright. - oneline_comment_style: recommends a more specific start_sequence (e.g. "@need") for the TS/JS family, since the default "@" collides with JSDoc tags (@param, @returns, @deprecated) whose description contains a comma. - TypeScript feature: notes that a @need-ids: marker sharing a line with a block comment's closing */ (as in a single-line JSDoc comment) has the */ swallowed into the last need id; keep such markers on their own line, or use // comments for reference markers. - Changelog entry under "Under development" for both fixes and the docs. --- docs/source/components/configuration.rst | 10 ++++++--- docs/source/components/features.rst | 6 ++++++ docs/source/development/change_log.rst | 27 ++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 315d4398..90a56eeb 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -175,7 +175,7 @@ Configures how **Sphinx-CodeLinks** discovers and processes source files within [codelinks.projects.my_project.source_discover] src_dir = "./" - exclude = [] + exclude = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"] include = [] gitignore = true follow_links = false @@ -217,7 +217,7 @@ exclude Defines a list of glob patterns for files and directories to exclude from discovery. This is useful for ignoring build artifacts, temporary files, or specific source files that shouldn't be processed. **Type:** ``list[str]`` -**Default:** ``[]`` +**Default:** ``["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"]`` .. code-block:: toml @@ -236,6 +236,8 @@ Defines a list of glob patterns for files and directories to exclude from discov - ``"**/__pycache__/**"`` - Exclude Python cache directories - ``"node_modules/**"`` - Exclude Node.js dependencies +.. note:: When ``exclude`` is not set, it defaults to a list of common generated-output and dependency directory globs (``node_modules``, ``dist``, ``build``, ``lib``, ``out``, ``coverage``). This matters most for the ``ts`` :ref:`comment_type `, which also discovers ``.js``/``.jsx``/``.mjs``/``.cjs`` files: without this default, checked-in bundler/``tsc`` output would be scanned as source alongside the ``.ts`` it was generated from, producing duplicate need ids for the same marker. Setting ``exclude`` explicitly — including to ``[]`` — replaces this default outright rather than adding to it. + include ^^^^^^^ @@ -403,7 +405,7 @@ Configures how **Sphinx-CodeLinks** analyse source files to extract markers from [codelinks.projects.my_project.source_discover] src_dir = "./" - exclude = [] + exclude = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"] include = [] gitignore = true follow_links = false @@ -545,6 +547,8 @@ Is equivalent to this RST directive: .. important:: The ``type`` and ``title`` fields must be configured in ``needs_fields`` as they are mandatory for **Sphinx-Needs**. +.. note:: For the TS/JS family (``comment_type = "ts"``), the default ``start_sequence = "@"`` collides with JSDoc tags such as ``@param``, ``@returns``, and ``@deprecated``: a tag description containing a comma is misparsed as a bogus one-line need. Set a more specific ``start_sequence`` (e.g. ``"@need"``) to avoid this. + analyse.need_id_refs ^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index ccfd31f1..6ffe6078 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -292,6 +292,12 @@ Features * File extensions ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, ``.mjs`` and ``.cjs`` auto-discovered when ``comment_type = "ts"`` + A ``@need-ids:`` reference marker that shares a line with a block comment's + closing ``*/`` — as in a single-line JSDoc comment such as + ``/** @need-ids: ID */`` — has the ``*/`` swallowed into the last need id. + Put the marker on its own line inside the block, or use a ``//`` comment + for reference markers, to avoid this. + .. fault:: Traceability objects are not detected in TypeScript language :id: FAULT_TS_1 diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index b63bad0f..78a13ec4 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -18,6 +18,33 @@ New and Improved discovery supports ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, ``.mjs`` and ``.cjs`` extensions by default. +Fixes +..... + +- 🐛 Recognized legacy HTML-style comments (````) under ``comment_type = "ts"``. + + The TypeScript and TSX grammars emit these as a separate ``html_comment`` node from + ``comment``. Markers written this way in a ``.js`` source were silently dropped, with + no warning; the extraction query now matches both node kinds. + +- 🐛 Excluded common generated-output and dependency directories from source discovery by default. + + With ``src_dir`` defaulting to ``"./"`` and ``comment_type = "ts"`` also discovering + ``.js``/``.jsx``/``.mjs``/``.cjs`` files, checked-in bundler/``tsc`` output was scanned as + source alongside the ``.ts`` it was generated from, producing duplicate need ids for the + same marker. ``exclude`` now defaults to ``["**/node_modules/**", "**/dist/**", + "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"]`` when not set explicitly; an + explicit ``exclude`` (including ``[]``) replaces this default outright. + +- 📚 Documented JSDoc caveats for the ``ts`` comment type. + + The default one-line ``start_sequence = "@"`` collides with JSDoc tags (``@param``, + ``@returns``, ``@deprecated``) whose description contains a comma; a more specific + sequence such as ``"@need"`` avoids this. Separately, a ``@need-ids:`` marker sharing a + line with a block comment's closing ``*/`` (as in a single-line JSDoc comment) has the + ``*/`` swallowed into the last need id — keep such markers on their own line, or use + ``//`` comments for reference markers. + .. _`release:1.4.0`: 1.4.0 From 45d6d1eec0cd9c9f9fbd5fe01fa43670d9ef7f9f Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 18 Aug 2026 18:03:33 +0200 Subject: [PATCH 16/20] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Scope=20the=20ts=20?= =?UTF-8?q?exclude=20default=20to=20comment=5Ftype,=20not=20every=20langua?= =?UTF-8?q?ge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 4ba26b7 made a module-level DEFAULT_EXCLUDE the default for every SourceDiscoverConfig.exclude regardless of comment_type, so cpp/python/go/ yaml/rust projects silently stopped scanning under node_modules/, dist/, build/, lib/, out/ and coverage/ too. **/lib/** was the worst case: C/C++ projects very commonly keep hand-written library source in lib/, so a marker there would silently go uncovered. Separately, the CLI's `discover` command declared `exclude: ... = []` and always passed it through, bypassing the default entirely - so `discover`/`analyse` and the Sphinx extension applied different excludes for the same project. `exclude` is now `list[str] | None = None` on SourceDiscoverConfig; a `None` value (the user didn't set it) is resolved in `__post_init__` via `default_exclude_for_comment_type`, per this project's own `comment_type`: - `comment_type == "ts"` (the TypeScript/JavaScript family) -> `TS_DEFAULT_EXCLUDE`: node_modules/, dist/, build/, out/, coverage/. `**/lib/**` is deliberately not in this list - it is ambiguous even within the JS/TS ecosystem, since many packages use lib/ for hand-written source rather than as a tsc outDir. Projects whose outDir is lib should add "**/lib/**" to their own exclude. - every other comment_type -> `[]`, byte-for-byte the behavior before 4ba26b7. An explicit `exclude` (including `[]`) still replaces the default outright, never merges with it - the `None` sentinel is what lets "unset" be told apart from "set to []". cmd.py's `discover` command now defaults `-e/--excludes` to `None` too and only forwards it when the user actually passed it, so the CLI resolves the same per-project default as the Sphinx extension / `analyse` path. Adds: a cpp-with-lib/ regression test for the D1 defect, a CLI-level test for the D2 defect, and updates the existing default/explicit-exclude tests for the narrowed five-item ts default. Updates the changelog entry and the configuration/exclude docs to describe the comment_type-derived default. --- docs/source/components/configuration.rst | 21 ++++-- docs/source/development/change_log.rst | 16 +++-- src/sphinx_codelinks/cmd.py | 19 +++-- .../source_discover/config.py | 72 ++++++++++++++----- .../source_discover/source_discover.py | 17 +++-- tests/test_cmd.py | 44 ++++++++++++ tests/test_source_discover.py | 34 +++++++-- 7 files changed, 184 insertions(+), 39 deletions(-) diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 90a56eeb..75ef26cd 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -175,7 +175,8 @@ Configures how **Sphinx-CodeLinks** discovers and processes source files within [codelinks.projects.my_project.source_discover] src_dir = "./" - exclude = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"] + # exclude is omitted here to keep its comment_type-derived default; see + # the `exclude` field below. include = [] gitignore = true follow_links = false @@ -217,7 +218,7 @@ exclude Defines a list of glob patterns for files and directories to exclude from discovery. This is useful for ignoring build artifacts, temporary files, or specific source files that shouldn't be processed. **Type:** ``list[str]`` -**Default:** ``["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"]`` +**Default:** Derived from ``comment_type`` — see the note below. ``[]`` for every ``comment_type`` except ``ts``, where it is ``["**/node_modules/**", "**/dist/**", "**/build/**", "**/out/**", "**/coverage/**"]``. .. code-block:: toml @@ -236,7 +237,18 @@ Defines a list of glob patterns for files and directories to exclude from discov - ``"**/__pycache__/**"`` - Exclude Python cache directories - ``"node_modules/**"`` - Exclude Node.js dependencies -.. note:: When ``exclude`` is not set, it defaults to a list of common generated-output and dependency directory globs (``node_modules``, ``dist``, ``build``, ``lib``, ``out``, ``coverage``). This matters most for the ``ts`` :ref:`comment_type `, which also discovers ``.js``/``.jsx``/``.mjs``/``.cjs`` files: without this default, checked-in bundler/``tsc`` output would be scanned as source alongside the ``.ts`` it was generated from, producing duplicate need ids for the same marker. Setting ``exclude`` explicitly — including to ``[]`` — replaces this default outright rather than adding to it. +.. note:: + + When ``exclude`` is not set, its default is derived from this project's own :ref:`comment_type `, not applied globally: + + - ``comment_type = "ts"`` (the TypeScript/JavaScript family, which also discovers ``.js``/``.jsx``/``.mjs``/``.cjs`` files) defaults ``exclude`` to ``["**/node_modules/**", "**/dist/**", "**/build/**", "**/out/**", "**/coverage/**"]``. Without this, checked-in bundler/``tsc`` output would be scanned as source alongside the ``.ts`` it was generated from, producing duplicate need ids for the same marker. + - Every other ``comment_type`` (``cpp``, ``python``, ``rust``, ``go``, ``yaml``, ``jsonc``, ``bash``, ``cs``, ...) defaults ``exclude`` to ``[]`` — no default exclusion at all. + + ``**/lib/**`` is deliberately **not** in the ``ts`` default: it is ambiguous even within the JS/TS ecosystem, since many packages use ``lib/`` for hand-written source rather than as a ``tsc`` ``outDir`` — and it is common hand-written C/C++ library source outside that ecosystem entirely. If your ``ts`` project's ``outDir`` is ``lib``, add ``"**/lib/**"`` to your own ``exclude`` explicitly. + + Setting ``exclude`` explicitly — including to ``[]`` — replaces the derived default outright rather than adding to it, and does so regardless of ``comment_type``. + + This is resolved identically whether the project is loaded through the Sphinx extension or through the ``discover``/``analyse`` CLI commands: passing ``-e``/``--excludes`` to ``discover`` behaves the same way — omit it to get the ``comment_type``-derived default, or pass it (one or more times) to replace that default outright. include ^^^^^^^ @@ -405,7 +417,8 @@ Configures how **Sphinx-CodeLinks** analyse source files to extract markers from [codelinks.projects.my_project.source_discover] src_dir = "./" - exclude = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"] + # exclude is omitted here to keep its comment_type-derived default; see + # the `exclude` field below. include = [] gitignore = true follow_links = false diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 78a13ec4..a87fe23a 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -27,14 +27,22 @@ Fixes ``comment``. Markers written this way in a ``.js`` source were silently dropped, with no warning; the extraction query now matches both node kinds. -- 🐛 Excluded common generated-output and dependency directories from source discovery by default. +- 🐛 Excluded common generated-output and dependency directories from ``ts`` source discovery by default. With ``src_dir`` defaulting to ``"./"`` and ``comment_type = "ts"`` also discovering ``.js``/``.jsx``/``.mjs``/``.cjs`` files, checked-in bundler/``tsc`` output was scanned as source alongside the ``.ts`` it was generated from, producing duplicate need ids for the - same marker. ``exclude`` now defaults to ``["**/node_modules/**", "**/dist/**", - "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"]`` when not set explicitly; an - explicit ``exclude`` (including ``[]``) replaces this default outright. + same marker. For ``comment_type = "ts"`` projects, ``exclude`` now defaults to + ``["**/node_modules/**", "**/dist/**", "**/build/**", "**/out/**", "**/coverage/**"]`` when + not set explicitly; an explicit ``exclude`` (including ``[]``) replaces this default + outright. ``**/lib/**`` is deliberately not in this list — it is ambiguous even within the + JS/TS ecosystem, where many packages use ``lib/`` for hand-written source rather than as a + ``tsc`` ``outDir``; projects whose ``outDir`` is ``lib`` should add ``"**/lib/**"`` to their + own ``exclude``. Every other ``comment_type`` still defaults ``exclude`` to ``[]`` — this + default never applies outside the ``ts`` family, so e.g. a ``cpp`` project's hand-written + ``lib/`` source is unaffected. The CLI's ``discover``/``analyse`` commands now resolve this + same per-project default too; they previously always scanned everything regardless of + ``comment_type``, disagreeing with the Sphinx extension. - 📚 Documented JSDoc caveats for the ``ts`` comment type. diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index a7b94b95..0bd86c20 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -195,13 +195,19 @@ def discover( # noqa: PLR0913 # CLI command requires multiple parameters ), ], exclude: Annotated[ - list[str], + list[str] | None, typer.Option( "--excludes", "-e", - help="Glob patterns to be excluded.", + help=( + "Glob patterns to be excluded. When omitted, defaults to " + "the comment-type-derived default (non-empty only for " + "--comment-type ts); passing this option one or more times " + "replaces that default outright." + ), + show_default=False, ), - ] = [], # noqa: B006 # to show the default value on CLI + ] = None, include: Annotated[ list[str], typer.Option( @@ -234,12 +240,17 @@ def discover( # noqa: PLR0913 # CLI command requires multiple parameters src_discover_dict: SourceDiscoverConfigType = { "src_dir": src_dir, - "exclude": exclude, "include": include, "gitignore": gitignore, "follow_links": follow_links, "comment_type": comment_type, } + # Only pass "exclude" through when the user actually gave -e/--excludes; + # otherwise leave it out so SourceDiscoverConfig resolves its own + # comment_type-derived default, same as the Sphinx extension / `analyse` + # path does for a project's TOML config that omits `exclude`. + if exclude is not None: + src_discover_dict["exclude"] = exclude src_discover_config = SourceDiscoverConfig(**src_discover_dict) diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index 97a76682..e2c860cb 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -30,26 +30,39 @@ } -# Default ``exclude`` glob patterns applied when a project's configuration does -# not set ``exclude`` explicitly. +# Default ``exclude`` glob patterns applied to ``ts`` (TypeScript/JavaScript +# family) projects when their configuration does not set ``exclude`` +# explicitly. # # ``src_dir`` defaults to ``"./"`` and the ``ts`` comment type claims ``.js``/ # ``.jsx``/``.mjs``/``.cjs`` in addition to TypeScript's own extensions, so a -# checked-in ``tsc``/bundler output directory (``dist/``, ``build/``, ``lib/``, -# ...) is otherwise scanned as source alongside the ``.ts`` it was generated -# from, producing duplicate need ids for the same marker. These directory -# names are common generated-output or dependency locations across the JS/TS -# ecosystem (and beyond), so excluding them by default avoids that duplication -# for most projects out of the box. +# checked-in ``tsc``/bundler output directory (``dist/``, ``build/``, ...) is +# otherwise scanned as source alongside the ``.ts`` it was generated from, +# producing duplicate need ids for the same marker. These directory names are +# common generated-output or dependency locations across the JS/TS ecosystem, +# so excluding them by default avoids that duplication for most projects out +# of the box. +# +# ``**/lib/**`` is deliberately NOT in this list: it is ambiguous even within +# the JS/TS ecosystem, where many packages use ``lib/`` for hand-written +# source rather than as a ``tsc`` ``outDir``. Projects whose ``outDir`` is +# ``lib`` should add ``"**/lib/**"`` to their own ``exclude`` explicitly. +# +# This default is applied only for ``comment_type == "ts"``. Every other +# ``comment_type`` (``cpp``, ``python``, ``rust``, ``go``, ``yaml``, ``jsonc``, +# ``bash``, ``cs``, ...) defaults ``exclude`` to ``[]`` — unchanged from +# before this default existed. ``cpp`` projects in particular very commonly +# keep hand-written library source under ``lib/``, so a directory-name-based +# default that isn't scoped to the language family would silently drop +# markers there. # # Setting ``exclude`` explicitly in a project's configuration replaces this -# default outright (dataclass fields don't merge) — including setting it to -# ``[]`` to scan everything. -DEFAULT_EXCLUDE = [ +# default outright (it is not merged) — including setting it to ``[]`` to +# scan everything. +TS_DEFAULT_EXCLUDE = [ "**/node_modules/**", "**/dist/**", "**/build/**", - "**/lib/**", "**/out/**", "**/coverage/**", ] @@ -72,6 +85,19 @@ class CommentType(str, Enum): bash = "bash" +def default_exclude_for_comment_type(comment_type: str) -> list[str]: + """Resolve the default ``exclude`` patterns for a given ``comment_type``. + + Only the ``ts`` (TypeScript/JavaScript) family gets a non-empty default — + see ``TS_DEFAULT_EXCLUDE`` for why. Every other ``comment_type`` defaults + to ``[]``, i.e. no default exclusion at all, matching the behavior before + a default was ever introduced. + """ + if comment_type == CommentType.ts: + return list(TS_DEFAULT_EXCLUDE) + return [] + + class SourceDiscoverSectionConfigType(TypedDict, total=False): """Define typing for loading configuration from TOML files""" @@ -105,12 +131,18 @@ def field_names(cls) -> set[str]: ) """The root of the source directory.""" - exclude: list[str] = field( - default_factory=lambda: list(DEFAULT_EXCLUDE), + exclude: list[str] | None = field( + default=None, metadata={"schema": {"type": "array", "items": {"type": "string"}}}, ) - """The glob pattern to exclude files. Defaults to ``DEFAULT_EXCLUDE``; set - this explicitly (e.g. to ``[]``) to replace that default outright.""" + """The glob pattern to exclude files. + + Leave unset (``None``) to get the ``comment_type``-derived default + resolved in ``__post_init__`` (see ``default_exclude_for_comment_type``): + ``TS_DEFAULT_EXCLUDE`` for ``comment_type == "ts"``, ``[]`` for every + other ``comment_type``. Set this explicitly — including to ``[]`` — to + replace that default outright; it is never merged with it. + """ include: list[str] = field( default_factory=list, @@ -135,6 +167,14 @@ def field_names(cls) -> set[str]: ) """The file types to discover.""" + def __post_init__(self) -> None: + # ``None`` means "the user didn't set exclude" (a dataclass field + # default can't otherwise be told apart from an explicit ``[]``) — + # resolve it to the comment_type-derived default. An explicit value, + # including ``[]``, is left untouched. + if self.exclude is None: + self.exclude = default_exclude_for_comment_type(self.comment_type) + @classmethod def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] _field = next(_field for _field in fields(cls) if _field.name is name) diff --git a/src/sphinx_codelinks/source_discover/source_discover.py b/src/sphinx_codelinks/source_discover/source_discover.py index f229464e..c4e72ea5 100644 --- a/src/sphinx_codelinks/source_discover/source_discover.py +++ b/src/sphinx_codelinks/source_discover/source_discover.py @@ -45,20 +45,23 @@ def _build_overrides(self) -> OverrideBuilder | None: Include patterns are added as whitelist globs. Exclude patterns are added as negated globs (prefixed with ``!``). """ - has_include = bool(self.src_discover_config.include) - has_exclude = bool(self.src_discover_config.exclude) + include = self.src_discover_config.include + # ``SourceDiscoverConfig.__post_init__`` always resolves ``exclude`` + # to a concrete list (never leaves it ``None``); the ``| None`` on + # the field itself only exists to detect "not explicitly set". + exclude = self.src_discover_config.exclude - if not has_include and not has_exclude: + if not include and not exclude: return None ob = OverrideBuilder(self.src_discover_config.src_dir) - if has_include: - for pattern in self.src_discover_config.include: + if include: + for pattern in include: ob.add(pattern) - if has_exclude: - for pattern in self.src_discover_config.exclude: + if exclude: + for pattern in exclude: ob.add(f"!{pattern}") return ob diff --git a/tests/test_cmd.py b/tests/test_cmd.py index d6533e4b..c990284b 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -160,6 +160,50 @@ def test_discover(options, stdout): assert stdout in result.stdout +def test_discover_cli_default_exclude_matches_ts_default(tmp_path: Path) -> None: + """Regression guard for D2: ``discover`` used to hardcode ``exclude=[]`` + and bypass the comment_type-derived default entirely, so the CLI and the + Sphinx extension / ``analyse`` path (which both go through + ``SourceDiscoverConfig``'s own resolution) would silently disagree on + what's excluded for the same project. With no ``-e/--excludes``, + ``discover`` must resolve the same ``ts`` default as + ``SourceDiscoverConfig`` itself.""" + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.ts").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + (tmp_path / "dist").mkdir() + (tmp_path / "dist" / "app.js").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + + result = runner.invoke( + app, + ["discover", str(tmp_path), "--comment-type", "ts", "--no-gitignore"], + ) + assert result.exit_code == 0 + assert "1 files discovered" in result.stdout + assert str(tmp_path / "src" / "app.ts") in result.stdout + assert str(tmp_path / "dist" / "app.js") not in result.stdout + + +def test_discover_cli_cpp_default_exclude_is_empty(tmp_path: Path) -> None: + """Same D2 guard as above, from the other side: a non-``ts`` + ``comment_type`` (``cpp``, the CLI default) must resolve to an empty + default exclude via the CLI too, so hand-written ``lib/`` source is + still discovered — matching ``SourceDiscoverConfig``'s own resolution.""" + lib_dir = tmp_path / "lib" + lib_dir.mkdir() + (lib_dir / "widget.cpp").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + + result = runner.invoke(app, ["discover", str(tmp_path), "--no-gitignore"]) + assert result.exit_code == 0 + assert "1 files discovered" in result.stdout + assert str(lib_dir / "widget.cpp") in result.stdout + + @pytest.mark.parametrize( ("src_discover_dict", "analyse_dict", "output_lines"), [ diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index ca37bb18..95425f0a 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -7,7 +7,7 @@ from sphinx_codelinks.source_discover.config import ( COMMENT_FILETYPE, - DEFAULT_EXCLUDE, + TS_DEFAULT_EXCLUDE, SourceDiscoverConfig, SourceDiscoverConfigType, ) @@ -245,14 +245,40 @@ def _make_generated_output_tree(tmp_path: Path) -> Path: def test_default_exclude_skips_generated_output(tmp_path: Path) -> None: - """The default ``exclude`` keeps generated/vendored JS out of discovery.""" + """The ``ts``-derived default ``exclude`` keeps generated/vendored JS out + of discovery, while ``lib/`` — deliberately not in ``TS_DEFAULT_EXCLUDE`` + — is still discovered (see the constant's docstring for why).""" src_dir = _make_generated_output_tree(tmp_path) config = SourceDiscoverConfig(src_dir=src_dir, comment_type="ts", gitignore=False) - assert config.exclude == DEFAULT_EXCLUDE + assert config.exclude == TS_DEFAULT_EXCLUDE discover = SourceDiscover(config) discovered = sorted(str(p.relative_to(src_dir)) for p in discover.source_paths) - assert discovered == [str(Path("src") / "app.ts")] + assert discovered == [ + str(Path("lib") / "app.js"), + str(Path("src") / "app.ts"), + ] + + +def test_cpp_project_default_exclude_is_empty_and_finds_lib_marker( + tmp_path: Path, +) -> None: + """Regression guard for the D1 defect: the ``ts``-family default exclude + must not leak to other ``comment_type`` values. ``cpp`` projects very + commonly keep hand-written library source under ``lib/`` — unlike a + ``tsc``/bundler ``lib/`` output dir, it must still be discovered.""" + lib_dir = tmp_path / "lib" + lib_dir.mkdir() + (lib_dir / "widget.cpp").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + + config = SourceDiscoverConfig(src_dir=tmp_path, comment_type="cpp", gitignore=False) + assert config.exclude == [] + + discover = SourceDiscover(config) + discovered = sorted(str(p.relative_to(tmp_path)) for p in discover.source_paths) + assert discovered == [str(Path("lib") / "widget.cpp")] def test_explicit_exclude_replaces_default(tmp_path: Path) -> None: From 90ca4599628095fccdacbca83b5e841fd3e0ec70 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Wed, 19 Aug 2026 19:54:44 +0200 Subject: [PATCH 17/20] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20correct=20marker=20?= =?UTF-8?q?field=20order=20in=20TypeScript=20demo=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit demo.ts:3 and demo.tsx:1 had one-line markers with fields in wrong order. The parser expects (title, id, type, links) but these had (type, id, title). This caused the integration test to pass while asserting wrong values. Corrected both markers to parse correctly with sensible titles and type=impl. --- tests/data/typescript/demo.ts | 2 +- tests/data/typescript/demo.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/typescript/demo.ts b/tests/data/typescript/demo.ts index 66aade0f..82b41b39 100644 --- a/tests/data/typescript/demo.ts +++ b/tests/data/typescript/demo.ts @@ -1,6 +1,6 @@ // regular comment function testA() { - // @type,TS_REQ_002,TypeScript one-line test + // @TypeScript one-line test,TS_REQ_002,impl return 1; } diff --git a/tests/data/typescript/demo.tsx b/tests/data/typescript/demo.tsx index a7e78969..20a60635 100644 --- a/tests/data/typescript/demo.tsx +++ b/tests/data/typescript/demo.tsx @@ -1,4 +1,4 @@ -// @type,TS_REQ_003,TypeScript JSX component test +// @TypeScript JSX component test,TS_REQ_003,impl export function Button() { return ; } From dd7554c8c0836a61b56773ac959cede0b5b6feb7 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Wed, 19 Aug 2026 19:54:50 +0200 Subject: [PATCH 18/20] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20replace=20non-exis?= =?UTF-8?q?tent=20LANGUAGE=5FANALYZERS=20with=20real=20architecture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md:61 and AGENTS.md:413 described a LANGUAGE_ANALYZERS dict that does not exist in the codebase. This sent contributors down an impossible path. Documented the real architecture: init_tree_sitter() if/elif chain per CommentType, paired with tree-sitter grammars and *_QUERY constants. Added real contribution steps: updating COMMENT_FILETYPE, adding to init_tree_sitter(), populating SCOPE_NODE_TYPES, defining *_QUERY, and adding tests. Also fixed supported-language list in CLAUDE.md to include Bash (was missing). --- AGENTS.md | 21 +++++++++++---------- CLAUDE.md | 14 ++++++++++---- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ef2607c3..cc744fb9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -399,20 +399,21 @@ The CLI uses Typer for command definitions: ### Adding Support for a New Language 1. Add tree-sitter parser dependency to `pyproject.toml` (e.g., `tree-sitter-java`) -2. Create language-specific analyzer in `analyse/projects.py`: +2. Add comment type to `CommentType` enum in `source_discover/config.py` +3. Add file extensions to `COMMENT_FILETYPE` dict in `source_discover/config.py` (e.g., `"java": ["java"]`) +4. Add a case to `init_tree_sitter()` in `analyse/utils.py` wiring the tree-sitter grammar: ```python - class JavaAnalyzer(BaseAnalyzer): - language = "java" - parser_language = "java" - - def get_comment_nodes(self, tree): - # Return comment nodes from tree + elif comment_type == CommentType.java: + import tree_sitter_java # noqa: PLC0415 + parsed_language = Language(tree_sitter_java.language()) + query = Query(parsed_language, JAVA_QUERY) ``` -3. Register analyzer in `LANGUAGE_ANALYZERS` dict in `projects.py` -4. Add test files in `tests/data//` -5. Add tests in `tests/test_analyse.py` +5. Define a `JAVA_QUERY` constant in `analyse/utils.py` extracting comments for the language +6. Add scope types to `SCOPE_NODE_TYPES` dict in `analyse/utils.py` (e.g., `CommentType.java: {"method_declaration", "class_declaration"}`) +7. Add test files in `tests/data//` +8. Add tests in `tests/test_analyse.py` ### Adding a New Marker Type diff --git a/CLAUDE.md b/CLAUDE.md index 4d3014e1..a4169834 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ covers what's needed to get moving quickly. ## What this project is sphinx-codelinks is a Sphinx extension providing fast source-code traceability for -Sphinx-Needs: it scans source files (C++, Python, C#, Rust, TypeScript, Go, YAML, JSON) for +Sphinx-Needs: it scans source files (C++, Python, C#, Rust, TypeScript, JavaScript, Go, Bash, YAML, JSON) for marker comments via tree-sitter, and generates Sphinx-Needs items / RST that link documentation back to exact source locations. @@ -57,8 +57,12 @@ The CLI itself is installed as `codelinks` (`codelinks analyse `, Pipeline: **Source Files → Discovery → Parsing → Analysis → Results (JSON) → RST Generation** - `source_discover/` — finds source files by include/exclude patterns, respects `.gitignore`. + `COMMENT_FILETYPE` dict in `source_discover/config.py` maps comment types to file extensions. +- `analyse/utils.py` — per-language setup via `init_tree_sitter(comment_type)`, an if/elif chain + that pairs each comment type with a tree-sitter grammar (`tree_sitter_cpp`, `tree_sitter_python`, + etc.) and a per-language `*_QUERY` constant. Scope detection uses the `SCOPE_NODE_TYPES` dict + to map comment types to their scope node types (functions, classes, etc.). - `analyse/oneline_parser.py` — tree-sitter based parser extracting comment marker nodes. -- `analyse/projects.py` — per-language analyzers, registered in a `LANGUAGE_ANALYZERS` dict. - `analyse/analyse.py` — orchestrates discovery + parsing + analysis into `analyse/models.py` Pydantic result models. - `needextend_write.py` — turns analysis JSON into RST with Sphinx-Needs `needextend` @@ -70,9 +74,11 @@ Pipeline: **Source Files → Discovery → Parsing → Analysis → Results (JSO options/types, generate standalone traced-source HTML pages, and inject CSS (`sphinx_extension/ub_sct.css`). See AGENTS.md for the full event table and mermaid diagram. -Adding a new language analyzer, marker type, CLI command, or config option each follow a +Adding a new language, marker type, CLI command, or config option each follow a short recipe documented in AGENTS.md under "Common Patterns" — follow those rather than -inventing a new approach. +inventing a new approach. To add a language: update `COMMENT_FILETYPE`, add a case to +`init_tree_sitter()` wiring the tree-sitter grammar, add scope types to `SCOPE_NODE_TYPES`, +define a `*_QUERY` constant, and add test data. ## Code style From 2da4c49e625c5b15ecb5187196773f41f8146fcf Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Wed, 19 Aug 2026 19:54:55 +0200 Subject: [PATCH 19/20] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20change=20changelog?= =?UTF-8?q?=20heading=20from=20'Under=20development'=20to=20'Unreleased'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align with established convention used in released sections. --- docs/source/development/change_log.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index a87fe23a..c3c295c9 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -3,8 +3,8 @@ Changelog ========= -Under development ------------------ +Unreleased +---------- New and Improved ................ From de4dfdaea479f033cf3c95b8b9c3528e335e16d0 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Wed, 19 Aug 2026 19:55:01 +0200 Subject: [PATCH 20/20] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20document=20.d.ts?= =?UTF-8?q?=20behavior=20and=20qualify=20JSX=20comment=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Document that .d.ts declaration files are discovered but markers in them will not resolve to an enclosing scope (ambient declarations only). 2. Qualify JSX comment marker documentation: the single-line form does not work due to closing */ being swallowed into the last field. Markers must appear on their own line inside the JSX comment block. --- docs/source/components/features.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index 6ffe6078..5909bd6f 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -280,8 +280,10 @@ Features angle-bracket type assertion (``x``) is valid there but is JSX syntax under the TSX grammar. Every other extension (``.tsx``, ``.jsx``, ``.js``, ``.mjs``, ``.cjs``) is parsed with the TSX grammar, which is safe for plain - JavaScript and additionally handles JSX (including JSX comments such as - ``{/* ... */}``) embedded in ``.tsx`` or ``.js`` sources. + JavaScript and additionally handles JSX embedded in ``.tsx`` or ``.js`` sources. + JSX comment markers (``{/* ... */}``) are supported only when the marker + appears on its own line inside the comment block; single-line JSX comments + (``{/* @Title, ID, impl, [REQ] */}``) are not currently supported. Key capabilities: @@ -291,6 +293,8 @@ Features a function or arrow function * File extensions ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, ``.mjs`` and ``.cjs`` auto-discovered when ``comment_type = "ts"`` + * Markers in TypeScript declaration files (``.d.ts``) are discovered but will not + resolve to an enclosing scope, since declaration files contain only type declarations A ``@need-ids:`` reference marker that shares a line with a block comment's closing ``*/`` — as in a single-line JSDoc comment such as