Docs: Cell reference as a custom documenter

Use autodocs to perform cell reference docs generation instead of generating rst files directly.
e.g.
```
.. autocell:: simlib.v:$alu
   :source:
   :linenos:
```

docs/source/conf.py

@@ -100,9 +100,14 @@ latex_elements = {
 sys.path += [os.path.dirname(__file__) + "/../"]
 extensions.append('util.cmdref')
 
-def setup(sphinx):
-	from util.RtlilLexer import RtlilLexer
-	sphinx.add_lexer("RTLIL", RtlilLexer)
+# use autodocs
+extensions.append('sphinx.ext.autodoc')
+extensions.append('util.cellref')
 
-	from util.YoscryptLexer import YoscryptLexer
-	sphinx.add_lexer("yoscrypt", YoscryptLexer)
+from sphinx.application import Sphinx
+def setup(app: Sphinx) -> None:
+    from util.RtlilLexer import RtlilLexer
+    app.add_lexer("RTLIL", RtlilLexer)
+
+    from util.YoscryptLexer import YoscryptLexer
+    app.add_lexer("yoscrypt", YoscryptLexer)

docs/util/cellref.py

@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+from pathlib import Path
+import re
+
+from typing import Any
+from sphinx.application import Sphinx
+from sphinx.ext import autodoc
+from sphinx.ext.autodoc import Documenter
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+# cell signature
+cell_ext_sig_re = re.compile(
+    r'''^ (?:([\w._/]+):)?       # explicit file name
+          ([\w$._]+?)?           # module and/or class name(s)
+          (?:\.([\w_]+))?        # optional: thing name
+          (::[\w_]+)?            #           attribute
+          \s* $                  # and nothing more
+          ''', re.VERBOSE)
+
+class SimHelper:
+    name: str = ""
+    title: str = ""
+    ports: str = ""
+    source: str = ""
+    desc: list[str]
+    code: list[str]
+    group: str = ""
+    ver: str = "1"
+
+    def __init__(self) -> None:
+        self.desc = []
+
+def simcells_reparse(cell: SimHelper):
+    # cut manual signature
+    cell.desc = cell.desc[3:]
+
+    # code-block truth table
+    new_desc = []
+    indent = ""
+    for line in cell.desc:
+        if line.startswith("Truth table:"):
+            indent = "   "
+            new_desc.pop()
+            new_desc.extend(["::", ""])
+        new_desc.append(indent + line)
+    cell.desc = new_desc
+
+    # set version
+    cell.ver = "2a"
+
+def load_cell_lib(file: Path):
+    simHelpers: dict[str, SimHelper] = {}
+    simHelper = SimHelper()
+    with open(file, "r") as f:
+        lines = f.readlines()
+
+    for lineno, line in enumerate(lines, 1):
+        line = line.rstrip()
+        # special comments
+        if line.startswith("//-"):
+            simHelper.desc.append(line[4:] if len(line) > 4 else "")
+        elif line.startswith("//* "):
+            _, key, val = line.split(maxsplit=2)
+            setattr(simHelper, key, val)
+        
+        # code parsing
+        if line.startswith("module "):
+            clean_line = line[7:].replace("\\", "").replace(";", "")
+            simHelper.name, simHelper.ports = clean_line.split(maxsplit=1)
+            simHelper.code = []
+            simHelper.source = f'{file.name}:{lineno}'
+        elif not line.startswith("endmodule"):
+            line = "    " + line
+        try:
+            simHelper.code.append(line.replace("\t", "    "))
+        except AttributeError:
+            # no module definition, ignore line
+            pass
+        if line.startswith("endmodule"):
+            if simHelper.ver == "1" and file.name == "simcells.v":
+                # default simcells parsing
+                simcells_reparse(simHelper)
+            if not simHelper.desc:
+                # no help
+                simHelper.desc.append("No help message for this cell type found.\n")
+            elif simHelper.ver == "1" and file.name == "simlib.v" and simHelper.desc[1].startswith('    '):
+                simHelper.desc.pop(1)
+            simHelpers[simHelper.name] = simHelper
+            simHelper = SimHelper()
+    return simHelpers
+
+class YosysCellDocumenter(Documenter):
+    objtype = 'cell'
+    parsed_libs: dict[Path, dict[str, SimHelper]] = {}
+    object: SimHelper
+
+    option_spec = {
+        'source': autodoc.bool_option,
+        'linenos': autodoc.bool_option,
+    }
+
+    @classmethod
+    def can_document_member(
+        cls,
+        member: Any,
+        membername: str,
+        isattr: bool,
+        parent: Any
+    ) -> bool:
+        sourcename = str(member).split(":")[0]
+        if not sourcename.endswith(".v"):
+            return False
+        if membername == "__source":
+            return False
+
+    def parse_name(self) -> bool:
+        try:
+            matched = cell_ext_sig_re.match(self.name)
+            path, modname, thing, attribute = matched.groups()
+        except AttributeError:
+            logger.warning(('invalid signature for auto%s (%r)') % (self.objtype, self.name),
+                           type='cellref')
+            return False
+        
+        if not path:
+            return False
+
+        self.modname = modname
+        self.objpath = [path]
+        self.attribute = attribute
+        self.fullname = ((self.modname) + (thing or ''))
+
+        return True
+    
+    def import_object(self, raiseerror: bool = False) -> bool:
+        # find cell lib file
+        objpath = Path('/'.join(self.objpath))
+        if not objpath.exists():
+            objpath = Path('source') / 'generated' / objpath
+
+        # load cell lib
+        try:
+            parsed_lib = self.parsed_libs[objpath]
+        except KeyError:
+            parsed_lib = load_cell_lib(objpath)
+            self.parsed_libs[objpath] = parsed_lib
+
+        # get cell
+        try:
+            self.object = parsed_lib[self.modname]
+        except KeyError:
+            return False
+
+        self.real_modname = f'{objpath}:{self.modname}'
+        return True
+    
+    def get_sourcename(self) -> str:
+        return self.objpath
+    
+    def format_name(self) -> str:
+        return self.object.name
+
+    def format_signature(self, **kwargs: Any) -> str:
+        return f"{self.object.name} {self.object.ports}"
+    
+    def add_directive_header(self, sig: str) -> None:
+        domain = getattr(self, 'domain', self.objtype)
+        directive = getattr(self, 'directivetype', 'def')
+        name = self.format_name()
+        sourcename = self.get_sourcename()
+        cell = self.object
+
+        # cell definition
+        self.add_line(f'.. {domain}:{directive}:: {name}', sourcename)
+
+        # options
+        opt_attrs = ["title", ]
+        for attr in opt_attrs:
+            val = getattr(cell, attr, None)
+            if val:
+                self.add_line(f'   :{attr}: {val}', sourcename)
+
+        if self.options.noindex:
+            self.add_line('   :noindex:', sourcename)
+    
+    def add_content(self, more_content: Any | None) -> None:
+        # set sourcename and add content from attribute documentation
+        sourcename = self.get_sourcename()
+        startline = int(self.object.source.split(":")[1])
+
+        for i, line in enumerate(self.object.desc, startline):
+            self.add_line(line, sourcename, i)
+
+        # add additional content (e.g. from document), if present
+        if more_content:
+            for line, src in zip(more_content.data, more_content.items):
+                self.add_line(line, src[0], src[1])
+
+    def filter_members(
+        self,
+        members: list[tuple[str, Any]],
+        want_all: bool
+    ) -> list[tuple[str, Any, bool]]:
+        return [(x[0], x[1], False) for x in members]
+
+    def get_object_members(
+        self,
+        want_all: bool
+    ) -> tuple[bool, list[tuple[str, Any]]]:
+        ret: list[tuple[str, str]] = []
+
+        if self.options.source:
+            ret.append(('__source', self.real_modname))
+
+        return False, ret
+
+    def document_members(self, all_members: bool = False) -> None:
+        want_all = (all_members or
+                    self.options.inherited_members)
+        # find out which members are documentable
+        members_check_module, members = self.get_object_members(want_all)
+
+        # document non-skipped members
+        memberdocumenters: list[tuple[Documenter, bool]] = []
+        for (mname, member, isattr) in self.filter_members(members, want_all):
+            classes = [cls for cls in self.documenters.values()
+                       if cls.can_document_member(member, mname, isattr, self)]
+            if not classes:
+                # don't know how to document this member
+                continue
+            # prefer the documenter with the highest priority
+            classes.sort(key=lambda cls: cls.priority)
+            # give explicitly separated module name, so that members
+            # of inner classes can be documented
+            full_mname = self.real_modname + '::' + mname
+            documenter = classes[-1](self.directive, full_mname, self.indent)
+            memberdocumenters.append((documenter, isattr))
+
+        member_order = self.options.member_order or self.config.autodoc_member_order
+        memberdocumenters = self.sort_members(memberdocumenters, member_order)
+
+        for documenter, isattr in memberdocumenters:
+            documenter.generate(
+                all_members=True, real_modname=self.real_modname,
+                check_module=members_check_module and not isattr)
+
+    def generate(
+        self,
+        more_content: Any | None = None,
+        real_modname: str | None = None,
+        check_module: bool = False,
+        all_members: bool = False
+    ) -> None:
+        if not self.parse_name():
+            # need a cell lib to import from
+            logger.warning(
+                f"don't know which cell lib to import for autodocumenting {self.name}",
+                type = 'cellref'
+            )
+            return
+
+        self.import_object()
+
+        # check __module__ of object (for members not given explicitly)
+        # if check_module:
+        #     if not self.check_module():
+        #         return
+
+        sourcename = self.get_sourcename()
+        self.add_line('', sourcename)
+
+        # format the object's signature, if any
+        try:
+            sig = self.format_signature()
+        except Exception as exc:
+            logger.warning(('error while formatting signature for %s: %s'),
+                           self.fullname, exc, type='cellref')
+            return
+
+        # generate the directive header and options, if applicable
+        self.add_directive_header(sig)
+        self.add_line('', sourcename)
+
+        # e.g. the module directive doesn't have content
+        self.indent += self.content_indent
+
+        # add all content (from docstrings, attribute docs etc.)
+        self.add_content(more_content)
+
+        # document members, if possible
+        self.document_members(all_members)
+
+class YosysCellSourceDocumenter(YosysCellDocumenter):
+    objtype = 'cellsource'
+    priority = 20
+
+    @classmethod
+    def can_document_member(
+        cls,
+        member: Any,
+        membername: str,
+        isattr: bool,
+        parent: Any
+    ) -> bool:
+        sourcename = str(member).split(":")[0]
+        if not sourcename.endswith(".v"):
+            return False
+        if membername != "__source":
+            return False
+        if isinstance(parent, YosysCellDocumenter):
+            return True
+        return False
+    
+    def add_directive_header(self, sig: str) -> None:
+        domain = getattr(self, 'domain', 'cell')
+        directive = getattr(self, 'directivetype', 'source')
+        name = self.format_name()
+        sourcename = self.get_sourcename()
+        cell = self.object
+
+        # cell definition
+        self.add_line(f'.. {domain}:{directive}:: {name}', sourcename)
+
+        if self.options.linenos:
+            self.add_line(f'   :source: {cell.source.split(":")[0]}', sourcename)
+        else:
+            self.add_line(f'   :source: {cell.source}', sourcename)
+        self.add_line(f'   :language: verilog', sourcename)
+
+        if self.options.linenos:
+            startline = int(self.object.source.split(":")[1])
+            self.add_line(f'   :lineno-start: {startline}', sourcename)
+
+        if self.options.noindex:
+            self.add_line('   :noindex:', sourcename)
+    
+    def add_content(self, more_content: Any | None) -> None:
+        # set sourcename and add content from attribute documentation
+        sourcename = self.get_sourcename()
+        startline = int(self.object.source.split(":")[1])
+
+        for i, line in enumerate(self.object.code, startline-1):
+            self.add_line(line, sourcename, i)
+
+        # add additional content (e.g. from document), if present
+        if more_content:
+            for line, src in zip(more_content.data, more_content.items):
+                self.add_line(line, src[0], src[1])
+
+    def get_object_members(
+        self,
+        want_all: bool
+    ) -> tuple[bool, list[tuple[str, Any]]]:
+        return False, []
+
+def setup(app: Sphinx) -> dict[str, Any]:
+    app.setup_extension('sphinx.ext.autodoc')
+    app.add_autodocumenter(YosysCellDocumenter)
+    app.add_autodocumenter(YosysCellSourceDocumenter)
+    return {
+        'version': '1',
+        'parallel_read_safe': True,
+    }

docs/util/cmdref.py

@@ -1,5 +1,9 @@
 # based on https://github.com/ofosos/sphinxrecipes/blob/master/sphinxrecipes/sphinxrecipes.py
 
+from __future__ import annotations
+
+from docutils import nodes
+from docutils.nodes import Node, Element
 from docutils.parsers.rst import directives
 from docutils.parsers.rst.states import Inliner
 from sphinx.application import Sphinx
@@ -7,24 +11,59 @@ from sphinx.domains import Domain, Index
 from sphinx.domains.std import StandardDomain
 from sphinx.roles import XRefRole
 from sphinx.directives import ObjectDescription
+from sphinx.directives.code import container_wrapper
 from sphinx.util.nodes import make_refnode
 from sphinx import addnodes
 
-class CommandNode(ObjectDescription):
+class TocNode(ObjectDescription):
+    def _object_hierarchy_parts(self, sig_node: addnodes.desc_signature) -> tuple[str, ...]:
+        if 'fullname' not in sig_node:
+            return ()
+
+        modname = sig_node.get('module')
+        fullname = sig_node['fullname']
+
+        if modname:
+            return (modname, *fullname.split('::'))
+        else:
+            return tuple(fullname.split('::'))
+
+    def _toc_entry_name(self, sig_node: addnodes.desc_signature) -> str:
+        if not sig_node.get('_toc_parts'):
+            return ''
+
+        config = self.env.app.config
+        objtype = sig_node.parent.get('objtype')
+        if config.add_function_parentheses and objtype in {'function', 'method'}:
+            parens = '()'
+        else:
+            parens = ''
+        *parents, name = sig_node['_toc_parts']
+        if config.toc_object_entries_show_parents == 'domain':
+            return sig_node.get('fullname', name) + parens
+        if config.toc_object_entries_show_parents == 'hide':
+            return name + parens
+        if config.toc_object_entries_show_parents == 'all':
+            return '.'.join(parents + [name + parens])
+        return ''
+
+class CommandNode(TocNode):
     """A custom node that describes a command."""
 
     name = 'cmd'
     required_arguments = 1
 
     option_spec = {
-        'title': directives.unchanged_required,
+        'title': directives.unchanged,
         'tags': directives.unchanged
     }
 
     def handle_signature(self, sig, signode: addnodes.desc_signature):
+        fullname = sig
+        signode['fullname'] = fullname
         signode += addnodes.desc_addname(text="yosys> help ")
         signode += addnodes.desc_name(text=sig)
-        return sig
+        return fullname
 
     def add_target_and_index(self, name_cls, sig, signode):
         signode['ids'].append(type(self).name + '-' + sig)
@@ -32,7 +71,7 @@ class CommandNode(ObjectDescription):
             name = "{}.{}.{}".format(self.name, type(self).__name__, sig)
             tagmap = self.env.domaindata[type(self).name]['obj2tag']
             tagmap[name] = list(self.options.get('tags', '').split(' '))
-            title = self.options.get('title')
+            title = self.options.get('title', sig)
             titlemap = self.env.domaindata[type(self).name]['obj2title']
             titlemap[name] = title
             objs = self.env.domaindata[type(self).name]['objects']
@@ -48,6 +87,111 @@ class CellNode(CommandNode):
 
     name = 'cell'
 
+class CellSourceNode(TocNode):
+    """A custom code block for including cell source."""
+
+    name = 'cellsource'
+
+    option_spec = {
+        "source": directives.unchanged_required,
+        "language": directives.unchanged_required,
+        'lineno-start': int,
+    }
+
+    def handle_signature(
+        self,
+        sig,
+        signode: addnodes.desc_signature
+    ) -> str:
+        language = self.options.get('language')
+        fullname = sig + "::" + language
+        signode['fullname'] = fullname
+        signode += addnodes.desc_name(text="Simulation model")
+        signode += addnodes.desc_sig_space()
+        signode += addnodes.desc_addname(text=f'({language})')
+        return fullname
+    
+    def add_target_and_index(
+        self,
+        name: str,
+        sig: str,
+        signode: addnodes.desc_signature
+    ) -> None:
+        idx = f'{".".join(self.name.split(":"))}.{sig}'
+        signode['ids'].append(idx)
+
+    def run(self) -> list[Node]:
+        """Override run to parse content as a code block"""
+        if ':' in self.name:
+            self.domain, self.objtype = self.name.split(':', 1)
+        else:
+            self.domain, self.objtype = '', self.name
+        self.indexnode = addnodes.index(entries=[])
+
+        node = addnodes.desc()
+        node.document = self.state.document
+        source, line = self.get_source_info()
+        if line is not None:
+            line -= 1
+        self.state.document.note_source(source, line)
+        node['domain'] = self.domain
+        # 'desctype' is a backwards compatible attribute
+        node['objtype'] = node['desctype'] = self.objtype
+        node['noindex'] = noindex = ('noindex' in self.options)
+        node['noindexentry'] = ('noindexentry' in self.options)
+        node['nocontentsentry'] = ('nocontentsentry' in self.options)
+        if self.domain:
+            node['classes'].append(self.domain)
+        node['classes'].append(node['objtype'])
+
+        self.names = []
+        signatures = self.get_signatures()
+        for sig in signatures:
+            # add a signature node for each signature in the current unit
+            # and add a reference target for it
+            signode = addnodes.desc_signature(sig, '')
+            self.set_source_info(signode)
+            node.append(signode)
+            try:
+                # name can also be a tuple, e.g. (classname, objname);
+                # this is strictly domain-specific (i.e. no assumptions may
+                # be made in this base class)
+                name = self.handle_signature(sig, signode)
+            except ValueError:
+                # signature parsing failed
+                signode.clear()
+                signode += addnodes.desc_name(sig, sig)
+                continue  # we don't want an index entry here
+            finally:
+                # Private attributes for ToC generation. Will be modified or removed
+                # without notice.
+                if self.env.app.config.toc_object_entries:
+                    signode['_toc_parts'] = self._object_hierarchy_parts(signode)
+                    signode['_toc_name'] = self._toc_entry_name(signode)
+                else:
+                    signode['_toc_parts'] = ()
+                    signode['_toc_name'] = ''
+            if name not in self.names:
+                self.names.append(name)
+                if not noindex:
+                    # only add target and index entry if this is the first
+                    # description of the object with this name in this desc block
+                    self.add_target_and_index(name, sig, signode)
+        
+        # handle code
+        code = '\n'.join(self.content)
+        literal: Element = nodes.literal_block(code, code)
+        if 'lineno-start' in self.options:
+            literal['linenos'] = True
+            literal['highlight_args'] = {
+                'linenostart': self.options['lineno-start']
+            }
+        literal['classes'] += self.options.get('class', [])
+        literal['language'] = self.options.get('language')
+        literal = container_wrapper(self, literal, self.options.get('source'))
+
+        return [self.indexnode, node, literal]
+
 class TagIndex(Index):
     """A custom directive that creates a tag matrix."""
     
@@ -223,6 +367,7 @@ class CellDomain(CommandDomain):
 
     directives = {
         'def': CellNode,
+        'source': CellSourceNode,
     }
 
     indices = {