From 25822be9301f2e2a08a87acced40f1395cd46201 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:37:02 +1300 Subject: [PATCH 01/22] docs: Initial documentation for generated help Very briefly on command help Mostly focused on v1 and v2 of the cell help, but still largely bullet points --- .../extending_yosys/contributing.rst | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/docs/source/yosys_internals/extending_yosys/contributing.rst b/docs/source/yosys_internals/extending_yosys/contributing.rst index 70170fc48..65f29e592 100644 --- a/docs/source/yosys_internals/extending_yosys/contributing.rst +++ b/docs/source/yosys_internals/extending_yosys/contributing.rst @@ -176,3 +176,163 @@ Additional details + if the design is too big for a comment, consider turning it into a `Gist`_ .. _Gist: https://gist.github.com/ + +Generated help messages and documentation +----------------------------------------- + +Command help +~~~~~~~~~~~~ + +- `help` without arguments + + - lists all commands with their ``Pass::short_help`` + +- ``help `` + + - calls ``Pass::help()`` for ```` + +.. note:: + + A more expressive way to generate formatted help messages is `in + development`_. + +.. _in development: https://github.com/YosysHQ/yosys/pull/4860 + +Cell help +~~~~~~~~~ + +- :file:`techlibs/common/simcells.v` and :file:`techlibs/common/simlib.v` +- parsed by :file:`techlibs/common/cellhelp.py` +- comments preceding cell type converted to a ``SimHelper`` struct, defined in + :file:`kernel/register.cc` +- ``#include``d in :file:`kernel/register.cc`, creating a map of cell types to + their ``SimHelper`` struct. + +- ``help -cells`` + + - lists all cells with their input/output ports + +- ``help `` + + - prints help message for ```` + - constructed from ``SimHelper`` content depending on version + +- ``help +`` + + - prints verilog code for ```` + +v1 (default) +^^^^^^^^^^^^ + +- Legacy format +- Expects formatting as follows: + +.. code-block:: verilog + + //- + //- $ () + //* group + //- + //- + //- + module \$ (); + // ... + endmodule + +- ``//* group`` line is generally after the cell signature, but can appear + anywhere in the comment block + + - determines where the cell is included in sphinx + - check :file:`docs/source/cell` for current groups + - a missing group will raise an error + - assigning an unused group will result in the cell not being included in the + sphinx docs + +- the port signature line (``//- $ ()``) must start with (at + least) 4 spaces (not tabs) + + - the empty lines (``//-``) before/after the signature are required + +- cell description can be multiple lines, but each line must start with ``//-`` + and a space + + - description should have a trailing empty line + - line breaks are preserved in `help` calls but will be rendered as text in + sphinx docs, with empty lines being required between paragraphs + +- cells in :file:`techlibs/common/simcells.v` can have optional truth table at + the end of the cell description which is rendered in sphinx docs as a literal + code block +- e.g. `$_NOT_`: + +.. code-block:: verilog + + //- + //- $_NOT_ (A, Y) + //* group comb_simple + //- + //- An inverter gate. + //- + //- Truth table: A | Y + //- ---+--- + //- 0 | 1 + //- 1 | 0 + //- + +v2 (more expressive) +^^^^^^^^^^^^^^^^^^^^ + +- each field of the ``SimHelper`` struct can be assigned with: + +.. code-block:: verilog + + //* + +- ``ver`` must be explicitly set as ``2`` +- optional fields ``title`` and ``tags`` + + - title used for better displaying of cell + - tags is a space-separated list of :doc:`/cell/properties` + +- the port signature is automatically generated from the ``module`` definition +- cell description is the same as v1 +- can link to commands or passes using backticks (`````) +- e.g. `$nex`: + +.. code-block:: verilog + + //* ver 2 + //* title Case inequality + //* group binary + //* tags x-aware + //- This corresponds to the Verilog '!==' operator. + //- + //- Refer to `$eqx` for more details. + //- + +- code blocks can be included as following: + +.. code-block:: verilog + + //- text + //- :: + //- + //- monospaced text + //- + //- indentation and line length will be preserved, giving a scroll bar if necessary for the browser window + //- + //- more text + +- the empty line after the ``::`` and before the text continues are required +- the ``::`` line will be ignored in the `help` call while sphinx docs will + treat everything that follows as a literal block until the next unindented + line: + + text + :: + + monospaced text + + indentation and line length will be preserved, giving a scroll bar if necessary for the browser window + + more text From b9182ab8b77cc0cf27cef791a013aafc3810805e Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 14 Nov 2025 13:37:02 +1300 Subject: [PATCH 02/22] Update checklist for adding internal cell types Add line about v2 cell help Update docs path to point to the right place --- kernel/rtlil.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/rtlil.cc b/kernel/rtlil.cc index adfda7d5f..ead667cbd 100644 --- a/kernel/rtlil.cc +++ b/kernel/rtlil.cc @@ -2475,7 +2475,8 @@ namespace { * * Things to do after finalizing the cell interface: * - Add support to kernel/satgen.h for the new cell type - * - Add to docs/source/CHAPTER_CellLib.rst (or just add a fixme to the bottom) + * - Maybe add v2 cell help fields (title, tags) + * - Add extra details to relevant docs/source/cell/word_*.rst (or just add a todo to the top) * - Maybe add support to the Verilog backend for dumping such cells as expression * */ From 2da76274916b1a5dec09b13f535812559e01b3d9 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 14 Nov 2025 14:21:59 +1300 Subject: [PATCH 03/22] docs: Move help guides to documenting.rst --- docs/source/cmd/index_internal.rst | 147 -------- .../extending_yosys/contributing.rst | 160 --------- .../extending_yosys/documenting.rst | 314 ++++++++++++++++++ .../yosys_internals/extending_yosys/index.rst | 1 + 4 files changed, 315 insertions(+), 307 deletions(-) create mode 100644 docs/source/yosys_internals/extending_yosys/documenting.rst diff --git a/docs/source/cmd/index_internal.rst b/docs/source/cmd/index_internal.rst index ab9c13aba..d485be705 100644 --- a/docs/source/cmd/index_internal.rst +++ b/docs/source/cmd/index_internal.rst @@ -3,150 +3,3 @@ Internal commands for developers .. autocmdgroup:: internal :members: - -Writing command help --------------------- - -- use `chformal` as an example -- generated help content below - -.. _chformal autocmd: - -.. autocmd:: chformal - :noindex: - -The ``formatted_help()`` method -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- ``PrettyHelp::get_current()`` -- ``PrettyHelp::set_group()`` - - + used with ``.. autocmdgroup:: `` - + can assign group and return false - + if no group is set, will try to use ``source_location`` and assign group - from path to source file - -- return value - - + true means help content added to current ``PrettyHelp`` - + false to use ``Pass::help()`` - -- adding content - - + help content is a list of ``ContentListing`` nodes, each one having a type, - body, and its own list of children ``ContentListing``\ s - + ``PrettyHelp::get_root()`` returns the root ``ContentListing`` (``type="root"``) - + ``ContentListing::{usage, option, codeblock, paragraph}`` each add a - ``ContentListing`` to the current node, with type the same as the method - - * the first argument is the body of the new node - * ``usage`` shows how to call the command (i.e. its "signature") - * ``paragraph`` content is formatted as a paragraph of text with line breaks - added automatically - * ``codeblock`` content is displayed verbatim, use line breaks as desired; - takes an optional ``language`` argument for assigning the language in RST - output for code syntax highlighting (use ``yoscrypt`` for yosys script - syntax highlighting) - * ``option`` lists a single option for the command, usually starting with a - dash (``-``); takes an optional second argument which adds a paragraph - node as a means of description - - + ``ContentListing::open_usage`` creates and returns a new usage node, can be - used to e.g. add text/options specific to a given usage of the command - + ``ContentListing::open_option`` creates and returns a new option node, can - be used to e.g. add multiple paragraphs to an option's description - + paragraphs are treated as raw RST, allowing for inline formatting and - references as if it were written in the RST file itself - -.. literalinclude:: /generated/chformal.cc - :language: c++ - :start-at: bool formatted_help() - :end-before: void execute - :caption: ``ChformalPass::formatted_help()`` from :file:`passes/cmds/chformal.cc` - -Dumping command help to json -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- `help -dump-cmds-json cmds.json` - - + generates a ``ContentListing`` for each command registered in Yosys - + tries to parse unformatted ``Pass::help()`` output if - ``Pass::formatted_help()`` is unimplemented or returns false - - * if a line starts with four spaces followed by the name of the command then - a space, it is parsed as a signature (usage node) - * if a line is indented and starts with a dash (``-``), it is parsed as an - option - * anything else is parsed as a codeblock and added to either the root node - or the current option depending on the indentation - - + dictionary of command name to ``ContentListing`` - - * uses ``ContentListing::to_json()`` recursively for each node in root - * root node used for source location of class definition - * includes flags set during pass constructor (e.g. ``experimental_flag`` set - by ``Pass::experimental()``) - * also title (``short_help`` argument in ``Pass::Pass``), group, and class - name - - + dictionary of group name to list of commands in that group - -- used by sphinx autodoc to generate help content - -.. literalinclude:: /generated/cmds.json - :language: json - :start-at: "chformal": { - :end-before: "chparam": { - :caption: `chformal` in generated :file:`cmds.json` - -.. note:: Synthesis command scripts are special cased - - If the final block of help output starts with the string `"The following - commands are executed by this synthesis command:\n"`, then the rest of the - code block is formatted as ``yoscrypt`` (e.g. `synth_ice40`). The caveat - here is that if the ``script()`` calls ``run()`` on any commands *prior* to - the first ``check_label`` then the auto detection will break and revert to - unformatted code (e.g. `synth_fabulous`). - -Command line rendering -~~~~~~~~~~~~~~~~~~~~~~ - -- if ``Pass::formatted_help()`` returns true, will call - ``PrettyHelp::log_help()`` - - + traverse over the children of the root node and render as plain text - + effectively the reverse of converting unformatted ``Pass::help()`` text - + lines are broken at 80 characters while maintaining indentation (controlled - by ``MAX_LINE_LEN`` in :file:`kernel/log_help.cc`) - + each line is broken into words separated by spaces, if a given word starts - and ends with backticks they will be stripped - -- if it returns false it will call ``Pass::help()`` which should call ``log()`` - directly to print and format help text - - + if ``Pass::help()`` is not overridden then a default message about missing - help will be displayed - -.. literalinclude:: /generated/chformal.log - :lines: 2- - -RST generated from autocmd -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- below is the raw RST output from ``autocmd`` (``YosysCmdDocumenter`` class in - :file:`docs/util/cmd_documenter.py`) for `chformal` command -- heading will be rendered as a subheading of the most recent heading (see - `chformal autocmd`_ above rendered under `Writing command help`_) -- ``.. cmd:def:: `` line is indexed for cross references with ``:cmd:ref:`` - directive (`chformal autocmd`_ above uses ``:noindex:`` option so that - `chformal` still links to the correct location) - - + ``:title:`` option controls text that appears when hovering over the - `chformal` link - -- commands with warning flags (experimental or internal) add a ``.. warning`` - block before any of the help content -- if a command has no ``source_location`` the ``.. note`` at the bottom will - instead link to :doc:`/cmd/index_other` - -.. autocmd_rst:: chformal diff --git a/docs/source/yosys_internals/extending_yosys/contributing.rst b/docs/source/yosys_internals/extending_yosys/contributing.rst index 65f29e592..70170fc48 100644 --- a/docs/source/yosys_internals/extending_yosys/contributing.rst +++ b/docs/source/yosys_internals/extending_yosys/contributing.rst @@ -176,163 +176,3 @@ Additional details + if the design is too big for a comment, consider turning it into a `Gist`_ .. _Gist: https://gist.github.com/ - -Generated help messages and documentation ------------------------------------------ - -Command help -~~~~~~~~~~~~ - -- `help` without arguments - - - lists all commands with their ``Pass::short_help`` - -- ``help `` - - - calls ``Pass::help()`` for ```` - -.. note:: - - A more expressive way to generate formatted help messages is `in - development`_. - -.. _in development: https://github.com/YosysHQ/yosys/pull/4860 - -Cell help -~~~~~~~~~ - -- :file:`techlibs/common/simcells.v` and :file:`techlibs/common/simlib.v` -- parsed by :file:`techlibs/common/cellhelp.py` -- comments preceding cell type converted to a ``SimHelper`` struct, defined in - :file:`kernel/register.cc` -- ``#include``d in :file:`kernel/register.cc`, creating a map of cell types to - their ``SimHelper`` struct. - -- ``help -cells`` - - - lists all cells with their input/output ports - -- ``help `` - - - prints help message for ```` - - constructed from ``SimHelper`` content depending on version - -- ``help +`` - - - prints verilog code for ```` - -v1 (default) -^^^^^^^^^^^^ - -- Legacy format -- Expects formatting as follows: - -.. code-block:: verilog - - //- - //- $ () - //* group - //- - //- - //- - module \$ (); - // ... - endmodule - -- ``//* group`` line is generally after the cell signature, but can appear - anywhere in the comment block - - - determines where the cell is included in sphinx - - check :file:`docs/source/cell` for current groups - - a missing group will raise an error - - assigning an unused group will result in the cell not being included in the - sphinx docs - -- the port signature line (``//- $ ()``) must start with (at - least) 4 spaces (not tabs) - - - the empty lines (``//-``) before/after the signature are required - -- cell description can be multiple lines, but each line must start with ``//-`` - and a space - - - description should have a trailing empty line - - line breaks are preserved in `help` calls but will be rendered as text in - sphinx docs, with empty lines being required between paragraphs - -- cells in :file:`techlibs/common/simcells.v` can have optional truth table at - the end of the cell description which is rendered in sphinx docs as a literal - code block -- e.g. `$_NOT_`: - -.. code-block:: verilog - - //- - //- $_NOT_ (A, Y) - //* group comb_simple - //- - //- An inverter gate. - //- - //- Truth table: A | Y - //- ---+--- - //- 0 | 1 - //- 1 | 0 - //- - -v2 (more expressive) -^^^^^^^^^^^^^^^^^^^^ - -- each field of the ``SimHelper`` struct can be assigned with: - -.. code-block:: verilog - - //* - -- ``ver`` must be explicitly set as ``2`` -- optional fields ``title`` and ``tags`` - - - title used for better displaying of cell - - tags is a space-separated list of :doc:`/cell/properties` - -- the port signature is automatically generated from the ``module`` definition -- cell description is the same as v1 -- can link to commands or passes using backticks (`````) -- e.g. `$nex`: - -.. code-block:: verilog - - //* ver 2 - //* title Case inequality - //* group binary - //* tags x-aware - //- This corresponds to the Verilog '!==' operator. - //- - //- Refer to `$eqx` for more details. - //- - -- code blocks can be included as following: - -.. code-block:: verilog - - //- text - //- :: - //- - //- monospaced text - //- - //- indentation and line length will be preserved, giving a scroll bar if necessary for the browser window - //- - //- more text - -- the empty line after the ``::`` and before the text continues are required -- the ``::`` line will be ignored in the `help` call while sphinx docs will - treat everything that follows as a literal block until the next unindented - line: - - text - :: - - monospaced text - - indentation and line length will be preserved, giving a scroll bar if necessary for the browser window - - more text diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst new file mode 100644 index 000000000..aee2cf80c --- /dev/null +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -0,0 +1,314 @@ +Generated help messages and documentation +========================================= + +- (virtually) all Yosys commands and built-in cell types have help text +- see `help` (or call ``yosys -h help``) +- this help text is also used to generate the :doc:`/cmd_ref` and + :doc:`/cell_index` documentation, available online via `ReadtheDocs`_ + +.. _ReadtheDocs: https://about.readthedocs.com/ + +Command help +------------ + +- `help` without arguments + + - lists all commands with their ``Pass::short_help`` + +- ``help `` + + - provides command specific help content + - as an example, the help content for `chformal` is shown below + +.. _chformal autocmd: + +.. autocmd:: chformal + :noindex: + +The ``Pass::help()`` method +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- call ``log()`` directly to print and format help text +- arbitrary formatting is possible, though some recommendations for consistency + and correct parsing during RST generation + + + see `Dumping command help to json`_ for details + +The ``Pass::formatted_help()`` method +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``PrettyHelp::get_current()`` +- ``PrettyHelp::set_group()`` + + + used with ``.. autocmdgroup:: `` + + can assign group and return false + + if no group is set, will try to use ``source_location`` and assign group + from path to source file + +- return value + + + true means help content added to current ``PrettyHelp`` + + false to use ``Pass::help()`` + +- adding content + + + help content is a list of ``ContentListing`` nodes, each one having a type, + body, and its own list of children ``ContentListing``\ s + + ``PrettyHelp::get_root()`` returns the root ``ContentListing`` (``type="root"``) + + ``ContentListing::{usage, option, codeblock, paragraph}`` each add a + ``ContentListing`` to the current node, with type the same as the method + + * the first argument is the body of the new node + * ``usage`` shows how to call the command (i.e. its "signature") + * ``paragraph`` content is formatted as a paragraph of text with line breaks + added automatically + * ``codeblock`` content is displayed verbatim, use line breaks as desired; + takes an optional ``language`` argument for assigning the language in RST + output for code syntax highlighting (use ``yoscrypt`` for yosys script + syntax highlighting) + * ``option`` lists a single option for the command, usually starting with a + dash (``-``); takes an optional second argument which adds a paragraph + node as a means of description + + + ``ContentListing::open_usage`` creates and returns a new usage node, can be + used to e.g. add text/options specific to a given usage of the command + + ``ContentListing::open_option`` creates and returns a new option node, can + be used to e.g. add multiple paragraphs to an option's description + + paragraphs are treated as raw RST, allowing for inline formatting and + references as if it were written in the RST file itself + +.. literalinclude:: /generated/chformal.cc + :language: c++ + :start-at: bool formatted_help() + :end-before: void execute + :caption: ``ChformalPass::formatted_help()`` from :file:`passes/cmds/chformal.cc` + + +Dumping command help to json +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- `help -dump-cmds-json cmds.json` + + + generates a ``ContentListing`` for each command registered in Yosys + + tries to parse unformatted ``Pass::help()`` output if + ``Pass::formatted_help()`` is unimplemented or returns false + + * if a line starts with four spaces followed by the name of the command then + a space, it is parsed as a signature (usage node) + * if a line is indented and starts with a dash (``-``), it is parsed as an + option + * anything else is parsed as a codeblock and added to either the root node + or the current option depending on the indentation + + + dictionary of command name to ``ContentListing`` + + * uses ``ContentListing::to_json()`` recursively for each node in root + * root node used for source location of class definition + * includes flags set during pass constructor (e.g. ``experimental_flag`` set + by ``Pass::experimental()``) + * also title (``short_help`` argument in ``Pass::Pass``), group, and class + name + + + dictionary of group name to list of commands in that group + +- used by sphinx autodoc to generate help content + +.. literalinclude:: /generated/cmds.json + :language: json + :start-at: "chformal": { + :end-before: "chparam": { + :caption: `chformal` in generated :file:`cmds.json` + +.. note:: Synthesis command scripts are special cased + + If the final block of help output starts with the string `"The following + commands are executed by this synthesis command:\n"`, then the rest of the + code block is formatted as ``yoscrypt`` (e.g. `synth_ice40`). The caveat + here is that if the ``script()`` calls ``run()`` on any commands *prior* to + the first ``check_label`` then the auto detection will break and revert to + unformatted code (e.g. `synth_fabulous`). + + +Command line rendering +~~~~~~~~~~~~~~~~~~~~~~ + +- if ``Pass::formatted_help()`` returns true, will call + ``PrettyHelp::log_help()`` + + + traverse over the children of the root node and render as plain text + + effectively the reverse of converting unformatted ``Pass::help()`` text + + lines are broken at 80 characters while maintaining indentation (controlled + by ``MAX_LINE_LEN`` in :file:`kernel/log_help.cc`) + + each line is broken into words separated by spaces, if a given word starts + and ends with backticks they will be stripped + +- if it returns false it will call ``Pass::help()`` which should call ``log()`` + directly to print and format help text + + + if ``Pass::help()`` is not overridden then a default message about missing + help will be displayed + +.. literalinclude:: /generated/chformal.log + :lines: 2- + + +RST generated from autocmd +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- below is the raw RST output from ``autocmd`` (``YosysCmdDocumenter`` class in + :file:`docs/util/cmd_documenter.py`) for `chformal` command +- heading will be rendered as a subheading of the most recent heading (see + `chformal autocmd`_ above rendered under `Command help`_) +- ``.. cmd:def:: `` line is indexed for cross references with ``:cmd:ref:`` + directive (`chformal autocmd`_ above uses ``:noindex:`` option so that + `chformal` still links to the correct location) + + + ``:title:`` option controls text that appears when hovering over the + `chformal` link + +- commands with warning flags (experimental or internal) add a ``.. warning`` + block before any of the help content +- if a command has no ``source_location`` the ``.. note`` at the bottom will + instead link to :doc:`/cmd/index_other` + +.. autocmd_rst:: chformal + + +Cell help +--------- + +- :file:`techlibs/common/simcells.v` and :file:`techlibs/common/simlib.v` +- parsed by :file:`techlibs/common/cellhelp.py` +- comments preceding cell type converted to a ``SimHelper`` struct, defined in + :file:`kernel/register.cc` +- ``#include``d in :file:`kernel/register.cc`, creating a map of cell types to + their ``SimHelper`` struct. + +- ``help -cells`` + + - lists all cells with their input/output ports + +- ``help `` + + - prints help message for ```` + - constructed from ``SimHelper`` content depending on version + +- ``help +`` + + - prints verilog code for ```` + +v1 (default) +~~~~~~~~~~~~ + +- Legacy format +- Expects formatting as follows: + +.. code-block:: verilog + + //- + //- $ () + //* group + //- + //- + //- + module \$ (); + // ... + endmodule + +- ``//* group`` line is generally after the cell signature, but can appear + anywhere in the comment block + + - determines where the cell is included in sphinx + - check :file:`docs/source/cell` for current groups + - a missing group will raise an error + - assigning an unused group will result in the cell not being included in the + sphinx docs + +- the port signature line (``//- $ ()``) must start with (at + least) 4 spaces (not tabs) + + - the empty lines (``//-``) before/after the signature are required + +- cell description can be multiple lines, but each line must start with ``//-`` + and a space + + - description should have a trailing empty line + - line breaks are preserved in `help` calls but will be rendered as text in + sphinx docs, with empty lines being required between paragraphs + +- cells in :file:`techlibs/common/simcells.v` can have optional truth table at + the end of the cell description which is rendered in sphinx docs as a literal + code block +- e.g. `$_NOT_`: + +.. code-block:: verilog + + //- + //- $_NOT_ (A, Y) + //* group comb_simple + //- + //- An inverter gate. + //- + //- Truth table: A | Y + //- ---+--- + //- 0 | 1 + //- 1 | 0 + //- + +v2 (more expressive) +~~~~~~~~~~~~~~~~~~~~ + +- each field of the ``SimHelper`` struct can be assigned with: + +.. code-block:: verilog + + //* + +- ``ver`` must be explicitly set as ``2`` +- optional fields ``title`` and ``tags`` + + - title used for better displaying of cell + - tags is a space-separated list of :doc:`/cell/properties` + +- the port signature is automatically generated from the ``module`` definition +- cell description is the same as v1 +- can link to commands or passes using backticks (`````) +- e.g. `$nex`: + +.. code-block:: verilog + + //* ver 2 + //* title Case inequality + //* group binary + //* tags x-aware + //- This corresponds to the Verilog '!==' operator. + //- + //- Refer to `$eqx` for more details. + //- + +- code blocks can be included as following: + +.. code-block:: verilog + + //- text + //- :: + //- + //- monospaced text + //- + //- indentation and line length will be preserved, giving a scroll bar if necessary for the browser window + //- + //- more text + +- the empty line after the ``::`` and before the text continues are required +- the ``::`` line will be ignored in the `help` call while sphinx docs will + treat everything that follows as a literal block until the next unindented + line: + + text + :: + + monospaced text + + indentation and line length will be preserved, giving a scroll bar if necessary for the browser window + + more text diff --git a/docs/source/yosys_internals/extending_yosys/index.rst b/docs/source/yosys_internals/extending_yosys/index.rst index 72843ecd6..3b8aad541 100644 --- a/docs/source/yosys_internals/extending_yosys/index.rst +++ b/docs/source/yosys_internals/extending_yosys/index.rst @@ -13,5 +13,6 @@ of interest for developers looking to customise Yosys builds. functional_ir advanced_bugpoint contributing + documenting test_suites From 625eae987f41a9b7ebe3da8d9317e8ee00ae00cd Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 14 Nov 2025 14:31:28 +1300 Subject: [PATCH 04/22] log_help.cc: Fix source_location on option --- kernel/log_help.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/log_help.cc b/kernel/log_help.cc index 93b91b08b..cb22d236e 100644 --- a/kernel/log_help.cc +++ b/kernel/log_help.cc @@ -44,7 +44,7 @@ void ContentListing::usage(const string &text, void ContentListing::option(const string &text, const string &description, const source_location location) { - auto option = open_option(text); + auto option = open_option(text, location); if (description.length()) option->add_content("text", description, location); } From 1186ca8f687aecaca0edee5182ebef4c221aa83e Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 14 Nov 2025 18:10:34 +1300 Subject: [PATCH 05/22] documenting.rst: WIP paragraphs First couple sections done, plus a couple formatting tidy ups later on. --- .../extending_yosys/documenting.rst | 73 +++++++++++++++---- 1 file changed, 57 insertions(+), 16 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index aee2cf80c..431c5ddda 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -1,24 +1,54 @@ Generated help messages and documentation ========================================= -- (virtually) all Yosys commands and built-in cell types have help text -- see `help` (or call ``yosys -h help``) -- this help text is also used to generate the :doc:`/cmd_ref` and - :doc:`/cell_index` documentation, available online via `ReadtheDocs`_ +All Yosys commands and built-in cell types should include help text, documenting +their functionality for users. This help text is made available through the +`help` command, and online via `ReadtheDocs`_ as part of the :doc:`/cmd_ref` and +:doc:`/cell_index` documentation. When running locally, any commands provided +by loaded plugins (either from the command line when calling ``yosys``, or +dynamically with the `plugin` command) will also be available to the `help` +command. .. _ReadtheDocs: https://about.readthedocs.com/ +.. note:: + + Since help text for commands is generated from compiled code, the online help + may differ from that produced by `help`. Some commands, like `abc`, may be + completely unavailable depending on compile flags; while others may limit + specific features, such as whether the `synth` script pass uses ABC. + Command help ------------ -- `help` without arguments +The first stop for command help text is the ``Pass::short_help``. This is a +short sentence describing the pass, and is set in the ``Pass`` constructor, as +demonstrated here with `chformal`. - - lists all commands with their ``Pass::short_help`` +.. literalinclude:: /generated/chformal.cc + :language: c++ + :start-at: public Pass { + :end-before: bool formatted_help() + :caption: ``ChformalPass()`` from :file:`passes/cmds/chformal.cc` + :append: // ... + } ChformalPass; -- ``help `` +All currently available commands are listed with their ``short_help`` string +when calling `help` without arguments, and is more or less the same as the +unlisted :ref:`command index `. The string is also used when +hovering over links to commands in the documentation, and in section headings +like :ref:`chformal autocmd`. - - provides command specific help content - - as an example, the help content for `chformal` is shown below +The next section shows the complete help text for the `chformal` command. This +can be displayed locally by using ``help `` (or ``yosys -h `` +from the command line). The general format is to show each usage signature, +followed by a paragraph describing what the pass does, and a list of options or +flags available. Additional arguments in the signature or option may use square +brackets (``[]``) to indicate optional parts, and angle brackets (``<>``) for +required parts. The pipe character ``|`` may be used to indicate mutually +exclusive arguments. + +.. todo:: decide on a formatting style for pass options .. _chformal autocmd: @@ -28,11 +58,22 @@ Command help The ``Pass::help()`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- call ``log()`` directly to print and format help text -- arbitrary formatting is possible, though some recommendations for consistency - and correct parsing during RST generation +This is the original way to provide help text, and as of this writing is still +the most common. The ``log()`` function should be called directly to print and +format the help text, and each line should be limited to 80 (printed) +characters. While it is possible to provide arbitrary formatting, it is +preferred to follow the guidelines here to maintain consistency with other +passes and to assist in correct parsing and formatting during RST generation +(i.e. these docs). - + see `Dumping command help to json`_ for details +The first line should always be a blank line, followed by the primary usage +signature for the command. Each usage signature should be indented with 4 +spaces, and followed by a blank line. Each option or flag should start on a new +line indented with 4 spaces, followed by a description of the option which is +indented by a further 4 spaces, and then a blank line. Option descriptions +typically start with lower case, and may forgo a trailing period (``.``). Where +multiple options share a description the blank line between options should be +omitted. The ``Pass::formatted_help()`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -121,8 +162,8 @@ Dumping command help to json .. note:: Synthesis command scripts are special cased - If the final block of help output starts with the string `"The following - commands are executed by this synthesis command:\n"`, then the rest of the + If the final block of help output starts with the string ``"The following + commands are executed by this synthesis command:\n"``, then the rest of the code block is formatted as ``yoscrypt`` (e.g. `synth_ice40`). The caveat here is that if the ``script()`` calls ``run()`` on any commands *prior* to the first ``check_label`` then the auto detection will break and revert to @@ -181,7 +222,7 @@ Cell help - parsed by :file:`techlibs/common/cellhelp.py` - comments preceding cell type converted to a ``SimHelper`` struct, defined in :file:`kernel/register.cc` -- ``#include``d in :file:`kernel/register.cc`, creating a map of cell types to +- ``#include``\ d in :file:`kernel/register.cc`, creating a map of cell types to their ``SimHelper`` struct. - ``help -cells`` From 0a6ce3d820349dfb2a7e840f0104030e2fbc3c57 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 15 Nov 2025 11:54:42 +1300 Subject: [PATCH 06/22] documenting.rst: formatted_help --- .../extending_yosys/documenting.rst | 155 +++++++++++++----- 1 file changed, 113 insertions(+), 42 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index 431c5ddda..67b3383aa 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -22,15 +22,16 @@ Command help ------------ The first stop for command help text is the ``Pass::short_help``. This is a -short sentence describing the pass, and is set in the ``Pass`` constructor, as -demonstrated here with `chformal`. +short sentence describing the pass, and is set in the ``Pass`` constructor with +the name of the pass, as demonstrated here with `chformal`. .. literalinclude:: /generated/chformal.cc :language: c++ :start-at: public Pass { - :end-before: bool formatted_help() + :end-at: ChformalPass() :caption: ``ChformalPass()`` from :file:`passes/cmds/chformal.cc` - :append: // ... + :append: + // ... } ChformalPass; All currently available commands are listed with their ``short_help`` string @@ -41,12 +42,19 @@ like :ref:`chformal autocmd`. The next section shows the complete help text for the `chformal` command. This can be displayed locally by using ``help `` (or ``yosys -h `` -from the command line). The general format is to show each usage signature, -followed by a paragraph describing what the pass does, and a list of options or -flags available. Additional arguments in the signature or option may use square -brackets (``[]``) to indicate optional parts, and angle brackets (``<>``) for -required parts. The pipe character ``|`` may be used to indicate mutually -exclusive arguments. +from the command line). The general format is to show each usage signature (how +the command is called), followed by a paragraph describing what the pass does, +and a list of options or flags available. Additional arguments in the signature +or option may use square brackets (``[]``) to indicate optional parts, and angle +brackets (``<>``) for required parts. The pipe character ``|`` may be used to +indicate mutually exclusive arguments. + +.. note:: + + Remember that when using ``Frontend`` and ``Backend`` the pass name will be + be prefixed with ``read_`` or ``write_`` respectively. Usage signatures must + match the pass name available in commands/scripts, which is available as + ``Pass::pass_name``. .. todo:: decide on a formatting style for pass options @@ -75,48 +83,70 @@ typically start with lower case, and may forgo a trailing period (``.``). Where multiple options share a description the blank line between options should be omitted. + The ``Pass::formatted_help()`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- ``PrettyHelp::get_current()`` -- ``PrettyHelp::set_group()`` +The ``formatted_help`` method serves two purposes in help generation, both of +which are optional. In both cases, any pass which uses the method should +``#include "kernel/log_help.h"``, and begin the method by calling ``auto *help = +PrettyHelp::get_current();``. The method finishes by returning a boolean value. +``true`` means help content has been added to the current ``PrettyHelp``, while +``false`` indicates that ``Pass::help()`` should be called instead. - + used with ``.. autocmdgroup:: `` - + can assign group and return false - + if no group is set, will try to use ``source_location`` and assign group - from path to source file +Setting a command group +^^^^^^^^^^^^^^^^^^^^^^^ -- return value +Command groups are used when `Dumping command help to json`_, so that related +commands can be presented together in documentation. For example, all of the +formal commands (which `chformal` is one of) are listed under +:doc:`/cmd/index_formal`, by using the ``autocmdgroup`` directive in +:file:`docs/source/cmd/index_formal.rst`. By default, commands are grouped by +their source location, such that the group is the same as the path to the source +file. - + true means help content added to current ``PrettyHelp`` - + false to use ``Pass::help()`` +.. note:: -- adding content + Source location tracking requires :makevar:`ENABLE_HELP_SOURCE` to be set in + the makefile. Some passes, like the ``opt_*`` family, are able to be grouped + by the name of the pass; but most will be assigned the ``unknown`` group. - + help content is a list of ``ContentListing`` nodes, each one having a type, - body, and its own list of children ``ContentListing``\ s - + ``PrettyHelp::get_root()`` returns the root ``ContentListing`` (``type="root"``) - + ``ContentListing::{usage, option, codeblock, paragraph}`` each add a - ``ContentListing`` to the current node, with type the same as the method + For frontends and backends, source code is structured such that different + formats are located in different folders. Default behavior is to instead + group all of these passes as :doc:`/cmd/index_frontends` and + :doc:`/cmd/index_backends` respectively. Without location tracking, the + fallback is to look for passes that start with ``read_`` or ``write_``. - * the first argument is the body of the new node - * ``usage`` shows how to call the command (i.e. its "signature") - * ``paragraph`` content is formatted as a paragraph of text with line breaks - added automatically - * ``codeblock`` content is displayed verbatim, use line breaks as desired; - takes an optional ``language`` argument for assigning the language in RST - output for code syntax highlighting (use ``yoscrypt`` for yosys script - syntax highlighting) - * ``option`` lists a single option for the command, usually starting with a - dash (``-``); takes an optional second argument which adds a paragraph - node as a means of description +It is possible to set the group of a command explicitly with the +``PrettyHelp::set_group()`` method. This allows grouping of commands which may +not share a common source location, as well as ensuring that commands are still +grouped when location tracking is disabled. Because ``Pass::formatted_help()`` +returns if it produced help content, it is completely valid to override the +method, get the current instance of ``PrettyHelp``, set the command group, and +then return ``false``. - + ``ContentListing::open_usage`` creates and returns a new usage node, can be - used to e.g. add text/options specific to a given usage of the command - + ``ContentListing::open_option`` creates and returns a new option node, can - be used to e.g. add multiple paragraphs to an option's description - + paragraphs are treated as raw RST, allowing for inline formatting and - references as if it were written in the RST file itself +.. warning:: + + There is currently no warning available for groups that do not have a + corresponding ``autocmdgroup``. If you add a new command group, make sure + that it has a corresponding index page. + + +Rich help text +^^^^^^^^^^^^^^ + +The second purpose of ``Pass::formatted_help`` is to provide richer help +content which is able to take advantage of the reStructuredText formatting used +here in the web docs. It also provides a more fluid way of writing help text, +without getting caught up in the terminal-first spacing requirements of writing +for ``Pass::help()``. + +Help content is a list of ``ContentListing`` nodes on a root node, which can be +found by calling ``PrettyHelp::get_root()``. Each node has a type, a body, and +its own list of children ``ContentListing``\ s. Adding content is done with the +``ContentListing::{usage, option, codeblock, paragraph}`` methods, which each +add a new child node with a type set to the calling method. Let's take a look +at the source code for `chformal`. .. literalinclude:: /generated/chformal.cc :language: c++ @@ -124,6 +154,47 @@ The ``Pass::formatted_help()`` method :end-before: void execute :caption: ``ChformalPass::formatted_help()`` from :file:`passes/cmds/chformal.cc` +We can see that each of the ``ContentListing`` methods have the body of the new +node as the first argument. For a ``usage`` node, this is how to call the +command (i.e. its usage signature). ``paragraph`` nodes contain a paragraph of +text with line breaks added automatically; the argument itself should contain +any line breaks, but the string can be broken across multiple lines as shown. +The body of a ``paragraph`` node is treated as raw RST, allowing for inline +formatting and references as if it were written in the RST file itself. As +shown in the example (and the :ref:`formatted output above `), +this includes using single backticks for linking to cells or commands, and +double backticks for raw code. + +The ``option`` method lists a single option for the command, usually starting +with a dash (``-``). An optional second argument can be provided with adds a +paragraph node as a child of the option, and is used for describing the option. +Where multiple options share a description, it should be added to the last +option. + +.. note:: + + To add multiple paragraphs to an option's description, + ``ContentListing::open_option()`` should be used instead. This method + returns the option node, which can then be used to call + ``ContentList::paragraph()`` multiple times. + +``codeblock`` content is displayed verbatim, and content should include line +breaks as desired. No extra formatting will be applied to the text, and it will +be rendered with a monospace font; making it perfect for code sections or ASCII +art diagrams which render the same on the web as they do in the terminal. An +optional second argument is available for specifying the language in RST output +for code syntax highlighting (use ``yoscrypt`` for yosys script syntax +highlighting). + +.. + not recommended since it (currently) doesn't render in the terminal + + The final method available is ``ContentListing::open_usage``. As with + ``open_option`` creates and returns a new node which can have additional content + added to it directly. For the usage node, this can be used for example to add + text/options specific to a given usage of the command. In the web documentation + any content added in this way will be indented under the usage signature. + Dumping command help to json ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 966f52e123e9e40a81b81bacec0696b8763db853 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 15 Nov 2025 16:06:09 +1300 Subject: [PATCH 07/22] documenting.rst: More json/sphinx bullets Also moved them to the bottom, instead of being under the "command help" heading. Now includes more info about cells, and the custom sphinx domains/directives/roles we have. --- .../extending_yosys/documenting.rst | 214 ++++++++++++------ 1 file changed, 146 insertions(+), 68 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index 67b3383aa..c0f51f637 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -83,6 +83,10 @@ typically start with lower case, and may forgo a trailing period (``.``). Where multiple options share a description the blank line between options should be omitted. +.. note:: + + `Dumping to json`_ has more on how formatting in ``help()`` gets parsed. + The ``Pass::formatted_help()`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -97,7 +101,7 @@ PrettyHelp::get_current();``. The method finishes by returning a boolean value. Setting a command group ^^^^^^^^^^^^^^^^^^^^^^^ -Command groups are used when `Dumping command help to json`_, so that related +Command groups are used when `Dumping to json`_, so that related commands can be presented together in documentation. For example, all of the formal commands (which `chformal` is one of) are listed under :doc:`/cmd/index_formal`, by using the ``autocmdgroup`` directive in @@ -196,51 +200,6 @@ highlighting). any content added in this way will be indented under the usage signature. -Dumping command help to json -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- `help -dump-cmds-json cmds.json` - - + generates a ``ContentListing`` for each command registered in Yosys - + tries to parse unformatted ``Pass::help()`` output if - ``Pass::formatted_help()`` is unimplemented or returns false - - * if a line starts with four spaces followed by the name of the command then - a space, it is parsed as a signature (usage node) - * if a line is indented and starts with a dash (``-``), it is parsed as an - option - * anything else is parsed as a codeblock and added to either the root node - or the current option depending on the indentation - - + dictionary of command name to ``ContentListing`` - - * uses ``ContentListing::to_json()`` recursively for each node in root - * root node used for source location of class definition - * includes flags set during pass constructor (e.g. ``experimental_flag`` set - by ``Pass::experimental()``) - * also title (``short_help`` argument in ``Pass::Pass``), group, and class - name - - + dictionary of group name to list of commands in that group - -- used by sphinx autodoc to generate help content - -.. literalinclude:: /generated/cmds.json - :language: json - :start-at: "chformal": { - :end-before: "chparam": { - :caption: `chformal` in generated :file:`cmds.json` - -.. note:: Synthesis command scripts are special cased - - If the final block of help output starts with the string ``"The following - commands are executed by this synthesis command:\n"``, then the rest of the - code block is formatted as ``yoscrypt`` (e.g. `synth_ice40`). The caveat - here is that if the ``script()`` calls ``run()`` on any commands *prior* to - the first ``check_label`` then the auto detection will break and revert to - unformatted code (e.g. `synth_fabulous`). - - Command line rendering ~~~~~~~~~~~~~~~~~~~~~~ @@ -264,33 +223,16 @@ Command line rendering :lines: 2- -RST generated from autocmd -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- below is the raw RST output from ``autocmd`` (``YosysCmdDocumenter`` class in - :file:`docs/util/cmd_documenter.py`) for `chformal` command -- heading will be rendered as a subheading of the most recent heading (see - `chformal autocmd`_ above rendered under `Command help`_) -- ``.. cmd:def:: `` line is indexed for cross references with ``:cmd:ref:`` - directive (`chformal autocmd`_ above uses ``:noindex:`` option so that - `chformal` still links to the correct location) - - + ``:title:`` option controls text that appears when hovering over the - `chformal` link - -- commands with warning flags (experimental or internal) add a ``.. warning`` - block before any of the help content -- if a command has no ``source_location`` the ``.. note`` at the bottom will - instead link to :doc:`/cmd/index_other` - -.. autocmd_rst:: chformal - - Cell help --------- - :file:`techlibs/common/simcells.v` and :file:`techlibs/common/simlib.v` - parsed by :file:`techlibs/common/cellhelp.py` + + + unlike commands, cell help text is generated at compile time + + only formatting occurs at run time + + no support for custom cells in plugins + - comments preceding cell type converted to a ``SimHelper`` struct, defined in :file:`kernel/register.cc` - ``#include``\ d in :file:`kernel/register.cc`, creating a map of cell types to @@ -299,6 +241,7 @@ Cell help - ``help -cells`` - lists all cells with their input/output ports + - again an unlisted :ref:`cell index ` - ``help `` @@ -424,3 +367,138 @@ v2 (more expressive) indentation and line length will be preserved, giving a scroll bar if necessary for the browser window more text + + +Dumping to json +--------------- + +- `help -dump-cmds-json cmds.json` + + + generates a ``ContentListing`` for each command registered in Yosys + + tries to parse unformatted ``Pass::help()`` output if + ``Pass::formatted_help()`` is unimplemented or returns false + + * if a line starts with four spaces followed by the name of the command then + a space, it is parsed as a signature (usage node) + * if a line is indented and starts with a dash (``-``), it is parsed as an + option + * anything else is parsed as a codeblock and added to either the root node + or the current option depending on the indentation + + + dictionary of command name to ``ContentListing`` + + * uses ``ContentListing::to_json()`` recursively for each node in root + * root node used for source location of class definition + * includes flags set during pass constructor (e.g. ``experimental_flag`` set + by ``Pass::experimental()``) + * also title (``short_help`` argument in ``Pass::Pass``), group, and class + name + + + dictionary of group name to list of commands in that group + +- used by sphinx autodoc to generate help content + +.. literalinclude:: /generated/cmds.json + :language: json + :start-at: "chformal": { + :end-before: "chparam": { + :caption: `chformal` in generated :file:`cmds.json` + +.. note:: Synthesis command scripts are special cased + + If the final block of help output starts with the string ``"The following + commands are executed by this synthesis command:\n"``, then the rest of the + code block is formatted as ``yoscrypt`` (e.g. `synth_ice40`). The caveat + here is that if the ``script()`` calls ``run()`` on any commands *prior* to + the first ``check_label`` then the auto detection will break and revert to + unformatted code (e.g. `synth_fabulous`). + + +Cells and commands in Sphinx +---------------------------- + +To support the rich documentation of commands and cells in Yosys, we use two +custom `Sphinx Domains`_. + +.. _Sphinx Domains: https://www.sphinx-doc.org/en/master/usage/domains/index.html + +- ``yoscrypt`` role allows inline code to have yosys script syntax highlighting +- default role of ``autoref`` + + + any text in single backticks without an explicit role will be assigned this one + + will convert to ``cell:ref`` if it begins with ``$``, otherwise ``cmd:ref`` + + to attempt linking there must be no spaces, and it must not begin with a + dash (``-``) + + ```chformal``` (or ``:autoref:`chformal```) -> ``:cmd:ref:`chformal``` -> `chformal` + + also works with two words, if the first one is ``help`` + + ```help $add``` -> ``:cell:ref:`help $add <$add>``` -> `help $add` + + fallback to formatting as inline yoscrypt + + ```-remove``` -> `-remove` + + ```chformal -remove``` -> ``:yoscrypt:`chformal -remove``` -> `chformal -remove` + +Using autodoc +~~~~~~~~~~~~~ + +- below is the raw RST output from ``autocmd`` (``YosysCmdDocumenter`` class in + :file:`docs/util/cmd_documenter.py`) for `chformal` command +- heading will be rendered as a subheading of the most recent heading (see + `chformal autocmd`_ above rendered under `Command help`_) +- ``.. cmd:def:: `` line is indexed for cross references with ``:cmd:ref:`` + directive (`chformal autocmd`_ above uses ``:noindex:`` option so that + `chformal` still links to the correct location) + + + ``:title:`` option controls text that appears when hovering over the + `chformal` link + +- commands with warning flags (experimental or internal) add a ``.. warning`` + block before any of the help content +- if a command has no ``source_location`` the ``.. note`` at the bottom will + instead link to :doc:`/cmd/index_other` + +.. _showing autocmd generated rst: + +.. autocmd_rst:: chformal + +- command groups documented with ``autocmdgroup `` + + + with ``:members:`` option this is the same as calling ``autocmd`` for each + member of the group + +- ``autocell`` and ``autocellgroup`` + + + very similar to ``autocmd`` and ``autocmdgroup`` but for cells instead of + commands (``YosysCellDocumenter`` in :file:`docs/util/cell_documenter.py`) + + optionally includes verilog source for cell(s) with ``:source:`` option + (plus ``:linenos:``) + + cell definitions do not include titles + + cells can have properties (:ref:`propindex`) + +- bonus ``autocmd_rst``, used exclusively on this page for `showing autocmd + generated rst`_ + +Our custom Sphinx domains +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``cmd`` and ``cell`` +- Directives + + + ``cmd:def`` provide command definition + + ``cmd:usage`` used by ``autocmd`` for command usage signatures + + ``cell:def`` provide cell definition + + ``cell:defprop`` provide cell property definition (used in + :doc:`/cell/properties`) + + ``cell:source`` used by ``autocell`` for simulation models + +- Roles + + + ``cmd:ref`` link to a ``cmd:def`` with the same name + + ``cmd:title`` same as ``cmd:ref``, but includes the short help in the text + + - ``:cmd:title:`chformal``` -> :cmd:title:`chformal` + + + ``cell:ref`` link to a ``cell:def`` with the same name + + ``cell:title`` + + - ``:cell:title:`$nex``` -> :cell:title:`$nex` + + + ``cell:prop`` link to a ``cell:defprop`` of the same name From e5b9401bf35a26ba1a00426875a18f2588ef2b99 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Mon, 17 Nov 2025 12:44:41 +1300 Subject: [PATCH 08/22] documenting.rst: Command line rendering --- .../extending_yosys/documenting.rst | 60 +++++++++++-------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index c0f51f637..7038b61d9 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -66,22 +66,22 @@ indicate mutually exclusive arguments. The ``Pass::help()`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This is the original way to provide help text, and as of this writing is still -the most common. The ``log()`` function should be called directly to print and -format the help text, and each line should be limited to 80 (printed) -characters. While it is possible to provide arbitrary formatting, it is -preferred to follow the guidelines here to maintain consistency with other +Overriding this method is the original way to provide help text, and as of this +writing is still the most common. The ``log()`` function should be called +directly to print and format the help text, and each line should be limited to +80 (printed) characters. While it is possible to provide arbitrary formatting, +it is preferred to follow the guidelines here to maintain consistency with other passes and to assist in correct parsing and formatting during RST generation (i.e. these docs). -The first line should always be a blank line, followed by the primary usage -signature for the command. Each usage signature should be indented with 4 -spaces, and followed by a blank line. Each option or flag should start on a new -line indented with 4 spaces, followed by a description of the option which is -indented by a further 4 spaces, and then a blank line. Option descriptions -typically start with lower case, and may forgo a trailing period (``.``). Where -multiple options share a description the blank line between options should be -omitted. +The first and last lines should always be blank (usually ``log("\n");``), +followed by the primary usage signature for the command. Each usage signature +should be indented with 4 spaces, and followed by a blank line. Each option or +flag should start on a new line indented with 4 spaces, followed by a +description of the option which is indented by a further 4 spaces, and then a +blank line. Option descriptions typically start with lower case, and may forgo a +trailing period (``.``). Where multiple options share a description the blank +line between options should be omitted. .. note:: @@ -203,24 +203,34 @@ highlighting). Command line rendering ~~~~~~~~~~~~~~~~~~~~~~ -- if ``Pass::formatted_help()`` returns true, will call - ``PrettyHelp::log_help()`` +Rendering text for the command line is done by the ``Pass::help`` method. When +this method is not overridden, the default behavior is to call +``Pass::formatted_help()``. If this method is also left unimplemented, or the +return value is explicitly false, then a default message about missing help text +for the command is displayed. Returning true, however, will then call +``PrettyHelp::log_help()`` to convert the formatted help content into plain +text. - + traverse over the children of the root node and render as plain text - + effectively the reverse of converting unformatted ``Pass::help()`` text - + lines are broken at 80 characters while maintaining indentation (controlled - by ``MAX_LINE_LEN`` in :file:`kernel/log_help.cc`) - + each line is broken into words separated by spaces, if a given word starts - and ends with backticks they will be stripped +Rendering rich help text as plain text is done by traversing over all the +``ContentListing`` nodes and printing the body text. ``usage`` nodes are +preceded by an empty line and indented one level (4 spaces). ``option`` nodes +are also indented one level, while their children are indented an extra level (8 +spaces). Each section of body text is broken into words separated by spaces. +If a word would cause the line to exceed 80 characters (controlled by +``MAX_LINE_LEN`` in :file:`kernel/log_help.cc`), then the word will instead be +placed on a new line, with the same level of indentation. -- if it returns false it will call ``Pass::help()`` which should call ``log()`` - directly to print and format help text +Special handling is included for words that begin and end with a backtick +(`````) so that these are stripped when printing to the command line. Compare +:ref:`chformal_help` below with the :ref:`chformal autocmd` above. The content +is still the same, but for the command line it uses a fixed width. - + if ``Pass::help()`` is not overridden then a default message about missing - help will be displayed +.. todo:: spaces in backticks (``assert(...)`` vs ````assert(s_eventually ...)````) .. literalinclude:: /generated/chformal.log :lines: 2- + :name: chformal_help + :caption: Command line output for `help chformal` Cell help From 5896579066b58a234e62895917746b2a707e76ad Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Mon, 17 Nov 2025 12:47:50 +1300 Subject: [PATCH 09/22] log_help.cc: Better line splitting If a word by itself meets the max line length it should only drop to a new line if the current line has content (would previously force an empty line before it). Include the length of indent when adding a line break (was previously only accounted for on the first line). --- kernel/log_help.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/log_help.cc b/kernel/log_help.cc index cb22d236e..3c22a80ac 100644 --- a/kernel/log_help.cc +++ b/kernel/log_help.cc @@ -84,6 +84,7 @@ void log_pass_str(const std::string &pass_str, std::string indent_str, bool lead std::istringstream iss(pass_str); if (leading_newline) log("\n"); + bool partial_line = false; for (std::string line; std::getline(iss, line);) { log("%s", indent_str); auto curr_len = indent_str.length(); @@ -91,13 +92,15 @@ void log_pass_str(const std::string &pass_str, std::string indent_str, bool lead for (std::string word; std::getline(lss, word, ' ');) { while (word[0] == '`' && word.back() == '`') word = word.substr(1, word.length()-2); - if (curr_len + word.length() >= MAX_LINE_LEN-1) { - curr_len = 0; + if (partial_line && curr_len + word.length() >= MAX_LINE_LEN-1) { + curr_len = indent_str.length(); log("\n%s", indent_str); + partial_line = false; } if (word.length()) { log("%s ", word); curr_len += word.length() + 1; + partial_line = true; } } log("\n"); From 4eff547f6fa869fd7cf2048335b078ffd3c87d1a Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:49:07 +1300 Subject: [PATCH 10/22] documenting.rst: Dumping to JSON Split into cmd/cell sections. cmd json is in paragraphs, cell json is bullet points (and pending example). Add (bullet point) section for warning flags. Add note to `Pass::help()` about how each line should be a single `log()` call. --- .../extending_yosys/documenting.rst | 153 +++++++++++++----- 1 file changed, 112 insertions(+), 41 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index 7038b61d9..08218b1de 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -46,7 +46,7 @@ from the command line). The general format is to show each usage signature (how the command is called), followed by a paragraph describing what the pass does, and a list of options or flags available. Additional arguments in the signature or option may use square brackets (``[]``) to indicate optional parts, and angle -brackets (``<>``) for required parts. The pipe character ``|`` may be used to +brackets (``<>``) for required parts. The pipe character (``|``) may be used to indicate mutually exclusive arguments. .. note:: @@ -63,6 +63,19 @@ indicate mutually exclusive arguments. .. autocmd:: chformal :noindex: +Warning flags +~~~~~~~~~~~~~ + +- flags set during pass constructor +- adds warnings to end of help output +- usually used by commands not intended for general use +- ``Pass::experimental()`` for experimental commands that may not be stable or + reliable +- ``Pass::internal()`` for commands aimed at developers rather than users + + + most of which end up in :doc:`/cmd/index_internal` + + these are often commands used for testing Yosys + The ``Pass::help()`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -74,18 +87,24 @@ it is preferred to follow the guidelines here to maintain consistency with other passes and to assist in correct parsing and formatting during RST generation (i.e. these docs). -The first and last lines should always be blank (usually ``log("\n");``), -followed by the primary usage signature for the command. Each usage signature -should be indented with 4 spaces, and followed by a blank line. Each option or -flag should start on a new line indented with 4 spaces, followed by a -description of the option which is indented by a further 4 spaces, and then a -blank line. Option descriptions typically start with lower case, and may forgo a -trailing period (``.``). Where multiple options share a description the blank -line between options should be omitted. +.. note:: + + It is good practice in the ``Pass::help`` method for each call to ``log()`` to + correspond to a single line, containing exactly one ``\n`` (at the end). This + allows the appearance in source to match the appearance in the terminal. + +The first and last lines should always be empty, followed by the primary usage +signature for the command. Each usage signature should be indented with 4 +spaces, and followed by an empty line. Each option or flag should start on a +new line indented with 4 spaces, followed by a description of the option which +is indented by a further 4 spaces, and then an empty line. Option descriptions +typically start with lower case, and may forgo a trailing period (``.``). Where +multiple options share a description the empty line between options should be +omitted. .. note:: - `Dumping to json`_ has more on how formatting in ``help()`` gets parsed. + `Commands JSON`_ has more on how formatting in ``help()`` gets parsed. The ``Pass::formatted_help()`` method @@ -101,7 +120,7 @@ PrettyHelp::get_current();``. The method finishes by returning a boolean value. Setting a command group ^^^^^^^^^^^^^^^^^^^^^^^ -Command groups are used when `Dumping to json`_, so that related +Command groups are used when `dumping to JSON`_, so that related commands can be presented together in documentation. For example, all of the formal commands (which `chformal` is one of) are listed under :doc:`/cmd/index_formal`, by using the ``autocmdgroup`` directive in @@ -157,6 +176,7 @@ at the source code for `chformal`. :start-at: bool formatted_help() :end-before: void execute :caption: ``ChformalPass::formatted_help()`` from :file:`passes/cmds/chformal.cc` + :name: chformal_source We can see that each of the ``ContentListing`` methods have the body of the new node as the first argument. For a ``usage`` node, this is how to call the @@ -199,6 +219,10 @@ highlighting). text/options specific to a given usage of the command. In the web documentation any content added in this way will be indented under the usage signature. +.. + When :makevar:`ENABLE_HELP_SOURCE` is set, each ``ContentListing`` node also + stores file path and line number of its source location. But I think this might + only be used when raising errors/warnings during ``autocmd``. Command line rendering ~~~~~~~~~~~~~~~~~~~~~~ @@ -379,49 +403,96 @@ v2 (more expressive) more text -Dumping to json +Dumping to JSON --------------- -- `help -dump-cmds-json cmds.json` +Once compiled, Yosys is able to dump both the internal command and cell +libraries to a machine-readable JSON file. Primarily intended for building this +documentation (more on that in the next section), this feature is not advertised +within Yosys itself, and can be done with `help -dump-cmds-json ` and +`help -dump-cells-json ` respectively. - + generates a ``ContentListing`` for each command registered in Yosys - + tries to parse unformatted ``Pass::help()`` output if - ``Pass::formatted_help()`` is unimplemented or returns false +Both JSON files are formatted very similarly, containing a single object. The +object has a ``version`` field which disambiguates between the two, a +``generator`` field which contains the Yosys version string used, a ``groups`` +object which maps each group to the list of commands/cells in that group, and +finally a ``cmds`` or ``cells`` object which maps each command/cell to its help +content. - * if a line starts with four spaces followed by the name of the command then - a space, it is parsed as a signature (usage node) - * if a line is indented and starts with a dash (``-``), it is parsed as an - option - * anything else is parsed as a codeblock and added to either the root node - or the current option depending on the indentation +Commands JSON +~~~~~~~~~~~~~ - + dictionary of command name to ``ContentListing`` - - * uses ``ContentListing::to_json()`` recursively for each node in root - * root node used for source location of class definition - * includes flags set during pass constructor (e.g. ``experimental_flag`` set - by ``Pass::experimental()``) - * also title (``short_help`` argument in ``Pass::Pass``), group, and class - name - - + dictionary of group name to list of commands in that group - -- used by sphinx autodoc to generate help content +Lets take a look at :ref:`chformal_json` as an example. We can see the bulk of +the object is taken up by the ``content`` field, which contains all the +``ContentListing`` nodes we added in :ref:`the formatted_help method for +chformal `, maintaining the structure of those nodes. The +command's ``short_help`` is given in the ``title`` field, with other fields for +the `Warning flags`_, source location, source function, and corresponding group +(either implicit or explicit). .. literalinclude:: /generated/cmds.json :language: json :start-at: "chformal": { :end-before: "chparam": { :caption: `chformal` in generated :file:`cmds.json` + :name: chformal_json -.. note:: Synthesis command scripts are special cased +Every command registered in Yosys (including those from currently installed +plugins) has a corresponding object in the JSON dump. For commands where +``Pass::formatted_help()`` is unimplemented or returns false, ``ContentListing`` +nodes will be generated by parsing the unformatted ``Pass::help()`` output. This +is largely the same as `Command line rendering`_ but in reverse, with a few +simple rules to try convert between raw text and the different node types. - If the final block of help output starts with the string ``"The following - commands are executed by this synthesis command:\n"``, then the rest of the - code block is formatted as ``yoscrypt`` (e.g. `synth_ice40`). The caveat - here is that if the ``script()`` calls ``run()`` on any commands *prior* to - the first ``check_label`` then the auto detection will break and revert to - unformatted code (e.g. `synth_fabulous`). +To be parsed as a ``usage`` node, the current line: + + must start with the name of the command (case sensitive), followed by a + space or a new line; + + may have up to four characters of whitespace as indentation; + + must be the first non-empty line, preceded by two empty lines, or + immediately following another usage signature with the same indentation. + +Any lines immediately after a usage signature which is indented more than the +signature will be appended to the usage signature. This allows for breaking +arguments across lines in the terminal output while still producing a single +``usage`` node. + +.. code-block:: cpp + :caption: Example code for a command with multiple usage signatures + + log("\n"); + log(" command\n"); + log(" command -argument\n"); + log(" -another argument\n"); + log("\n"); + log("\n"); + log("command description.\n"); // not a signature because it is dedented + log("\n"); + log("\n"); + log(" command -different argument\n"); + log("\n"); + +If a line is indented and starts with a dash (``-``), and does not immediately +follow a usage signature, it is parsed as an ``option`` node. Anything else is +parsed as a ``codeblock`` and added to either the root node or the current +option depending on the indentation. This allows yosys script syntax +highlighting for (most) options, while still respecting help content which +relies on the fixed-width rendering. + +To enable syntax highlighting in synthesis command scripts, if the final block +of help output starts with the string ``"The following commands are executed by +this synthesis command:\n"``, then the rest of the code block is formatted as +``yoscrypt`` (e.g. `synth_ice40`). The caveat here is that if the ``script()`` +calls ``run()`` on any commands *prior* to the first ``check_label`` then the +auto detection will break and revert to unformatted code (e.g. +`synth_fabulous`). + + +Cells JSON +~~~~~~~~~~ + +- effectively (if not literally) the ``SimHelper`` struct formatted as JSON + +.. todo:: get an example here (`$nex`\ ?) Cells and commands in Sphinx From f2a80e4d7f53f89ca735a922b049a3cb2ff7de16 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:51:01 +1300 Subject: [PATCH 11/22] register.cc: Tidying Use `yosys_maybe_version()` in `dump_cmds_json()`. Replace `&it` with `&[name, pass]`. Add a space when folding indented lines into usage signature. --- kernel/register.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/register.cc b/kernel/register.cc index bd12dcc38..de815d532 100644 --- a/kernel/register.cc +++ b/kernel/register.cc @@ -739,16 +739,14 @@ struct HelpPass : public Pass { // init json json.begin_object(); json.entry("version", "Yosys command reference"); - json.entry("generator", yosys_version_str); + json.entry("generator", yosys_maybe_version()); bool raise_error = false; std::map> groups; json.name("cmds"); json.begin_object(); // iterate over commands - for (auto &it : pass_register) { - auto name = it.first; - auto pass = it.second; + for (auto &[name, pass] : pass_register) { auto title = pass->short_help; auto cmd_help = PrettyHelp(); @@ -863,7 +861,7 @@ struct HelpPass : public Pass { if (current_buffer.empty()) current_buffer = stripped_line; else if (current_state == PUState_signature && IsIndent) - current_buffer += stripped_line; + current_buffer += " " + stripped_line; else if (current_state == PUState_none) { current_buffer += (blank_lines > 0 ? "\n\n" : "\n") + line; } else From 657b0bd92b6090b757086f203ec32967e2806c95 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Tue, 18 Nov 2025 12:20:33 +1300 Subject: [PATCH 12/22] documenting.rst: literalinclude cell doc examples Add a sed command to the (top level) makefile for extract comment block for the specified cell. Works with both simlib.v and simcells.v (by abusing `%` pattern matching slightly to disambiguate which to search). --- Makefile | 22 +++++-- .../extending_yosys/documenting.rst | 57 +++++++++---------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index dd7a8c022..8f61a1a8d 100644 --- a/Makefile +++ b/Makefile @@ -1082,7 +1082,7 @@ docs/source/generated/functional/rosette.diff: backends/functional/smtlib.cc bac $(Q) mkdir -p $(@D) $(Q) diff -U 20 $^ > $@ || exit 0 -PHONY: docs/gen/functional_ir +.PHONY: docs/gen/functional_ir docs/gen/functional_ir: docs/source/generated/functional/smtlib.cc docs/source/generated/functional/rosette.diff docs/source/generated/%.log: docs/source/generated $(TARGETS) $(EXTRA_TARGETS) @@ -1091,10 +1091,24 @@ docs/source/generated/%.log: docs/source/generated $(TARGETS) $(EXTRA_TARGETS) docs/source/generated/chformal.cc: passes/cmds/chformal.cc docs/source/generated $(Q) cp $< $@ -PHONY: docs/gen/chformal +.PHONY: docs/gen/chformal docs/gen/chformal: docs/source/generated/chformal.log docs/source/generated/chformal.cc -PHONY: docs/gen docs/usage docs/reqs +# e.g. simlib.nex.v -> extract $nex from simlib.v +# sed command adds all non-empty lines to the hold space +# when an empty line is reached, the hold space is moved to the pattern space +# if it includes the desired module, print it +# this gives us the raw comment block immediately before the cell +docs/source/generated/%.v: $(addprefix techlibs/common/,simlib.v simcells.v) + $(Q) mkdir -p $(@D) + $(Q) sed --posix -n -e '/./{H;d} ; x' \ + -e "/module .\$$$(lastword $(subst ., ,$*))/p" \ + techlibs/common/$(basename $*).v >> $@ + +.PHONY: docs/gen/raw_cells +docs/gen/raw_cells: $(addprefix docs/source/generated/,simlib.nex.v simcells._NOT_.v) + +.PHONY: docs/gen docs/usage docs/reqs docs/gen: $(TARGETS) $(Q) $(MAKE) -C docs gen @@ -1129,7 +1143,7 @@ docs/reqs: $(Q) $(MAKE) -C docs reqs .PHONY: docs/prep -docs/prep: docs/source/generated/cells.json docs/source/generated/cmds.json docs/gen docs/usage docs/gen/functional_ir docs/gen/chformal +docs/prep: docs/source/generated/cells.json docs/source/generated/cmds.json docs/gen docs/usage docs/gen/functional_ir docs/gen/chformal docs/gen/raw_cells DOC_TARGET ?= html docs: docs/prep diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index 08218b1de..c64e57f97 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -239,13 +239,17 @@ Rendering rich help text as plain text is done by traversing over all the ``ContentListing`` nodes and printing the body text. ``usage`` nodes are preceded by an empty line and indented one level (4 spaces). ``option`` nodes are also indented one level, while their children are indented an extra level (8 -spaces). Each section of body text is broken into words separated by spaces. -If a word would cause the line to exceed 80 characters (controlled by -``MAX_LINE_LEN`` in :file:`kernel/log_help.cc`), then the word will instead be -placed on a new line, with the same level of indentation. +spaces). Any ``codeblock`` nodes are rendered as-is at the current indentation, +with no further formatting applied. -Special handling is included for words that begin and end with a backtick -(`````) so that these are stripped when printing to the command line. Compare +.. TODO:: are codeblocks actually rendered as-is? They definitely should be + +``paragraph`` nodes are broken into words separated by spaces, and each word is +printed. If a word would cause the current line to exceed 80 characters +(controlled by ``MAX_LINE_LEN`` in :file:`kernel/log_help.cc`), then the word +will instead be placed on a new line with the same level of indentation. Special +handling is included for words that begin and end with a backtick (`````) so +that these are stripped when printing to the command line. Compare :ref:`chformal_help` below with the :ref:`chformal autocmd` above. The content is still the same, but for the command line it uses a fixed width. @@ -330,19 +334,11 @@ v1 (default) code block - e.g. `$_NOT_`: -.. code-block:: verilog +.. literalinclude:: /generated/simcells._NOT_.v + :language: verilog + :start-at: //- + :end-at: module \$_NOT_ - //- - //- $_NOT_ (A, Y) - //* group comb_simple - //- - //- An inverter gate. - //- - //- Truth table: A | Y - //- ---+--- - //- 0 | 1 - //- 1 | 0 - //- v2 (more expressive) ~~~~~~~~~~~~~~~~~~~~ @@ -364,16 +360,8 @@ v2 (more expressive) - can link to commands or passes using backticks (`````) - e.g. `$nex`: -.. code-block:: verilog - - //* ver 2 - //* title Case inequality - //* group binary - //* tags x-aware - //- This corresponds to the Verilog '!==' operator. - //- - //- Refer to `$eqx` for more details. - //- +.. literalinclude:: /generated/simlib.nex.v + :language: verilog - code blocks can be included as following: @@ -490,9 +478,18 @@ auto detection will break and revert to unformatted code (e.g. Cells JSON ~~~~~~~~~~ -- effectively (if not literally) the ``SimHelper`` struct formatted as JSON +- effectively the ``SimHelper`` struct formatted as JSON (drops ``ver``, copies + tags to ``properties``) +- plus additional fields from the ``CellType`` -.. todo:: get an example here (`$nex`\ ?) + + inputs, outputs, property flags + +.. literalinclude:: /generated/cells.json + :language: json + :start-at: "$nex": { + :end-at: }, + :caption: `$nex` in generated :file:`cells.json` + :name: nex_json Cells and commands in Sphinx From 695bace331d149d726b031faf5595547ee7620da Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:12:27 +1300 Subject: [PATCH 13/22] log_help.cc: Don't re-flow codeblocks Move to a (single) `log_content_body()` instead of the dual `log_pass_str()` (nothing used the string-based indent arg), and pass the `ContentListing` directly; printing codeblocks verbatim (with indent). Refactor (non-codeblock) line splitting slightly so that it can print up to and including 80 characters before the line break. Add comments. --- kernel/log_help.cc | 67 ++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/kernel/log_help.cc b/kernel/log_help.cc index 3c22a80ac..e8f7844db 100644 --- a/kernel/log_help.cc +++ b/kernel/log_help.cc @@ -78,38 +78,53 @@ ContentListing* ContentListing::open_option(const string &text, } #define MAX_LINE_LEN 80 -void log_pass_str(const std::string &pass_str, std::string indent_str, bool leading_newline=false) { - if (pass_str.empty()) +void log_content_body(const ContentListing &content, int indent=0, bool leading_newline=false) { + // skip empty nodes + if (content.body.empty()) return; - std::istringstream iss(pass_str); + if (leading_newline) log("\n"); + + // iterate over lines in content + std::string indent_str(indent*4, ' '); + std::istringstream iss(content.body); bool partial_line = false; for (std::string line; std::getline(iss, line);) { log("%s", indent_str); - auto curr_len = indent_str.length(); - std::istringstream lss(line); - for (std::string word; std::getline(lss, word, ' ');) { - while (word[0] == '`' && word.back() == '`') - word = word.substr(1, word.length()-2); - if (partial_line && curr_len + word.length() >= MAX_LINE_LEN-1) { - curr_len = indent_str.length(); - log("\n%s", indent_str); - partial_line = false; - } - if (word.length()) { - log("%s ", word); - curr_len += word.length() + 1; - partial_line = true; + if (content.type == "code") { + // code blocks are verbatim + log("%s", line); + } else { + // iterate over words and break at max line length + auto curr_len = indent_str.length(); + std::istringstream lss(line); + for (std::string word; std::getline(lss, word, ' ');) { + // remove inline rst formatting + while (word[0] == '`' && word.back() == '`') + word = word.substr(1, word.length()-2); + + // if the current line is not empty, break before going over max + if (partial_line && (curr_len + word.length()) >= MAX_LINE_LEN) { + curr_len = indent_str.length(); + log("\n%s", indent_str); + partial_line = false; + } + + // print non-empty words + if (word.length()) { + if (partial_line) + // add space after prior word + word.insert(0, " "); + log("%s", word); + curr_len += word.length(); + partial_line = true; + } } } log("\n"); } } -void log_pass_str(const std::string &pass_str, int indent=0, bool leading_newline=false) { - std::string indent_str(indent*4, ' '); - log_pass_str(pass_str, indent_str, leading_newline); -} PrettyHelp *current_help = nullptr; @@ -137,16 +152,16 @@ void PrettyHelp::log_help() const { for (auto &content : _root_listing) { if (content.type.compare("usage") == 0) { - log_pass_str(content.body, 1, true); + log_content_body(content, 1, true); log("\n"); } else if (content.type.compare("option") == 0) { - log_pass_str(content.body, 1); - for (auto text : content) { - log_pass_str(text.body, 2); + log_content_body(content, 1); + for (auto &child : content) { + log_content_body(child, 2); log("\n"); } } else { - log_pass_str(content.body, 0); + log_content_body(content, 0); log("\n"); } } From 784c73ad389187879d6ba141e372885ecc4bf433 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:33:13 +1300 Subject: [PATCH 14/22] documenting.rst: Warning flags Add `test_generic.cc` to docs gen, and rename `docs/gen/chformal` to the more generic `docs/gen/raw_commands` (in line with `docs/gen/raw_cells`). --- Makefile | 6 +-- .../extending_yosys/documenting.rst | 53 +++++++++++++++---- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 8f61a1a8d..24e9fdba1 100644 --- a/Makefile +++ b/Makefile @@ -1091,8 +1091,8 @@ docs/source/generated/%.log: docs/source/generated $(TARGETS) $(EXTRA_TARGETS) docs/source/generated/chformal.cc: passes/cmds/chformal.cc docs/source/generated $(Q) cp $< $@ -.PHONY: docs/gen/chformal -docs/gen/chformal: docs/source/generated/chformal.log docs/source/generated/chformal.cc +.PHONY: docs/gen/raw_commands +docs/gen/raw_commands: docs/source/generated/chformal.log docs/source/generated/chformal.cc docs/source/generated/functional/test_generic.cc # e.g. simlib.nex.v -> extract $nex from simlib.v # sed command adds all non-empty lines to the hold space @@ -1143,7 +1143,7 @@ docs/reqs: $(Q) $(MAKE) -C docs reqs .PHONY: docs/prep -docs/prep: docs/source/generated/cells.json docs/source/generated/cmds.json docs/gen docs/usage docs/gen/functional_ir docs/gen/chformal docs/gen/raw_cells +docs/prep: docs/source/generated/cells.json docs/source/generated/cmds.json docs/gen docs/usage docs/gen/functional_ir docs/gen/raw_commands docs/gen/raw_cells DOC_TARGET ?= html docs: docs/prep diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index c64e57f97..8d8a5d138 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -66,15 +66,43 @@ indicate mutually exclusive arguments. Warning flags ~~~~~~~~~~~~~ -- flags set during pass constructor -- adds warnings to end of help output -- usually used by commands not intended for general use -- ``Pass::experimental()`` for experimental commands that may not be stable or - reliable -- ``Pass::internal()`` for commands aimed at developers rather than users +In order to support commands which are not intended for general use, a number of +warning flags are provided to the ``Pass`` class. Take the +:ref:`internal_flag_example` as an example. In the body of the constructor, we +call ``Pass::internal()`` to set the warning flag that this is an internal; i.e. +one aimed at Yosys *developers* rather than users. Commands with the +``internal`` flag are often used for testing Yosys, and expose functionality +that would normally be abstracted. Setting this flag also ensures that commands +will be included in :doc:`/cmd/index_internal`. + +.. literalinclude:: /generated/functional/test_generic.cc + :language: cpp + :start-at: FunctionalTestGeneric() + :end-at: } + :dedent: + :caption: `test_generic` pass constructor + :name: internal_flag_example + +The other warning flag available is ``Pass::experimental()``, also to be called +during the constructor. This should used for experimental commands that may be +unstable, unreliable, incomplete, and/or subject to change. Experimental passes +also typically have the text ``(experimental)`` at the start of their +``short_help``, but this is not always the case. + +.. todo:: should the experimental flag add ``(experimental)`` automatically? + +In both cases, commands with these flags set will print additional warning text +in the help output. Calling commands with the ``experimental`` flag set, will +also call ``log_experimental()`` with the name of the pass, providing an +additional warning any time the pass is used. + +.. note:: + + When testing the handling of expected error/warning messages with e.g. + `logger`, it is possible to disable the warnings for a given experimental + feature. This can be done by calling Yosys with ``--experimental + ``, where ```` is the name of the experimental pass. - + most of which end up in :doc:`/cmd/index_internal` - + these are often commands used for testing Yosys The ``Pass::help()`` method ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -235,6 +263,13 @@ for the command is displayed. Returning true, however, will then call ``PrettyHelp::log_help()`` to convert the formatted help content into plain text. +.. note:: + + Regardless of which help method is used, any `warning flags`_ set on the pass + will display a message to warn the user. These are regular messages, using + ``log()`` rather than ``log_warning()``, meaning (for example) they will + be suppressed by the ``-q`` command line option. + Rendering rich help text as plain text is done by traversing over all the ``ContentListing`` nodes and printing the body text. ``usage`` nodes are preceded by an empty line and indented one level (4 spaces). ``option`` nodes @@ -242,8 +277,6 @@ are also indented one level, while their children are indented an extra level (8 spaces). Any ``codeblock`` nodes are rendered as-is at the current indentation, with no further formatting applied. -.. TODO:: are codeblocks actually rendered as-is? They definitely should be - ``paragraph`` nodes are broken into words separated by spaces, and each word is printed. If a word would cause the current line to exceed 80 characters (controlled by ``MAX_LINE_LEN`` in :file:`kernel/log_help.cc`), then the word From bb8ef02e8572d4e742c35b36fb6fcc2df4fe4dc9 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:29:13 +1300 Subject: [PATCH 15/22] docs: Intro to cell help Allow autoref to render `help *` as yoscrypt instead of trying to link when `*` starts with `<` or `-`, as in `help ` and `help -all`. Fix autoref `help *` being linkable when there is a line break in the middle. --- .../extending_yosys/documenting.rst | 52 +++++++++++-------- docs/util/custom_directives.py | 4 +- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index 8d8a5d138..e793de968 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -41,7 +41,7 @@ hovering over links to commands in the documentation, and in section headings like :ref:`chformal autocmd`. The next section shows the complete help text for the `chformal` command. This -can be displayed locally by using ``help `` (or ``yosys -h `` +can be displayed locally by using `help ` (or ``yosys -h `` from the command line). The general format is to show each usage signature (how the command is called), followed by a paragraph describing what the pass does, and a list of options or flags available. Additional arguments in the signature @@ -297,31 +297,35 @@ is still the same, but for the command line it uses a fixed width. Cell help --------- -- :file:`techlibs/common/simcells.v` and :file:`techlibs/common/simlib.v` -- parsed by :file:`techlibs/common/cellhelp.py` +Unlike commands, cell help text is generated at compile time, and is not +affected by platform or compile flags. This also means that it is not possible +to provide help content for custom cell types in plugins or technology +libraries. - + unlike commands, cell help text is generated at compile time - + only formatting occurs at run time - + no support for custom cells in plugins +Two verilog simulation libraries provide models for all built-in cell types. +These are defined in :file:`techlibs/common/simcells.v` (for +:doc:`/cell/index_gate`) and :file:`techlibs/common/simlib.v` (for +:doc:`/cell/index_word`). Each model is preceded by a structured comment block, +formatted as either :ref:`v1` or :ref:`v2`. These comment blocks are processed +by a python script, :file:`techlibs/common/cellhelp.py`, to generate the help +content used in :file:`kernel/register.cc`. -- comments preceding cell type converted to a ``SimHelper`` struct, defined in - :file:`kernel/register.cc` -- ``#include``\ d in :file:`kernel/register.cc`, creating a map of cell types to - their ``SimHelper`` struct. +.. note:: -- ``help -cells`` + Each verilog module (and its comment block) is parsed into a C++ ``dict``, + mapping the cell type (the name of the verilog module) to a ``SimHelper`` + struct in :file:`kernel/register.cc` with ``#include``\ s. Calling `help + ` then retrieves the corresponding ``SimHelper`` and displays the + help text contained. - - lists all cells with their input/output ports - - again an unlisted :ref:`cell index ` +Calling `help -cells` will list all built-in cell types with their input/output +ports. There is again an unlisted :ref:`cell index ` which shows all +cell types with their title. Unlike commands, providing a title is optional, so +most just use the name of the cell (qualified with the containing group). It is +also possible to display the verilog simulation model by calling `help ++`. -- ``help `` - - - prints help message for ```` - - constructed from ``SimHelper`` content depending on version - -- ``help +`` - - - prints verilog code for ```` +.. _v1: v1 (default) ~~~~~~~~~~~~ @@ -359,8 +363,9 @@ v1 (default) and a space - description should have a trailing empty line - - line breaks are preserved in `help` calls but will be rendered as text in - sphinx docs, with empty lines being required between paragraphs + - lines should be limited to 80 characters (84 including ``//-``), and are + rendered to the terminal as-is + - web docs will render as text, with empty lines used to separate paragraphs - cells in :file:`techlibs/common/simcells.v` can have optional truth table at the end of the cell description which is rendered in sphinx docs as a literal @@ -372,6 +377,7 @@ v1 (default) :start-at: //- :end-at: module \$_NOT_ +.. _v2: v2 (more expressive) ~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/util/custom_directives.py b/docs/util/custom_directives.py index b90584aa7..dc6dd29a4 100644 --- a/docs/util/custom_directives.py +++ b/docs/util/custom_directives.py @@ -693,8 +693,8 @@ class CellDomain(CommandDomain): def autoref(name, rawtext: str, text: str, lineno, inliner: Inliner, options=None, content=None): - words = text.split(' ') - if len(words) == 2 and words[0] == "help": + words = text.split() + if len(words) == 2 and words[0] == "help" and words[1][0] not in ['<', '-']: IsLinkable = True thing = words[1] else: From fc8800226691712be1de6ce9168073c1343f2f27 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:29:44 +1300 Subject: [PATCH 16/22] cellhelp.py: Unify manual signature handling Both files strip it, but previously the exact check differed. It is also now safer, checking the length before trying to access elements by index. --- techlibs/common/cellhelp.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/techlibs/common/cellhelp.py b/techlibs/common/cellhelp.py index 741558730..d61a24111 100644 --- a/techlibs/common/cellhelp.py +++ b/techlibs/common/cellhelp.py @@ -38,9 +38,6 @@ class SimHelper: return val def simcells_reparse(cell: SimHelper): - # cut manual signature - cell.desc = cell.desc[3:] - # code-block truth table new_desc = [] indent = "" @@ -58,6 +55,7 @@ def simcells_reparse(cell: SimHelper): simHelper = SimHelper() for line in fileinput.input(): + short_filename = Path(fileinput.filename()).name line = line.rstrip() # special comments if line.startswith("//-"): @@ -71,7 +69,6 @@ for line in fileinput.input(): clean_line = line[7:].replace("\\", "").replace(";", "") simHelper.name, simHelper.ports = clean_line.split(maxsplit=1) simHelper.code = [] - short_filename = Path(fileinput.filename()).name simHelper.source = f'{short_filename}:{fileinput.filelineno()}' elif not line.startswith("endmodule"): line = " " + line @@ -81,14 +78,14 @@ for line in fileinput.input(): # no module definition, ignore line pass if line.startswith("endmodule"): - short_filename = Path(fileinput.filename()).name - if simHelper.ver == "1" and short_filename == "simcells.v": - # default simcells parsing - simcells_reparse(simHelper) + if simHelper.ver == "1": + # cut manual signature + if len(simHelper.desc) > 3 and simHelper.desc[0] == "" and simHelper.desc[2] == "" and simHelper.desc[1].startswith(" "): + simHelper.desc = simHelper.desc[3:] - # check help - if simHelper.desc and simHelper.ver == "1" and short_filename == "simlib.v" and simHelper.desc[1].startswith(' '): - simHelper.desc.pop(1) + # default simcells parsing + if short_filename == "simcells.v": + simcells_reparse(simHelper) # check group assert simHelper.group, f"techlibs/common/{simHelper.source}: {simHelper.name} cell missing group" From da65409d69ed145c3e06ce8ed66f6f942fe1dce6 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:30:54 +1300 Subject: [PATCH 17/22] documenting.rst: Cell help v1 and v2 Also a (commented) mention of v2a for the extra curious. --- .../extending_yosys/documenting.rst | 138 ++++++++++-------- 1 file changed, 79 insertions(+), 59 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index e793de968..1b54070d4 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -320,89 +320,104 @@ content used in :file:`kernel/register.cc`. Calling `help -cells` will list all built-in cell types with their input/output ports. There is again an unlisted :ref:`cell index ` which shows all -cell types with their title. Unlike commands, providing a title is optional, so -most just use the name of the cell (qualified with the containing group). It is -also possible to display the verilog simulation model by calling `help -+`. +cell types with their title. Unlike commands, providing a title is optional, +and only available with `v2`_ formatting, so most just use the name of the cell +(qualified with the containing group). It is also possible to display the +verilog simulation model by calling `help +`. .. _v1: v1 (default) ~~~~~~~~~~~~ -- Legacy format -- Expects formatting as follows: +As mentioned previously, the verilog simulation models are preceded by a +structured comment block. Each line starting with ``//-`` is added to the +description of the next verilog module. Non-empty lines must have a space after +the dash before text, and should be limited to 80 characters (84 including the +``//-``). The description is rendered to the terminal as-is when calling `help +`, while the web docs will render it as text, with empty lines being +used to separate paragraphs. -.. code-block:: verilog +.. + Descriptions can extend into the verilog module itself, including *all* comment + lines that start with a dash prior to the ``endmodule``. However, everything in + the ``module .. endmodule`` block is considered source code, so this is not + recommended. - //- - //- $ () - //* group - //- - //- - //- - module \$ (); - // ... - endmodule +.. note:: -- ``//* group`` line is generally after the cell signature, but can appear - anywhere in the comment block + Most of the legacy cell descriptions include a signature line (``//- + $ ()``). More recent versions of the help generation will + automatically produce this signature from the verilog declaration, making + this an optional inclusion. Note that if a signature line *is* included, it + *must* start with at least 4 spaces (not tabs), and include one empty line + (``//-``) before and after. - - determines where the cell is included in sphinx - - check :file:`docs/source/cell` for current groups - - a missing group will raise an error - - assigning an unused group will result in the cell not being included in the - sphinx docs +Each cell type must also be assigned a group, failing to do so will produce an +error. This can be done by adding ``//* group `` anywhere in the +comment block. As with commands, the group determines where the cell appears in +the Sphinx documentation, but does not otherwise impact the output of `help`. As +with commands, there is no warning produced if cells are assigned a group which +is not used in the documentation. Make sure to check :file:`docs/source/cell` +for the groups currently available. -- the port signature line (``//- $ ()``) must start with (at - least) 4 spaces (not tabs) - - - the empty lines (``//-``) before/after the signature are required - -- cell description can be multiple lines, but each line must start with ``//-`` - and a space - - - description should have a trailing empty line - - lines should be limited to 80 characters (84 including ``//-``), and are - rendered to the terminal as-is - - web docs will render as text, with empty lines used to separate paragraphs - -- cells in :file:`techlibs/common/simcells.v` can have optional truth table at - the end of the cell description which is rendered in sphinx docs as a literal - code block -- e.g. `$_NOT_`: +For the cell models in :file:`techlibs/common/simcells.v`, it is possible to +provide a truth table at the end of the cell description which is rendered in +sphinx docs as a literal code block. We can look at the :ref:`NOT_module` to +see this in action. .. literalinclude:: /generated/simcells._NOT_.v :language: verilog :start-at: //- :end-at: module \$_NOT_ + :name: NOT_module + :caption: `$_NOT_` cell comment block from :file:`techlibs/common/simcells.v` + +.. + v1 descriptions in :file:`techlibs/common/simcells.v` have their version + unconditionally changed to ``2a`` to facilitate the truth table rendering, + making use of the v2 handling of codeblocks with ``::``. This also means A. + using ``::`` on its own in a v1 (gate-level) description should be avoided, and + B. *all* text after the ``"Truth table:"`` line is included in the codeblock. + .. _v2: v2 (more expressive) ~~~~~~~~~~~~~~~~~~~~ -- each field of the ``SimHelper`` struct can be assigned with: +Fields can be directly assigned with a ``//* `` comment line. We +saw this in the `v1`_ format with the group, but this is actually possible with +*all* fields of the ``SimHelper`` struct. In order to use the extra fields, +``ver`` must be explicitly set as ``2``. The extra fields available are as +follows, with an example provided by the :ref:`nex_module`. -.. code-block:: verilog +- title + A short title for the cell, equivalent to ``short_help`` in commands. + Rendered before the description and when hovering over links in + documentation. - //* - -- ``ver`` must be explicitly set as ``2`` -- optional fields ``title`` and ``tags`` - - - title used for better displaying of cell - - tags is a space-separated list of :doc:`/cell/properties` - -- the port signature is automatically generated from the ``module`` definition -- cell description is the same as v1 -- can link to commands or passes using backticks (`````) -- e.g. `$nex`: +- tags + A space-separated list of :doc:`/cell/properties`. Not used in `help` + output, but provided when dumping to JSON and in the Sphinx docs. .. literalinclude:: /generated/simlib.nex.v :language: verilog + :name: nex_module + :caption: `$nex` cell comment block from :file:`techlibs/common/simlib.v` -- code blocks can be included as following: +.. warning:: + + While it is possible to assign values to any of the ``SimHelper`` fields, + some fields are automatically assigned and explicitly setting them may result + in errors, or discarding of the assigned value. These fields are the name, + ports, code, source, and desc. + +The cell description is provided in the same way as in `v1`_, with each line +starting with a ``//-``. When generating the Sphinx documentation, the cell +description is interpreted as raw RST. This allows both in-line formatting like +linking to commands or passes using backticks (`````), and literal code blocks +with the ``::`` marker as in the following example: .. code-block:: verilog @@ -415,10 +430,10 @@ v2 (more expressive) //- //- more text -- the empty line after the ``::`` and before the text continues are required -- the ``::`` line will be ignored in the `help` call while sphinx docs will - treat everything that follows as a literal block until the next unindented - line: +Note that the empty line after the ``::`` and before the text continues are +required, as is the indentation before the literal contents. When rendering to +the terminal with `help `, the ``::`` line will be ignored, while +Sphinx displays the section verbatim like so: text :: @@ -429,6 +444,11 @@ v2 (more expressive) more text +.. todo:: in line formatting for web docs isn't exclusive to v2, + + but it does raise the question of if we should be doing something to prevent + v1 descriptions being treated as raw RST. + Dumping to JSON --------------- From 56c1ef2679e5e21f04357bf8e5d0e4d210db6ebb Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Fri, 21 Nov 2025 17:52:54 +1300 Subject: [PATCH 18/22] documenting.rst: Finished cells and starting on Sphinx Use an external file to demonstrate the `autoref` role (the external file lets us `literalinclude` and `include` without writing out the source twice, while still showing all the different ways of using our custom roles). --- .../extending_yosys/documenting.rst | 136 +++++++++++++----- .../extending_yosys/formatting_sample.txt | 16 +++ 2 files changed, 118 insertions(+), 34 deletions(-) create mode 100644 docs/source/yosys_internals/extending_yosys/formatting_sample.txt diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index 1b54070d4..87d398f73 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -419,21 +419,20 @@ description is interpreted as raw RST. This allows both in-line formatting like linking to commands or passes using backticks (`````), and literal code blocks with the ``::`` marker as in the following example: -.. code-block:: verilog +.. tab:: Verilog comment - //- text - //- :: - //- - //- monospaced text - //- - //- indentation and line length will be preserved, giving a scroll bar if necessary for the browser window - //- - //- more text + .. code-block:: verilog -Note that the empty line after the ``::`` and before the text continues are -required, as is the indentation before the literal contents. When rendering to -the terminal with `help `, the ``::`` line will be ignored, while -Sphinx displays the section verbatim like so: + //- text + //- :: + //- + //- monospaced text + //- + //- indentation and line length will be preserved, giving a scroll bar if necessary for the browser window + //- + //- more text + +.. tab:: formatted output text :: @@ -444,6 +443,11 @@ Sphinx displays the section verbatim like so: more text +Note that the empty line after the ``::`` and before the text continues are +required, as is the indentation before the literal contents. When rendering to +the terminal with `help `, the ``::`` line will be ignored, while +Sphinx displays the section verbatim like shown. + .. todo:: in line formatting for web docs isn't exclusive to v2, but it does raise the question of if we should be doing something to prevent @@ -537,11 +541,14 @@ auto detection will break and revert to unformatted code (e.g. Cells JSON ~~~~~~~~~~ -- effectively the ``SimHelper`` struct formatted as JSON (drops ``ver``, copies - tags to ``properties``) -- plus additional fields from the ``CellType`` - - + inputs, outputs, property flags +Dumping the cell help contents to JSON follows a very similar format as the +``SimHelper`` struct. The main difference is that there is no ``ver`` or +``group`` field, and the ``tags`` have become ``properties``. Each cell type +also has a corresponding ``CellType`` struct defined in +:file:`kernel/celltypes.h` which we now have access to. This allows us to +distinguish which ports are inputs and which are outputs, as well as some extra +property flags. The :ref:`nex_json` is reproduced here to show this +transformation. .. literalinclude:: /generated/cells.json :language: json @@ -551,27 +558,88 @@ Cells JSON :name: nex_json -Cells and commands in Sphinx ----------------------------- +Working with Sphinx +------------------- -To support the rich documentation of commands and cells in Yosys, we use two -custom `Sphinx Domains`_. +This documentation is built on Sphinx using `reStructuredText`_. To support the +rich documentation of commands and cells in Yosys, as well as the Yosys +scripting language and RTLIL, we use some custom extensions and will touch on +those here. + +.. _reStructuredText: https://docutils.sourceforge.io/rst.html + +Sphinx uses `Pygments`_ for syntax highlighting code blocks, for which we +provide to additional lexers. The first of these is ``RTLIL`` for the +:doc:`/yosys_internals/formats/rtlil_rep`, and is exclusive to the Yosys docs. +The second lexer, ``yoscrypt``, is for :doc:`/getting_started/scripting_intro` +and is available across all of the YosysHQ docs through `furo-ys`_, our custom +fork of the `furo`_ theme for Sphinx. These languages are automatically +associated with the ``.il`` and ``.ys`` file extensions respectively, and can be +selected for use in any ``literalinclude`` or ``code-block`` segments. + +.. _Pygments: https://pygments.org/ +.. _furo-ys: https://github.com/YosysHQ/furo-ys/ +.. _furo: https://github.com/pradyunsg/furo + +To simplify inline Yosys script syntax highlighting, these docs provide the +``yoscrypt`` role. This role renders (e.g.) ``:yoscrypt:`chformal -remove``` +into :yoscrypt:`chformal -remove`. For linking to command and cell +documentation, we also use a default role of ``autoref``. Any text in single +backticks without an explicit role will be assigned this one. We've already +seen this being used above in the help text for `chformal` and `$nex` (which +were themselves written as ```chformal``` and ```$nex``` respectively). + +By using the `autodoc extension`_ and two custom `Sphinx Domains`_ (more on them +later), ``autoref`` is able to produce links to any commands or cells available +in Yosys. So long as there are no spaces in the text, and it doesn't begin with +a dash (``-``), it will try to convert it to a link. If the text begins with +``$`` then it will use the ``cell:ref`` role, otherwise it will use ``cmd:ref``. +Let's take a look at some examples: .. _Sphinx Domains: https://www.sphinx-doc.org/en/master/usage/domains/index.html +.. _autodoc extension: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html -- ``yoscrypt`` role allows inline code to have yosys script syntax highlighting -- default role of ``autoref`` +.. tab:: reStructuredText - + any text in single backticks without an explicit role will be assigned this one - + will convert to ``cell:ref`` if it begins with ``$``, otherwise ``cmd:ref`` - + to attempt linking there must be no spaces, and it must not begin with a - dash (``-``) - + ```chformal``` (or ``:autoref:`chformal```) -> ``:cmd:ref:`chformal``` -> `chformal` - + also works with two words, if the first one is ``help`` - + ```help $add``` -> ``:cell:ref:`help $add <$add>``` -> `help $add` - + fallback to formatting as inline yoscrypt - + ```-remove``` -> `-remove` - + ```chformal -remove``` -> ``:yoscrypt:`chformal -remove``` -> `chformal -remove` + .. literalinclude:: formatting_sample.txt + :language: reStructuredText + :start-after: .. 1 + :end-before: .. 2 + +.. tab:: formatted output + + .. include:: formatting_sample.txt + :start-after: .. 1 + :end-before: .. 2 + +The ``autoref`` role also works with two words, if the first one is "help": + +.. tab:: reStructuredText + + .. literalinclude:: formatting_sample.txt + :language: reStructuredText + :start-after: .. 2 + :end-before: .. 3 + +.. tab:: formatted output + + .. include:: formatting_sample.txt + :start-after: .. 2 + :end-before: .. 3 + +And if the text begins with a dash, or doesn't match the "help" formatting, it +will fallback to formatting as inline yoscrypt. + +.. tab:: reStructuredText + + .. literalinclude:: formatting_sample.txt + :language: reStructuredText + :start-after: .. 3 + +.. tab:: formatted output + + .. include:: formatting_sample.txt + :start-after: .. 3 Using autodoc ~~~~~~~~~~~~~ diff --git a/docs/source/yosys_internals/extending_yosys/formatting_sample.txt b/docs/source/yosys_internals/extending_yosys/formatting_sample.txt new file mode 100644 index 000000000..280c524ae --- /dev/null +++ b/docs/source/yosys_internals/extending_yosys/formatting_sample.txt @@ -0,0 +1,16 @@ +.. 1 + +- `chformal` +- :autoref:`chformal` +- :cmd:ref:`chformal` + +.. 2 + +- `help $add` +- :autoref:`help $add` +- :cell:ref:`help $add <$add>` + +.. 3 + +- `-remove` +- `chformal -remove` From 5318df784a996b7cb9f6bb3724ce0938c8cbc149 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 22 Nov 2025 10:04:53 +1300 Subject: [PATCH 19/22] contributing.rst: Reference documenting.rst for help text --- docs/source/yosys_internals/extending_yosys/contributing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/yosys_internals/extending_yosys/contributing.rst b/docs/source/yosys_internals/extending_yosys/contributing.rst index 70170fc48..92c7c64a8 100644 --- a/docs/source/yosys_internals/extending_yosys/contributing.rst +++ b/docs/source/yosys_internals/extending_yosys/contributing.rst @@ -31,6 +31,8 @@ Formatting of code .. _Linux Kernel Coding Style: https://www.kernel.org/doc/Documentation/process/coding-style.rst +- Check out :doc:`documenting` for guidelines on providing help text for + commands and cells. C++ Language ~~~~~~~~~~~~ From dc8198561abe7cae0d6e5918e67c20ae82808630 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 22 Nov 2025 12:56:30 +1300 Subject: [PATCH 20/22] documenting.rst: Using autodoc Switch `autocmd` to use the `seealso` admonition for referencing source file location instead of `note`. --- .../extending_yosys/documenting.rst | 91 +++++++++++++------ .../extending_yosys/formatting_sample.txt | 2 + docs/util/cmd_documenter.py | 2 +- 3 files changed, 66 insertions(+), 29 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index 87d398f73..0870c1901 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -30,7 +30,8 @@ the name of the pass, as demonstrated here with `chformal`. :start-at: public Pass { :end-at: ChformalPass() :caption: ``ChformalPass()`` from :file:`passes/cmds/chformal.cc` - :append: + :name: chformal_pass + :append: // ... } ChformalPass; @@ -484,7 +485,9 @@ the `Warning flags`_, source location, source function, and corresponding group .. literalinclude:: /generated/cmds.json :language: json :start-at: "chformal": { - :end-before: "chparam": { + :end-at: "internal_flag": false + :append: } + :dedent: :caption: `chformal` in generated :file:`cmds.json` :name: chformal_json @@ -635,51 +638,83 @@ will fallback to formatting as inline yoscrypt. .. literalinclude:: formatting_sample.txt :language: reStructuredText :start-after: .. 3 + :end-before: .. 4 .. tab:: formatted output .. include:: formatting_sample.txt :start-after: .. 3 + :end-before: .. 4 Using autodoc ~~~~~~~~~~~~~ -- below is the raw RST output from ``autocmd`` (``YosysCmdDocumenter`` class in - :file:`docs/util/cmd_documenter.py`) for `chformal` command -- heading will be rendered as a subheading of the most recent heading (see - `chformal autocmd`_ above rendered under `Command help`_) -- ``.. cmd:def:: `` line is indexed for cross references with ``:cmd:ref:`` - directive (`chformal autocmd`_ above uses ``:noindex:`` option so that - `chformal` still links to the correct location) +The vast majority of command and cell help content in these docs is done with +the the `autodoc extension`_. By generating Sphinx documentation from our JSON +dumps of commands and cells, not only are we able to write the help content once +and have it available both in Yosys itself and online, we also ensure that any +code changes or additions are automatically propagated to the web docs. - + ``:title:`` option controls text that appears when hovering over the - `chformal` link +.. note:: -- commands with warning flags (experimental or internal) add a ``.. warning`` - block before any of the help content -- if a command has no ``source_location`` the ``.. note`` at the bottom will - instead link to :doc:`/cmd/index_other` + We are focusing on the ``autocmd`` directive here because it is easier to + demonstrate. In practice we don't really use it directly outside of this + page, and instead make use of the ``autocmdgroup`` directive. By providing + the ``:members:`` option, this is the same as calling ``autocmd`` for each + command in the group and means that any new commands are added automatically. -.. _showing autocmd generated rst: +Now let's take a look at the :ref:`chformal_rst` behind :ref:`chformal autocmd`. +This conversion is done by the ``YosysCmdDocumenter`` class in +:file:`docs/util/cmd_documenter.py`. We can see all of our ``paragraph`` and +``option`` nodes from :ref:`ChformalPass::formatted_help() ` +have made it through, as has the ``short_help`` from our :ref:`ChformalPass() +constructor `. The heading will be rendered as a subheading of +the most recent heading (notice how the `chformal` help content above is listed +under `Command help`_ in the table of contents). + +.. _chformal_rst: .. autocmd_rst:: chformal -- command groups documented with ``autocmdgroup `` +To support cross references with the ``cmd:ref`` role, we see everything is +under the ``cmd:def`` directive. The ``:title:`` option is what controls the +text that appears when hovering over the `chformal` link, and when using the +``cmd:title`` role. For commands with `warning flags`_, a ``.. warning`` block +is added to the generated RST before any of the help content. This is the same +`warning admonition`_ that we've seen elsewhere on this page. For commands with +no ``source_location``, the ``.. seealso`` block at the bottom will instead link +to :doc:`/cmd/index_other`. - + with ``:members:`` option this is the same as calling ``autocmd`` for each - member of the group +.. _warning admonition: https://pradyunsg.me/furo/reference/admonitions/#warning -- ``autocell`` and ``autocellgroup`` +.. hint:: - + very similar to ``autocmd`` and ``autocmdgroup`` but for cells instead of - commands (``YosysCellDocumenter`` in :file:`docs/util/cell_documenter.py`) - + optionally includes verilog source for cell(s) with ``:source:`` option - (plus ``:linenos:``) - + cell definitions do not include titles - + cells can have properties (:ref:`propindex`) + The :ref:`chformal autocmd` on this page uses the ``:noindex:`` option so + that references to `chformal` link to the :doc:`/cmd_ref` instead of this + page. -- bonus ``autocmd_rst``, used exclusively on this page for `showing autocmd - generated rst`_ +For documenting cells we have ``autocell`` and ``autocellgroup``, which function +pretty similarly to their command-based counter parts, ``autocmd`` and +``autocmdgroup``. These directives are provided by the ``YosysCellDocumenter`` +in :file:`docs/util/cell_documenter.py`. Like with `help +`, we are +able to include verilog simulation models in our ``autodoc`` with the +``:source:`` option. We can then also include line numbers by adding +``:linenos:``, which is very useful when trying to find the source code being +referenced. + +.. todo:: would be nice to get a ``.. autocell:: $nex`` + + like we did with `chformal autocmd`_, but it doesn't seem to like the + `:noindex:` option, or using `:source:` without it being ``binary::$nex``. + +.. todo:: cells can have properties (:ref:`propindex`) + +.. note:: + + For :ref:`showing autocmd generated rst ` on this page, we also + have the ``autocmd_rst`` directive. This is not used anywhere else in the + documentation, but it's mentioned here since we're already deep in the weeds + of how these docs are made. Our custom Sphinx domains ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/yosys_internals/extending_yosys/formatting_sample.txt b/docs/source/yosys_internals/extending_yosys/formatting_sample.txt index 280c524ae..d2cf211a9 100644 --- a/docs/source/yosys_internals/extending_yosys/formatting_sample.txt +++ b/docs/source/yosys_internals/extending_yosys/formatting_sample.txt @@ -14,3 +14,5 @@ - `-remove` - `chformal -remove` + +.. 4 diff --git a/docs/util/cmd_documenter.py b/docs/util/cmd_documenter.py index 9347d8ffd..18be204fd 100644 --- a/docs/util/cmd_documenter.py +++ b/docs/util/cmd_documenter.py @@ -394,7 +394,7 @@ class YosysCmdDocumenter(YosysCmdGroupDocumenter): if self.get_sourcename() != 'unknown': self.add_line('\n', source_name) - self.add_line(f'.. note:: Help text automatically generated from :file:`{source_name}:{source_line}`', source_name) + self.add_line(f'.. seealso:: Help text automatically generated from :file:`{source_name}:{source_line}`', source_name) # add additional content (e.g. from document), if present if more_content: From 39e3e392411225593f2d6a5ad9b3fce0164d3d6d Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 22 Nov 2025 15:36:53 +1300 Subject: [PATCH 21/22] documenting.rst: Our custom Sphinx domains --- .../extending_yosys/documenting.rst | 97 +++++++++++++++---- .../extending_yosys/formatting_sample.txt | 6 ++ 2 files changed, 82 insertions(+), 21 deletions(-) diff --git a/docs/source/yosys_internals/extending_yosys/documenting.rst b/docs/source/yosys_internals/extending_yosys/documenting.rst index 0870c1901..5d5e3299b 100644 --- a/docs/source/yosys_internals/extending_yosys/documenting.rst +++ b/docs/source/yosys_internals/extending_yosys/documenting.rst @@ -37,9 +37,9 @@ the name of the pass, as demonstrated here with `chformal`. All currently available commands are listed with their ``short_help`` string when calling `help` without arguments, and is more or less the same as the -unlisted :ref:`command index `. The string is also used when -hovering over links to commands in the documentation, and in section headings -like :ref:`chformal autocmd`. +:ref:`command index `. The string is also used when hovering over +links to commands in the documentation, and in section headings like +:ref:`chformal autocmd`. The next section shows the complete help text for the `chformal` command. This can be displayed locally by using `help ` (or ``yosys -h `` @@ -471,6 +471,19 @@ object which maps each group to the list of commands/cells in that group, and finally a ``cmds`` or ``cells`` object which maps each command/cell to its help content. +.. TODO:: Document how things get to Read the Docs + + - :file:`.github/workflows/prepare-docs.yml` + - github job compiles Yosys (with Verific) + - dumps JSON + - dumps program usage output for :doc:`/cmd_ref` and + :doc:`/appendix/auxprogs` + - runs examples, producing logs and images + - copies (some) source files for inclusion + - compresses and uploads artifact + - conditionally triggers RTDs to build + - ``rtds_action`` extension + Commands JSON ~~~~~~~~~~~~~ @@ -615,7 +628,10 @@ Let's take a look at some examples: :start-after: .. 1 :end-before: .. 2 -The ``autoref`` role also works with two words, if the first one is "help": +Notice how referencing `chformal` also puts the command name in an inline code +block. This is automatically done thanks to the use of `Sphinx Domains`_ and +helps to distinguish commands (and cells) from other types of links. The +``autoref`` role also works with two words, if the first one is "help": .. tab:: reStructuredText @@ -705,7 +721,8 @@ referenced. .. todo:: would be nice to get a ``.. autocell:: $nex`` like we did with `chformal autocmd`_, but it doesn't seem to like the - `:noindex:` option, or using `:source:` without it being ``binary::$nex``. + ``:noindex:`` option, or using ``:source:`` without it being + ``binary::$nex``. .. todo:: cells can have properties (:ref:`propindex`) @@ -719,26 +736,64 @@ referenced. Our custom Sphinx domains ~~~~~~~~~~~~~~~~~~~~~~~~~ -- ``cmd`` and ``cell`` -- Directives +To round out this document about documentation, let's take a brief look at our +custom Sphinx domains and what they provide. As you might expect from `Using +autodoc`_, these docs come with a domain for Yosys commands (``cmd``), and a +domain for built-in cells (``cell``). These are both provided in +:file:`docs/util/custom_directives.py`. From these domains we have the +following directives (``.. ::`` in RST): - + ``cmd:def`` provide command definition - + ``cmd:usage`` used by ``autocmd`` for command usage signatures - + ``cell:def`` provide cell definition - + ``cell:defprop`` provide cell property definition (used in - :doc:`/cell/properties`) - + ``cell:source`` used by ``autocell`` for simulation models +- ``cmd:def`` provide command definition, +- ``cmd:usage`` used by ``autocmd`` for command usage signatures, +- ``cell:def`` provide cell definition, +- ``cell:defprop`` provide cell property definition (used in + :doc:`/cell/properties`), and +- ``cell:source`` used by ``autocell`` for simulation models. -- Roles +For general documentation, it should not be necessary to interact with any of +these directives. Rather, everything should be accomplished through the use of +``autocmdgroup`` and ``autocellgroup``. We also have a few roles provided +(``::```` in RST): - + ``cmd:ref`` link to a ``cmd:def`` with the same name - + ``cmd:title`` same as ``cmd:ref``, but includes the short help in the text +- ``cmd:ref`` link to a ``cmd:def`` with the same name +- ``cmd:title`` same as ``cmd:ref``, but includes the short help in the text +- ``cell:ref`` link to a ``cell:def`` with the same name +- ``cell:title`` same as ``cell:ref``, but includes the title in the text +- ``cell:prop`` link to a ``cell:defprop`` of the same name - - ``:cmd:title:`chformal``` -> :cmd:title:`chformal` +For the ``:ref`` roles it's almost always easier to just not specify the +role; that's why ``autoref`` is there. And since all of the built-in cell types +start with ``$``, it's very easy to distinguish between a ``cmd:ref`` and a +``cell:ref``. When introducing a command it can be useful to quickly insert a +short description of it, so ``cmd:title`` sees a fair bit of use across the +documentation; particularly when it comes to the user-facing sections: - + ``cell:ref`` link to a ``cell:def`` with the same name - + ``cell:title`` +.. TODO:: is this the first time we mention the user/developer split? - - ``:cell:title:`$nex``` -> :cell:title:`$nex` +.. tab:: reStructuredText - + ``cell:prop`` link to a ``cell:defprop`` of the same name + .. literalinclude:: formatting_sample.txt + :language: reStructuredText + :start-after: .. 4 + :end-before: .. 5 + +.. tab:: formatted output + + .. include:: formatting_sample.txt + :start-after: .. 4 + :end-before: .. 5 + +Since only a small subset of cells provide titles (at the time of writing), +``cell:title`` is much less reliable, and more likely to give something that +isn't intended for the reader to see (like with `$_NOT_` in the above example). +The existence of ``cell:title`` is mostly an artifact of the ``CellDomain`` +being a subclass of the ``CommandDomain``. + +.. warning:: + + Because of how Sphinx caches domains (and/or because of how the + ``CommandDomain`` is setup), rebuilding pages with ``autocmdgroup`` or + ``autocellgroup`` directives can result in duplicate definitions on the + :ref:`command ` and :ref:`cell ` indices. A ``make + clean`` or ``rm -rf docs/build`` will resolve this. The online documentation + is not affected by this, since it always performs a clean build. \ No newline at end of file diff --git a/docs/source/yosys_internals/extending_yosys/formatting_sample.txt b/docs/source/yosys_internals/extending_yosys/formatting_sample.txt index d2cf211a9..1416ab8d1 100644 --- a/docs/source/yosys_internals/extending_yosys/formatting_sample.txt +++ b/docs/source/yosys_internals/extending_yosys/formatting_sample.txt @@ -16,3 +16,9 @@ - `chformal -remove` .. 4 + +- :cmd:title:`chformal` +- :cell:title:`$nex` +- :cell:title:`$_NOT_` + +.. 5 From 069f6ef34831d7623126b774af69fc760c115668 Mon Sep 17 00:00:00 2001 From: Krystine Sherwin <93062060+KrystalDelusion@users.noreply.github.com> Date: Sat, 22 Nov 2025 15:39:22 +1300 Subject: [PATCH 22/22] Docs: Fix cell option_spec inheritance Doesn't actually resolve the `:noindex:` failure, which is how I discovered the option wasn't being inherited, but it is a step in the right direction. --- docs/util/cell_documenter.py | 5 +++-- docs/util/custom_directives.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/util/cell_documenter.py b/docs/util/cell_documenter.py index 58e65c2ea..f30e88bcf 100644 --- a/docs/util/cell_documenter.py +++ b/docs/util/cell_documenter.py @@ -41,12 +41,13 @@ class YosysCellGroupDocumenter(Documenter): object: tuple[str, list[str]] lib_key = 'groups' - option_spec = { + option_spec = Documenter.option_spec.copy() + option_spec.update({ 'caption': autodoc.annotation_option, 'members': autodoc.members_option, 'source': autodoc.bool_option, 'linenos': autodoc.bool_option, - } + }) __cell_lib: dict[str, list[str] | dict[str]] | None = None @property diff --git a/docs/util/custom_directives.py b/docs/util/custom_directives.py index dc6dd29a4..4a23b44e4 100644 --- a/docs/util/custom_directives.py +++ b/docs/util/custom_directives.py @@ -245,12 +245,13 @@ class CellNode(TocNode): """A custom node that describes an internal cell.""" name = 'cell' - - option_spec = { + + option_spec = TocNode.option_spec.copy() + option_spec.update({ 'title': directives.unchanged, 'ports': directives.unchanged, 'properties': directives.unchanged, - } + }) doc_field_types = [ CellGroupedField('props', label='Properties', rolename='prop',