diff --git a/doc/api.rst b/doc/api.rst new file mode 100644 index 000000000..9950235c8 --- /dev/null +++ b/doc/api.rst @@ -0,0 +1,17 @@ +.. _api: + +:octicon:`cpu` API Reference +============================= + +.. _workflow_exceptions: + +Workflow Exceptions +------------------- +These custom exceptions are associated with generating +the workflows provided by the PTB. They are located in the +``exasol.toolbox.util.workflows.exceptions`` module. + +.. currentmodule:: exasol.toolbox.util.workflows.exceptions + +.. automodule:: exasol.toolbox.util.workflows.exceptions + :members: diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index a2e204d6c..43cca531a 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -5,6 +5,7 @@ ## Feature * #691: Started customization of PTB workflows by defining the YML schema +* #712: Added basic logging to workflow processing ## Documentation diff --git a/doc/conf.py b/doc/conf.py index c9522f9ca..afc9576f4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -42,7 +42,7 @@ "sphinx_copybutton", "exasol.toolbox.sphinx.multiversion", ] - +add_module_names = False intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} # Make sure the target is unique diff --git a/doc/index.rst b/doc/index.rst index e84152e40..bc00b76a2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,6 +31,12 @@ Documentation of the Exasol-Toolbox Custom GitHub Actions providing functionality that is commonly needed in our projects. + .. grid-item-card:: :octicon:`cpu` API Reference + :link: api + :link-type: ref + + Comprehensive technical documentation for API endpoints and methods + .. grid-item-card:: :octicon:`repo` Design Document :link: design_document :link-type: ref @@ -46,4 +52,5 @@ Documentation of the Exasol-Toolbox developer_guide/developer_guide tools github_actions/github_actions + api changes/changelog diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index b0f05b0ad..52dedaf39 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -8,6 +8,7 @@ Enabling GitHub Workflows :hidden: configuration + troubleshooting The PTB ships with configurable GitHub workflow templates covering the most common CI/CD setup variants for Python projects. The templates are defined in: diff --git a/doc/user_guide/features/github_workflows/troubleshooting.rst b/doc/user_guide/features/github_workflows/troubleshooting.rst new file mode 100644 index 000000000..1913048d4 --- /dev/null +++ b/doc/user_guide/features/github_workflows/troubleshooting.rst @@ -0,0 +1,9 @@ +.. _workflows_troubleshooting: + +Troubleshooting +=============== + +.. toctree:: + :maxdepth: 2 + + ../../troubleshooting/debug_github_workflows diff --git a/doc/user_guide/troubleshooting/debug_github_workflows.rst b/doc/user_guide/troubleshooting/debug_github_workflows.rst new file mode 100644 index 000000000..f3860c204 --- /dev/null +++ b/doc/user_guide/troubleshooting/debug_github_workflows.rst @@ -0,0 +1,26 @@ +.. _debug_workflows_troubleshooting: + +Debugging Generated GitHub Workflows +==================================== + +This troubleshooting guide is helpful if you run into issues installing or updating +the GitHub workflows provided by the PTB. + +Enabling Debug Logging +---------------------- + +To get more detailed output, set the ``LOG_LEVEL`` environment variable to ``DEBUG`` before executing a CLI command. +By default, the ``LOG_LEVEL`` is set to ``INFO``. + +.. code-block:: bash + + export LOG_LEVEL=DEBUG + +Checking Custom Exceptions +---------------------------- + +Certain pain points are associated with custom exceptions. These give a brief statement +on what could be wrong and in which file. For further information, check the traceback. + +For the list of the custom exceptions for installing or updating the GitHub workflows, +see the :ref:`workflow_exceptions`. diff --git a/doc/user_guide/troubleshooting/ignore_ruff_findings.rst b/doc/user_guide/troubleshooting/ignore_ruff_findings.rst index 6ca9d020d..c54d052eb 100644 --- a/doc/user_guide/troubleshooting/ignore_ruff_findings.rst +++ b/doc/user_guide/troubleshooting/ignore_ruff_findings.rst @@ -4,13 +4,13 @@ Ignoring Ruff Findings ====================== A typical example is when importing all PTB's Nox sessions in your -``noxfile.py``, which may cause ruff to report error "F403 unused import". +``noxfile.py``, which may cause ruff to report error "F401 unused import". You can ignore this finding by appending a comment to the code line: .. code-block:: python - from exasol.toolbox.nox.tasks import * # noqa: F403 + from exasol.toolbox.nox.tasks import * # noqa: F401 See also diff --git a/doc/user_guide/troubleshooting/index.rst b/doc/user_guide/troubleshooting/index.rst index 585fc995a..b6de56021 100644 --- a/doc/user_guide/troubleshooting/index.rst +++ b/doc/user_guide/troubleshooting/index.rst @@ -16,5 +16,6 @@ proposed mitigations, some potentially specific to the related tool. format_check_errors_due_to_configuration_issues format_check_reports_unmodified_files formatting_disable - "F403 unused import" (reported by Ruff) + "F401 unused import" (reported by Ruff) Sonar findings <../features/metrics/ignore_findings> + debug_github_workflows diff --git a/exasol/toolbox/__init__.py b/exasol/toolbox/__init__.py index e69de29bb..efbda7a1f 100644 --- a/exasol/toolbox/__init__.py +++ b/exasol/toolbox/__init__.py @@ -0,0 +1,33 @@ +import logging +import os + +import structlog +from structlog.dev import ConsoleRenderer +from structlog.processors import ( + CallsiteParameter, + CallsiteParameterAdder, + TimeStamper, + add_log_level, + format_exc_info, +) + +log_level = os.getenv("LOG_LEVEL", "INFO").upper() + +structlog.configure( + wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, log_level)), + processors=[ + # 1. Enrich the data first + add_log_level, + TimeStamper(fmt="iso"), + CallsiteParameterAdder( + { + CallsiteParameter.MODULE, + CallsiteParameter.FUNC_NAME, + } + ), + # 2. Handle exceptions + format_exc_info, + # 3. Rendering option + ConsoleRenderer(), + ], +) diff --git a/exasol/toolbox/util/workflows/__init__.py b/exasol/toolbox/util/workflows/__init__.py index e69de29bb..6eb60c176 100644 --- a/exasol/toolbox/util/workflows/__init__.py +++ b/exasol/toolbox/util/workflows/__init__.py @@ -0,0 +1,3 @@ +import structlog + +logger = structlog.get_logger(__name__).bind(subsystem="workflows") diff --git a/exasol/toolbox/util/workflows/exceptions.py b/exasol/toolbox/util/workflows/exceptions.py index 662126998..82155657a 100644 --- a/exasol/toolbox/util/workflows/exceptions.py +++ b/exasol/toolbox/util/workflows/exceptions.py @@ -26,7 +26,7 @@ class YamlOutputError(YamlError): class YamlParsingError(YamlError): """ Raised when the rendered template is not a valid YAML file, as it cannot be - parsed by ruamel-yaml. + parsed by ruamel-yaml. """ message_template = ( diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index 800acd5e6..a3b5b7fb1 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -15,6 +15,7 @@ ) from ruamel.yaml.error import YAMLError +from exasol.toolbox.util.workflows import logger from exasol.toolbox.util.workflows.exceptions import ( TemplateRenderingError, YamlOutputError, @@ -60,6 +61,11 @@ def _render_with_jinja(self, input_str: str) -> str: """ Render the template with Jinja. """ + logger.debug( + "Render template with Jinja", + jinja_dict_source="PROJECT_CONFIG.github_template_dict", + jinja_dict_values=self.github_template_dict, + ) jinja_template = jinja_env.from_string(input_str) return jinja_template.render(self.github_template_dict) @@ -72,6 +78,7 @@ def get_yaml_dict(self) -> CommentedMap: try: workflow_string = self._render_with_jinja(raw_content) yaml = self._get_standard_yaml() + logger.debug("Parse template with ruamel-yaml") return yaml.load(workflow_string) except TemplateError as ex: raise TemplateRenderingError(file_path=self.file_path) from ex @@ -84,6 +91,7 @@ def get_as_string(self, yaml_dict: CommentedMap) -> str: """ yaml = self._get_standard_yaml() try: + logger.debug("Output workflow as string") with io.StringIO() as stream: yaml.dump(yaml_dict, stream) workflow_string = stream.getvalue() diff --git a/exasol/toolbox/util/workflows/workflow.py b/exasol/toolbox/util/workflows/workflow.py index ff44dae62..db4b6b0ad 100644 --- a/exasol/toolbox/util/workflows/workflow.py +++ b/exasol/toolbox/util/workflows/workflow.py @@ -5,7 +5,11 @@ BaseModel, ConfigDict, ) +from structlog.contextvars import ( + bound_contextvars, +) +from exasol.toolbox.util.workflows import logger from exasol.toolbox.util.workflows.exceptions import YamlError from exasol.toolbox.util.workflows.process_template import WorkflowRenderer @@ -17,18 +21,21 @@ class Workflow(BaseModel): @classmethod def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any]): - if not file_path.exists(): - raise FileNotFoundError(file_path) - - try: - workflow_renderer = WorkflowRenderer( - github_template_dict=github_template_dict, - file_path=file_path, - ) - workflow = workflow_renderer.render() - return cls(content=workflow) - except YamlError as ex: - raise ex - except Exception as ex: - # Wrap all other "non-special" exceptions - raise ValueError(f"Error rendering file: {file_path}") from ex + with bound_contextvars(template_file_name=file_path.name): + logger.info("Load workflow from template") + + if not file_path.exists(): + raise FileNotFoundError(file_path) + + try: + workflow_renderer = WorkflowRenderer( + github_template_dict=github_template_dict, + file_path=file_path, + ) + workflow = workflow_renderer.render() + return cls(content=workflow) + except YamlError as ex: + raise ex + except Exception as ex: + # Wrap all other "non-special" exceptions + raise ValueError(f"Error rendering file: {file_path}") from ex diff --git a/poetry.lock b/poetry.lock index 7b341aca5..634301c0c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3637,6 +3637,21 @@ files = [ {file = "stevedore-5.6.0.tar.gz", hash = "sha256:f22d15c6ead40c5bbfa9ca54aa7e7b4a07d59b36ae03ed12ced1a54cf0b51945"}, ] +[[package]] +name = "structlog" +version = "25.5.0" +description = "Structured Logging for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f"}, + {file = "structlog-25.5.0.tar.gz", hash = "sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98"}, +] + +[package.dependencies] +typing-extensions = {version = "*", markers = "python_version < \"3.11\""} + [[package]] name = "tabulate" version = "0.9.0" @@ -3938,4 +3953,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "d1f9d88ca70834f069d89f24b82e2109852867a1a58396250c9c4ad89119f9af" +content-hash = "231df9e065279a52f02698bee3a9eab2706b8be77ec1b72d8ef61ff4a9e6af75" diff --git a/pyproject.toml b/pyproject.toml index 617ff6acd..d01a5f05b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,9 +55,10 @@ dependencies = [ "sphinx-inline-tabs>=2023.4.21,<2024", "sphinx-design>=0.5.0,<1", "sphinx-toolbox>=4.0.0,<5", + "sphinxcontrib-mermaid (>=2.0.0,<3.0.0)", + "structlog (>=25.5.0,<26.0.0)", "typer[all]>=0.7.0", "twine>=6.1.0,<7", - "sphinxcontrib-mermaid (>=2.0.0,<3.0.0)", ] [project.scripts] diff --git a/test/integration/tools/workflow_integration_test.py b/test/integration/tools/workflow_integration_test.py index a49fb7e23..cf85d0120 100644 --- a/test/integration/tools/workflow_integration_test.py +++ b/test/integration/tools/workflow_integration_test.py @@ -1,6 +1,7 @@ from unittest.mock import patch import pytest +from structlog.testing import capture_logs from exasol.toolbox.tools.workflow import CLI @@ -67,10 +68,11 @@ def test_show_workflow(cli_runner): ], ) def test_diff_workflow(cli_runner, tmp_path, workflow): - # set up with file in tmp_path so checks files are the same - cli_runner.invoke(CLI, ["install", workflow, str(tmp_path)]) + with capture_logs(): + # set up with file in tmp_path so checks files are the same + cli_runner.invoke(CLI, ["install", workflow, str(tmp_path)]) - result = cli_runner.invoke(CLI, ["diff", workflow, str(tmp_path)]) + result = cli_runner.invoke(CLI, ["diff", workflow, str(tmp_path)]) assert result.exit_code == 0 # as the files are the same, we expect no difference @@ -118,7 +120,8 @@ def test_install_twice_no_error(cli_runner, tmp_path): class TestUpdateWorkflow: @staticmethod def test_when_file_does_not_previously_exist(cli_runner, tmp_path): - result = cli_runner.invoke(CLI, ["update", "checks", str(tmp_path)]) + with capture_logs(): + result = cli_runner.invoke(CLI, ["update", "checks", str(tmp_path)]) assert result.exit_code == 0 assert result.output.strip() == f"Updated checks in \n{tmp_path}/checks.yml"