From 6d3e031fdf6db53ff91344e621872ac3fff447ce Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 17 Feb 2026 13:14:43 +0100 Subject: [PATCH 1/9] Add structlog as a dependency --- poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 7b341aca5..472ad7147 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3637,6 +3637,21 @@ files = [ {file = "stevedore-5.6.0.tar.gz", hash = "sha256:f22d15c6ead40c5bbfa9ca54aa7e7b4a07d59b36ae03ed12ced1a54cf0b51945"}, ] +[[package]] +name = "structlog" +version = "25.5.0" +description = "Structured Logging for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f"}, + {file = "structlog-25.5.0.tar.gz", hash = "sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98"}, +] + +[package.dependencies] +typing-extensions = {version = "*", markers = "python_version < \"3.11\""} + [[package]] name = "tabulate" version = "0.9.0" @@ -3938,4 +3953,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "d1f9d88ca70834f069d89f24b82e2109852867a1a58396250c9c4ad89119f9af" +content-hash = "172f284e885bc6277b300c747a6c04fbbd77522590e0a2b1deded4291f3c3529" diff --git a/pyproject.toml b/pyproject.toml index 617ff6acd..3dc863472 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "typer[all]>=0.7.0", "twine>=6.1.0,<7", "sphinxcontrib-mermaid (>=2.0.0,<3.0.0)", + "structlog (>=25.5.0,<26.0.0)", ] [project.scripts] From a30ac23ebe953e86217e934ffb82885342b715f7 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 17 Feb 2026 13:39:50 +0100 Subject: [PATCH 2/9] Add basic logging --- exasol/toolbox/__init__.py | 3 +++ exasol/toolbox/util/workflows/__init__.py | 3 +++ exasol/toolbox/util/workflows/render_yaml.py | 8 ++++++++ exasol/toolbox/util/workflows/workflow.py | 10 ++++++++++ 4 files changed, 24 insertions(+) diff --git a/exasol/toolbox/__init__.py b/exasol/toolbox/__init__.py index e69de29bb..ff82875b6 100644 --- a/exasol/toolbox/__init__.py +++ b/exasol/toolbox/__init__.py @@ -0,0 +1,3 @@ +import structlog + +structlog.configure() diff --git a/exasol/toolbox/util/workflows/__init__.py b/exasol/toolbox/util/workflows/__init__.py index e69de29bb..6eb60c176 100644 --- a/exasol/toolbox/util/workflows/__init__.py +++ b/exasol/toolbox/util/workflows/__init__.py @@ -0,0 +1,3 @@ +import structlog + +logger = structlog.get_logger(__name__).bind(subsystem="workflows") diff --git a/exasol/toolbox/util/workflows/render_yaml.py b/exasol/toolbox/util/workflows/render_yaml.py index 800acd5e6..a3b5b7fb1 100644 --- a/exasol/toolbox/util/workflows/render_yaml.py +++ b/exasol/toolbox/util/workflows/render_yaml.py @@ -15,6 +15,7 @@ ) from ruamel.yaml.error import YAMLError +from exasol.toolbox.util.workflows import logger from exasol.toolbox.util.workflows.exceptions import ( TemplateRenderingError, YamlOutputError, @@ -60,6 +61,11 @@ def _render_with_jinja(self, input_str: str) -> str: """ Render the template with Jinja. """ + logger.debug( + "Render template with Jinja", + jinja_dict_source="PROJECT_CONFIG.github_template_dict", + jinja_dict_values=self.github_template_dict, + ) jinja_template = jinja_env.from_string(input_str) return jinja_template.render(self.github_template_dict) @@ -72,6 +78,7 @@ def get_yaml_dict(self) -> CommentedMap: try: workflow_string = self._render_with_jinja(raw_content) yaml = self._get_standard_yaml() + logger.debug("Parse template with ruamel-yaml") return yaml.load(workflow_string) except TemplateError as ex: raise TemplateRenderingError(file_path=self.file_path) from ex @@ -84,6 +91,7 @@ def get_as_string(self, yaml_dict: CommentedMap) -> str: """ yaml = self._get_standard_yaml() try: + logger.debug("Output workflow as string") with io.StringIO() as stream: yaml.dump(yaml_dict, stream) workflow_string = stream.getvalue() diff --git a/exasol/toolbox/util/workflows/workflow.py b/exasol/toolbox/util/workflows/workflow.py index ff44dae62..cbdf70e95 100644 --- a/exasol/toolbox/util/workflows/workflow.py +++ b/exasol/toolbox/util/workflows/workflow.py @@ -5,7 +5,12 @@ BaseModel, ConfigDict, ) +from structlog.contextvars import ( + bind_contextvars, + clear_contextvars, +) +from exasol.toolbox.util.workflows import logger from exasol.toolbox.util.workflows.exceptions import YamlError from exasol.toolbox.util.workflows.process_template import WorkflowRenderer @@ -17,6 +22,9 @@ class Workflow(BaseModel): @classmethod def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any]): + bind_contextvars(template_file_name=file_path.name) + logger.info("Load workflow from template") + if not file_path.exists(): raise FileNotFoundError(file_path) @@ -32,3 +40,5 @@ def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any except Exception as ex: # Wrap all other "non-special" exceptions raise ValueError(f"Error rendering file: {file_path}") from ex + finally: + clear_contextvars() From fab9b12c79c0bd20352bede91cc91737d82a6621 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Tue, 17 Feb 2026 13:59:42 +0100 Subject: [PATCH 3/9] Add documentation about logs and custom exceptions --- doc/api.rst | 17 ++++++++++++ doc/changes/unreleased.md | 1 + doc/conf.py | 2 +- doc/index.rst | 7 +++++ .../features/github_workflows/index.rst | 1 + .../github_workflows/troubleshooting.rst | 26 +++++++++++++++++++ doc/user_guide/troubleshooting.rst | 1 + exasol/toolbox/__init__.py | 9 ++++++- exasol/toolbox/util/workflows/exceptions.py | 2 +- 9 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 doc/api.rst create mode 100644 doc/user_guide/features/github_workflows/troubleshooting.rst diff --git a/doc/api.rst b/doc/api.rst new file mode 100644 index 000000000..9950235c8 --- /dev/null +++ b/doc/api.rst @@ -0,0 +1,17 @@ +.. _api: + +:octicon:`cpu` API Reference +============================= + +.. _workflow_exceptions: + +Workflow Exceptions +------------------- +These custom exceptions are associated with generating +the workflows provided by the PTB. They are located in the +``exasol.toolbox.util.workflows.exceptions`` module. + +.. currentmodule:: exasol.toolbox.util.workflows.exceptions + +.. automodule:: exasol.toolbox.util.workflows.exceptions + :members: diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 77106267c..65339cd8a 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -5,6 +5,7 @@ ## Feature * #691: Started customization of PTB workflows by defining the YML schema +* #712: Added basic logging to workflow processing ## Refactoring diff --git a/doc/conf.py b/doc/conf.py index fc1342280..a411dc4e7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -42,7 +42,7 @@ "sphinx_copybutton", "exasol.toolbox.sphinx.multiversion", ] - +add_module_names = False intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} # Make sure the target is unique diff --git a/doc/index.rst b/doc/index.rst index e84152e40..bc00b76a2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,6 +31,12 @@ Documentation of the Exasol-Toolbox Custom GitHub Actions providing functionality that is commonly needed in our projects. + .. grid-item-card:: :octicon:`cpu` API Reference + :link: api + :link-type: ref + + Comprehensive technical documentation for API endpoints and methods + .. grid-item-card:: :octicon:`repo` Design Document :link: design_document :link-type: ref @@ -46,4 +52,5 @@ Documentation of the Exasol-Toolbox developer_guide/developer_guide tools github_actions/github_actions + api changes/changelog diff --git a/doc/user_guide/features/github_workflows/index.rst b/doc/user_guide/features/github_workflows/index.rst index b0f05b0ad..52dedaf39 100644 --- a/doc/user_guide/features/github_workflows/index.rst +++ b/doc/user_guide/features/github_workflows/index.rst @@ -8,6 +8,7 @@ Enabling GitHub Workflows :hidden: configuration + troubleshooting The PTB ships with configurable GitHub workflow templates covering the most common CI/CD setup variants for Python projects. The templates are defined in: diff --git a/doc/user_guide/features/github_workflows/troubleshooting.rst b/doc/user_guide/features/github_workflows/troubleshooting.rst new file mode 100644 index 000000000..56d04032e --- /dev/null +++ b/doc/user_guide/features/github_workflows/troubleshooting.rst @@ -0,0 +1,26 @@ +.. _github_workflows_troubleshooting: + +Troubleshooting +=============== + +This troubleshooting guide is helpful if you run into issues installing or updating +the GitHub workflows provided by the PTB. + +Enabling Debug Logging +---------------------- + +To get more detailed output, set the ``LOG_LEVEL`` environment variable to ``DEBUG`` before executing a CLI command. +By default, the ``LOG_LEVEL`` is set to ``INFO``. + +.. code-block:: bash + + export LOG_LEVEL=DEBUG + +Checking Custom Exceptions +---------------------------- + +Certain pain points are associated with custom exceptions. These give a brief statement +on what could be wrong and in which file. For further information, check the traceback. + +For the list of the custom exceptions for installing or updating the GitHub workflows, +see the :ref:`workflow_exceptions`. diff --git a/doc/user_guide/troubleshooting.rst b/doc/user_guide/troubleshooting.rst index ba1232fca..df1a2be95 100644 --- a/doc/user_guide/troubleshooting.rst +++ b/doc/user_guide/troubleshooting.rst @@ -7,3 +7,4 @@ Troubleshooting :maxdepth: 1 features/metrics/ignore_findings + Updating GitHub Workflows diff --git a/exasol/toolbox/__init__.py b/exasol/toolbox/__init__.py index ff82875b6..15b87bce1 100644 --- a/exasol/toolbox/__init__.py +++ b/exasol/toolbox/__init__.py @@ -1,3 +1,10 @@ +import logging +import os + import structlog -structlog.configure() +log_level = os.getenv("LOG_LEVEL", "INFO").upper() + +structlog.configure( + wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, log_level)) +) diff --git a/exasol/toolbox/util/workflows/exceptions.py b/exasol/toolbox/util/workflows/exceptions.py index 662126998..82155657a 100644 --- a/exasol/toolbox/util/workflows/exceptions.py +++ b/exasol/toolbox/util/workflows/exceptions.py @@ -26,7 +26,7 @@ class YamlOutputError(YamlError): class YamlParsingError(YamlError): """ Raised when the rendered template is not a valid YAML file, as it cannot be - parsed by ruamel-yaml. + parsed by ruamel-yaml. """ message_template = ( From a4e9eaabc23d5ff4f2d3d1ffaa8eb57d32344c42 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 18 Feb 2026 08:42:16 +0100 Subject: [PATCH 4/9] Switch to context manager so closes for all --- exasol/toolbox/util/workflows/workflow.py | 41 +++++++++++------------ 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/exasol/toolbox/util/workflows/workflow.py b/exasol/toolbox/util/workflows/workflow.py index cbdf70e95..db4b6b0ad 100644 --- a/exasol/toolbox/util/workflows/workflow.py +++ b/exasol/toolbox/util/workflows/workflow.py @@ -6,8 +6,7 @@ ConfigDict, ) from structlog.contextvars import ( - bind_contextvars, - clear_contextvars, + bound_contextvars, ) from exasol.toolbox.util.workflows import logger @@ -22,23 +21,21 @@ class Workflow(BaseModel): @classmethod def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any]): - bind_contextvars(template_file_name=file_path.name) - logger.info("Load workflow from template") - - if not file_path.exists(): - raise FileNotFoundError(file_path) - - try: - workflow_renderer = WorkflowRenderer( - github_template_dict=github_template_dict, - file_path=file_path, - ) - workflow = workflow_renderer.render() - return cls(content=workflow) - except YamlError as ex: - raise ex - except Exception as ex: - # Wrap all other "non-special" exceptions - raise ValueError(f"Error rendering file: {file_path}") from ex - finally: - clear_contextvars() + with bound_contextvars(template_file_name=file_path.name): + logger.info("Load workflow from template") + + if not file_path.exists(): + raise FileNotFoundError(file_path) + + try: + workflow_renderer = WorkflowRenderer( + github_template_dict=github_template_dict, + file_path=file_path, + ) + workflow = workflow_renderer.render() + return cls(content=workflow) + except YamlError as ex: + raise ex + except Exception as ex: + # Wrap all other "non-special" exceptions + raise ValueError(f"Error rendering file: {file_path}") from ex From a356f56685bce51557585c9ce75aed8679606126 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 18 Feb 2026 08:42:46 +0100 Subject: [PATCH 5/9] Add callsite information, which means default processors need to be added --- exasol/toolbox/__init__.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/exasol/toolbox/__init__.py b/exasol/toolbox/__init__.py index 15b87bce1..efbda7a1f 100644 --- a/exasol/toolbox/__init__.py +++ b/exasol/toolbox/__init__.py @@ -2,9 +2,32 @@ import os import structlog +from structlog.dev import ConsoleRenderer +from structlog.processors import ( + CallsiteParameter, + CallsiteParameterAdder, + TimeStamper, + add_log_level, + format_exc_info, +) log_level = os.getenv("LOG_LEVEL", "INFO").upper() structlog.configure( - wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, log_level)) + wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, log_level)), + processors=[ + # 1. Enrich the data first + add_log_level, + TimeStamper(fmt="iso"), + CallsiteParameterAdder( + { + CallsiteParameter.MODULE, + CallsiteParameter.FUNC_NAME, + } + ), + # 2. Handle exceptions + format_exc_info, + # 3. Rendering option + ConsoleRenderer(), + ], ) From 18dc655896e8ea4be029634def9f9fc11a950bf4 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 18 Feb 2026 09:04:59 +0100 Subject: [PATCH 6/9] Capture structlog separately so output as before --- test/integration/tools/workflow_integration_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/integration/tools/workflow_integration_test.py b/test/integration/tools/workflow_integration_test.py index a49fb7e23..cf85d0120 100644 --- a/test/integration/tools/workflow_integration_test.py +++ b/test/integration/tools/workflow_integration_test.py @@ -1,6 +1,7 @@ from unittest.mock import patch import pytest +from structlog.testing import capture_logs from exasol.toolbox.tools.workflow import CLI @@ -67,10 +68,11 @@ def test_show_workflow(cli_runner): ], ) def test_diff_workflow(cli_runner, tmp_path, workflow): - # set up with file in tmp_path so checks files are the same - cli_runner.invoke(CLI, ["install", workflow, str(tmp_path)]) + with capture_logs(): + # set up with file in tmp_path so checks files are the same + cli_runner.invoke(CLI, ["install", workflow, str(tmp_path)]) - result = cli_runner.invoke(CLI, ["diff", workflow, str(tmp_path)]) + result = cli_runner.invoke(CLI, ["diff", workflow, str(tmp_path)]) assert result.exit_code == 0 # as the files are the same, we expect no difference @@ -118,7 +120,8 @@ def test_install_twice_no_error(cli_runner, tmp_path): class TestUpdateWorkflow: @staticmethod def test_when_file_does_not_previously_exist(cli_runner, tmp_path): - result = cli_runner.invoke(CLI, ["update", "checks", str(tmp_path)]) + with capture_logs(): + result = cli_runner.invoke(CLI, ["update", "checks", str(tmp_path)]) assert result.exit_code == 0 assert result.output.strip() == f"Updated checks in \n{tmp_path}/checks.yml" From 2e46789fc0be5cc39f8daa0f5fb8170a48b38e21 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Wed, 18 Feb 2026 09:07:56 +0100 Subject: [PATCH 7/9] Alphabetize & relock --- poetry.lock | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 472ad7147..634301c0c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3953,4 +3953,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "172f284e885bc6277b300c747a6c04fbbd77522590e0a2b1deded4291f3c3529" +content-hash = "231df9e065279a52f02698bee3a9eab2706b8be77ec1b72d8ef61ff4a9e6af75" diff --git a/pyproject.toml b/pyproject.toml index 3dc863472..d01a5f05b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,10 +55,10 @@ dependencies = [ "sphinx-inline-tabs>=2023.4.21,<2024", "sphinx-design>=0.5.0,<1", "sphinx-toolbox>=4.0.0,<5", - "typer[all]>=0.7.0", - "twine>=6.1.0,<7", "sphinxcontrib-mermaid (>=2.0.0,<3.0.0)", "structlog (>=25.5.0,<26.0.0)", + "typer[all]>=0.7.0", + "twine>=6.1.0,<7", ] [project.scripts] From 321c435c664d98058323a8cda8edaeecccc593af Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 19 Feb 2026 08:59:13 +0100 Subject: [PATCH 8/9] Fix to F401 --- doc/user_guide/troubleshooting/ignore_ruff_findings.rst | 4 ++-- doc/user_guide/troubleshooting/index.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/user_guide/troubleshooting/ignore_ruff_findings.rst b/doc/user_guide/troubleshooting/ignore_ruff_findings.rst index 6ca9d020d..c54d052eb 100644 --- a/doc/user_guide/troubleshooting/ignore_ruff_findings.rst +++ b/doc/user_guide/troubleshooting/ignore_ruff_findings.rst @@ -4,13 +4,13 @@ Ignoring Ruff Findings ====================== A typical example is when importing all PTB's Nox sessions in your -``noxfile.py``, which may cause ruff to report error "F403 unused import". +``noxfile.py``, which may cause ruff to report error "F401 unused import". You can ignore this finding by appending a comment to the code line: .. code-block:: python - from exasol.toolbox.nox.tasks import * # noqa: F403 + from exasol.toolbox.nox.tasks import * # noqa: F401 See also diff --git a/doc/user_guide/troubleshooting/index.rst b/doc/user_guide/troubleshooting/index.rst index 585fc995a..95d9abf74 100644 --- a/doc/user_guide/troubleshooting/index.rst +++ b/doc/user_guide/troubleshooting/index.rst @@ -16,5 +16,5 @@ proposed mitigations, some potentially specific to the related tool. format_check_errors_due_to_configuration_issues format_check_reports_unmodified_files formatting_disable - "F403 unused import" (reported by Ruff) + "F401 unused import" (reported by Ruff) Sonar findings <../features/metrics/ignore_findings> From 55bfb78687daa4a30defb8747c3c8e117f36a9d7 Mon Sep 17 00:00:00 2001 From: Ariel Schulz Date: Thu, 19 Feb 2026 09:05:30 +0100 Subject: [PATCH 9/9] Move troubleshooting to new area --- .../github_workflows/troubleshooting.rst | 25 +++--------------- .../debug_github_workflows.rst | 26 +++++++++++++++++++ doc/user_guide/troubleshooting/index.rst | 1 + 3 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 doc/user_guide/troubleshooting/debug_github_workflows.rst diff --git a/doc/user_guide/features/github_workflows/troubleshooting.rst b/doc/user_guide/features/github_workflows/troubleshooting.rst index 56d04032e..1913048d4 100644 --- a/doc/user_guide/features/github_workflows/troubleshooting.rst +++ b/doc/user_guide/features/github_workflows/troubleshooting.rst @@ -1,26 +1,9 @@ -.. _github_workflows_troubleshooting: +.. _workflows_troubleshooting: Troubleshooting =============== -This troubleshooting guide is helpful if you run into issues installing or updating -the GitHub workflows provided by the PTB. +.. toctree:: + :maxdepth: 2 -Enabling Debug Logging ----------------------- - -To get more detailed output, set the ``LOG_LEVEL`` environment variable to ``DEBUG`` before executing a CLI command. -By default, the ``LOG_LEVEL`` is set to ``INFO``. - -.. code-block:: bash - - export LOG_LEVEL=DEBUG - -Checking Custom Exceptions ----------------------------- - -Certain pain points are associated with custom exceptions. These give a brief statement -on what could be wrong and in which file. For further information, check the traceback. - -For the list of the custom exceptions for installing or updating the GitHub workflows, -see the :ref:`workflow_exceptions`. + ../../troubleshooting/debug_github_workflows diff --git a/doc/user_guide/troubleshooting/debug_github_workflows.rst b/doc/user_guide/troubleshooting/debug_github_workflows.rst new file mode 100644 index 000000000..f3860c204 --- /dev/null +++ b/doc/user_guide/troubleshooting/debug_github_workflows.rst @@ -0,0 +1,26 @@ +.. _debug_workflows_troubleshooting: + +Debugging Generated GitHub Workflows +==================================== + +This troubleshooting guide is helpful if you run into issues installing or updating +the GitHub workflows provided by the PTB. + +Enabling Debug Logging +---------------------- + +To get more detailed output, set the ``LOG_LEVEL`` environment variable to ``DEBUG`` before executing a CLI command. +By default, the ``LOG_LEVEL`` is set to ``INFO``. + +.. code-block:: bash + + export LOG_LEVEL=DEBUG + +Checking Custom Exceptions +---------------------------- + +Certain pain points are associated with custom exceptions. These give a brief statement +on what could be wrong and in which file. For further information, check the traceback. + +For the list of the custom exceptions for installing or updating the GitHub workflows, +see the :ref:`workflow_exceptions`. diff --git a/doc/user_guide/troubleshooting/index.rst b/doc/user_guide/troubleshooting/index.rst index 95d9abf74..b6de56021 100644 --- a/doc/user_guide/troubleshooting/index.rst +++ b/doc/user_guide/troubleshooting/index.rst @@ -18,3 +18,4 @@ proposed mitigations, some potentially specific to the related tool. formatting_disable "F401 unused import" (reported by Ruff) Sonar findings <../features/metrics/ignore_findings> + debug_github_workflows