diff --git a/.github/workflows/python-daily-tests.yml b/.github/workflows/python-daily-tests.yml new file mode 100644 index 0000000..e5df009 --- /dev/null +++ b/.github/workflows/python-daily-tests.yml @@ -0,0 +1,23 @@ +--- +on: + push: + branches: + - master + pull_request: + branches: + - master + +name: Run Tox tests on daily-tests +jobs: + tox_test: + strategy: + matrix: + tox_env: [py311, py312, py313, py314] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: fedora-python/tox-github-action@main + with: + tox_env: ${{ matrix.tox_env }} + workdir: "daily_tests/" +... diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-ocp-generator-tests.yml similarity index 62% rename from .github/workflows/python-tests.yml rename to .github/workflows/python-ocp-generator-tests.yml index 83b9ccc..17905a8 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-ocp-generator-tests.yml @@ -7,14 +7,17 @@ on: branches: - master -name: Run Tox tests on ci-scripts +name: Run Tox tests on ci-scripts/ocp_stream_generator jobs: tox_test: + strategy: + matrix: + tox_env: [py311, py312, py313, py314] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: fedora-python/tox-github-action@main with: - tox_env: py311 + tox_env: ${{ matrix.tox_env }} workdir: "ocp-stream-generator/" ... diff --git a/.gitignore b/.gitignore index 2c7c6e9..ae49b59 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .mypy_cache .idea .vscode +__pycache__ diff --git a/daily_tests/__init_.py b/daily_tests/__init_.py new file mode 100644 index 0000000..e69de29 diff --git a/daily_tests/daily_nightly_tests_report.py b/daily_tests/daily_nightly_tests_report.py index cb27eb4..718d971 100755 --- a/daily_tests/daily_nightly_tests_report.py +++ b/daily_tests/daily_nightly_tests_report.py @@ -15,7 +15,6 @@ "phracek@redhat.com", "hhorak@redhat.com", "pkubat@redhat.com", - "anezbeda@redhat.com", "pkhartsk@redhat.com", ] upstream_mails = [ diff --git a/daily_tests/pytest.ini b/daily_tests/pytest.ini new file mode 100644 index 0000000..c7b23ec --- /dev/null +++ b/daily_tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +pythonpath = . +testpaths = tests diff --git a/show_logs.py b/daily_tests/show_logs.py similarity index 100% rename from show_logs.py rename to daily_tests/show_logs.py diff --git a/show_logs.sh b/daily_tests/show_logs.sh similarity index 100% rename from show_logs.sh rename to daily_tests/show_logs.sh diff --git a/daily_tests/tests/test_daily_grades.py b/daily_tests/tests/test_daily_grades.py new file mode 100644 index 0000000..834484b --- /dev/null +++ b/daily_tests/tests/test_daily_grades.py @@ -0,0 +1,141 @@ +# pylint: disable=import-error,redefined-outer-name +import importlib +import sys +import types +from pathlib import Path + +import pytest + + +@pytest.fixture +def daily_grades_module(): + fake_main = types.ModuleType("rh_cwt.main") + + class FakeRhelImageRebuilder: + def __init__(self, base_image): + self.base_image = base_image + self.exclude_image = None + self.configs = [] + self.last_config = None + self.grades_by_config = {} + + def set_config(self, config): + self.last_config = config + self.configs.append(config) + + def check_rhcc_grades(self): + return self.grades_by_config.get(self.last_config, []) + + fake_main.RhelImageRebuilder = FakeRhelImageRebuilder + fake_pkg = types.ModuleType("rh_cwt") + fake_pkg.main = fake_main + + original_modules = { + "rh_cwt": sys.modules.get("rh_cwt"), + "rh_cwt.main": sys.modules.get("rh_cwt.main"), + "daily_grades": sys.modules.get("daily_grades"), + } + sys.modules["rh_cwt"] = fake_pkg + sys.modules["rh_cwt.main"] = fake_main + + daily_tests_dir = Path(__file__).resolve().parents[1] / "daily_tests" + original_sys_path = list(sys.path) + sys.path.insert(0, str(daily_tests_dir)) + + try: + module = importlib.import_module("daily_grades") + importlib.reload(module) + yield module + finally: + sys.path[:] = original_sys_path + if original_modules["daily_grades"] is None: + sys.modules.pop("daily_grades", None) + else: + sys.modules["daily_grades"] = original_modules["daily_grades"] + if original_modules["rh_cwt"] is None: + sys.modules.pop("rh_cwt", None) + else: + sys.modules["rh_cwt"] = original_modules["rh_cwt"] + if original_modules["rh_cwt.main"] is None: + sys.modules.pop("rh_cwt.main", None) + else: + sys.modules["rh_cwt.main"] = original_modules["rh_cwt.main"] + + +@pytest.fixture +def report(daily_grades_module): + return daily_grades_module.DailyGradesReport() + + +@pytest.fixture +def smtp_spy(daily_grades_module): + instances = [] + original_smtp = daily_grades_module.smtplib.SMTP + + class FakeSMTP: + def __init__(self, host): + self.host = host + self.sent = [] + self.closed = False + instances.append(self) + + def sendmail(self, send_from, send_to, msg): + self.sent.append((send_from, send_to, msg)) + + def close(self): + self.closed = True + + daily_grades_module.smtplib.SMTP = FakeSMTP + try: + yield instances + finally: + daily_grades_module.smtplib.SMTP = original_smtp + + +def test_get_grades_collects_all_configs(report): + report.rhcwt_api.grades_by_config = { + "rhel8.yaml": [("nodejs", "A", 0)], + "rhel9.yaml": [("php", "B", 2)], + "rhel10.yaml": [("python", "C", 1)], + } + + report.get_grades() + + assert report.grades_dict["RHEL8"] == [("nodejs", "A", 0)] + assert report.grades_dict["RHEL9"] == [("php", "B", 2)] + assert report.grades_dict["RHEL10"] == [("python", "C", 1)] + assert report.rhcwt_api.exclude_image == "nodejs-10" + assert report.rhcwt_api.configs == ["rhel8.yaml", "rhel9.yaml", "rhel10.yaml"] + + +def test_check_grades_builds_body(report): + report.grades_dict = { + "RHEL8": [("nodejs", "B", 5), ("ruby", "NONE", 0)], + "RHEL9": [("php", "C", 3)], + "RHEL10": [("python", "D", -2)], + } + + report.check_grades() + + assert "Nightly report for container grades for RHEL8" in report.body + assert "nodejs [B] 5 days until grade C!" in report.body + assert "Some images were not found in container catalog." in report.body + assert "php [C] 3 days since grade C!" in report.body + assert "python [D] 2 days last grade change!" in report.body + assert "The rest of images are in grade A." in report.body + + +def test_send_email_uses_smtp(report, smtp_spy): + report.body = "Report" + + report.send_email() + + assert smtp_spy + instance = smtp_spy[0] + assert instance.host == "127.0.0.1" + assert instance.closed is True + assert instance.sent + send_from, send_to, msg = instance.sent[0] + assert send_from == "phracek@redhat.com" + assert "Container Grades" in msg + assert ", ".join(send_to) in msg diff --git a/daily_tests/tests/test_show_logs.py b/daily_tests/tests/test_show_logs.py new file mode 100644 index 0000000..ec4ead0 --- /dev/null +++ b/daily_tests/tests/test_show_logs.py @@ -0,0 +1,122 @@ +# pylint: disable=import-error +import sys + +from datetime import date as real_date +from pathlib import Path + +import pytest + + +import show_logs + +TEST_DIR = Path(__file__).parent.absolute() +sys.path.append(str(TEST_DIR)) + + +class FixedDate(real_date): + @classmethod + def today(cls): + return cls(2024, 1, 2) + + +@pytest.fixture +def report_env(tmp_path): + reports_dir = tmp_path / "reports" + scl_dir = tmp_path / "scl" + original_reports_dir = show_logs.DAILY_REPORTS_DIR + original_scl_tests_dir = show_logs.DAILY_SCL_TESTS_DIR + original_date = show_logs.date + show_logs.DAILY_REPORTS_DIR = reports_dir + show_logs.DAILY_SCL_TESTS_DIR = scl_dir + show_logs.date = FixedDate + try: + yield show_logs.PVCWatcherReport(), reports_dir, scl_dir + finally: + show_logs.DAILY_REPORTS_DIR = original_reports_dir + show_logs.DAILY_SCL_TESTS_DIR = original_scl_tests_dir + show_logs.date = original_date + + +def test_return_failed_tests_finds_logs(tmp_path): + item_name = "c9s-example" + base_dir = tmp_path + logs_dir = ( + base_dir / item_name / "plans/nightly/nightly-c9s/data/results" / "nested" + ) + logs_dir.mkdir(parents=True) + log_file = logs_dir / "failed.log" + log_file.write_text("failed") + + report = show_logs.PVCWatcherReport() + results = report.return_failed_tests(base_dir, Path(item_name)) + + assert log_file in results + + +def test_iter_over_executed_tests_no_failures(report_env, capsys): + report, _, _ = report_env + report.scl_tests_dir.mkdir(parents=True) + (report.scl_tests_dir / "c9s-run").mkdir() + + report.iter_over_executed_tests() + output = capsys.readouterr().out + + assert "No container test failures found in" in output + + +def test_show_all_available_tests_lists_dirs(report_env, capsys): + report, reports_dir, _ = report_env + reports_dir.mkdir() + (reports_dir / "2024-01-01").mkdir() + (reports_dir / "2024-01-02").mkdir() + (reports_dir / "not-a-dir.txt").write_text("skip") + + report.show_all_available_tests() + output = capsys.readouterr().out + + assert "2024-01-01" in output + assert "2024-01-02" in output + assert "not-a-dir.txt" not in output + + +def test_print_report_missing_directories(report_env, capsys): + report, _, _ = report_env + + report.print_report() + output = capsys.readouterr().out + + assert "Tests were not executed yet." in output + assert "Tests were not finished yet." in output + + +def test_iter_results_in_directory_summarizes(report_env, capsys): + report, _, _ = report_env + report.scl_tests_dir.mkdir(parents=True) + running_dir = report.scl_tests_dir / "rhel9-test-pytest" + running_dir.mkdir() + (running_dir / "tmt_running").write_text("running") + + report.reports_dir.mkdir(parents=True) + success_dir = report.reports_dir / "c9s-test" + success_dir.mkdir() + (success_dir / "tmt_success").write_text("success") + + failed_dir = report.reports_dir / "rhel9-test" + failed_dir.mkdir() + failed_log = failed_dir / "plans/nightly/nightly-rhel9/data/results" / "fail.log" + (failed_dir / "tmt_failed").write_text("failed") + failed_log.parent.mkdir(parents=True) + failed_log.write_text("failed") + + report.item_dir = report.reports_dir + report.iter_results_in_directory() + output = capsys.readouterr().out + + assert "Running TMT plans" in output + assert "rhel9-test-pytest" in output + assert "Success TMT plans" in output + assert "c9s-test" in output + assert "Failed TMT plans" in output + assert "rhel9-test" in output + assert "Failed container tests" in output + assert "fail.log" in output diff --git a/daily_tests/tox.ini b/daily_tests/tox.ini new file mode 100644 index 0000000..9366ad0 --- /dev/null +++ b/daily_tests/tox.ini @@ -0,0 +1,5 @@ +[testenv] +commands = python3 -m pytest --color=yes -v +deps = + pytest + PyYAML