diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea7a62de745..6d3049aa16b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -411,14 +411,20 @@ system_tests_param: python3 -c " import json, shlex with open('raw_params.txt') as f: - for line in f: - line = line.strip() - if '=' not in line: continue - key, val = line.split('=', 1) - val = json.loads(val) - if key == 'libraries': print('export LIBRARIES=' + shlex.quote(str(val))) - elif key == 'scenarios': print('export SCENARIOS=' + shlex.quote(str(val))) - elif key == 'scenarios_groups': print('export SCENARIO_GROUPS=' + shlex.quote(str(val))) + params = dict(line.strip().split('=', 1) for line in f if '=' in line) + params = {key: json.loads(val) for key, val in params.items()} + if 'library_weblog_scenarios' in params and 'libraries' not in params: + # alongside library_weblog_scenarios, build_pipeline.py needs libraries for scenarios/scenarios_groups + params.pop('scenarios', None) + params.pop('scenarios_groups', None) + variables = { + 'LIBRARIES': 'libraries', + 'SCENARIOS': 'scenarios', + 'SCENARIO_GROUPS': 'scenarios_groups', + 'LIBRARY_WEBLOG_SCENARIOS': 'library_weblog_scenarios', + } + for variable, key in variables.items(): + if key in params: print('export ' + variable + '=' + shlex.quote(str(params[key]))) " > set_vars.sh source set_vars.sh - export EXCLUDED_SCENARIOS='DEBUGGER_EXPRESSION_LANGUAGE' diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index 1e050546b28..c138902dfdd 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -624,9 +624,12 @@ def test_agentless_ffe_mocked_backend_file(self, file: str): "file", [ "utils/mocked_backend/__init__.py", + "utils/mocked_backend/__main__.py", "utils/mocked_backend/_backend.py", + "utils/mocked_backend/_client.py", "utils/mocked_backend/_request_handler.py", "utils/mocked_backend/_utils.py", + "utils/mocked_backend/tls/mocked-backend.pem", ], ) def test_generic_mocked_backend_file(self, file: str): @@ -740,13 +743,15 @@ def test_pr_pipeline_selects_c(self, monkeypatch: pytest.MonkeyPatch, source: st assert 'libraries="c"' in process(inputs) - def test_pr_pipeline_excludes_python(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_main_push_ignores_weblog_map(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GITLAB_CI", "true") - monkeypatch.setenv("CI_PIPELINE_SOURCE", "merge_request_event") - monkeypatch.setenv("CI_COMMIT_REF_NAME", "feature/python") + monkeypatch.setenv("CI_PIPELINE_SOURCE", "push") + monkeypatch.setenv("CI_COMMIT_REF_NAME", "main") inputs = build_inputs(modified_files=["utils/build/docker/python/flask-poc.Dockerfile"]) - assert not any(line.startswith("libraries=") for line in process(inputs)) + output = process(inputs) + assert 'libraries="python"' in output + assert not any(line.startswith("library_weblog_scenarios=") for line in output) @pytest.mark.parametrize( ("source", "ref"), diff --git a/tests/test_the_test/test_gitlab_library_weblog_scenarios.py b/tests/test_the_test/test_gitlab_library_weblog_scenarios.py index 975a30903da..e80a4fa50fd 100644 --- a/tests/test_the_test/test_gitlab_library_weblog_scenarios.py +++ b/tests/test_the_test/test_gitlab_library_weblog_scenarios.py @@ -143,16 +143,35 @@ def test_pipeline_variable_overrides_input(tmp_path: Path) -> None: ("args", "environ"), [ (["--libraries", "python"], {}), + ([], {"LIBRARIES": "java python"}), (["--scenarios", "DEFAULT"], {}), ([], {"SCENARIO_GROUPS": "appsec"}), ([], {"WEBLOGS": "flask-poc"}), ], ) def test_not_compatible_with_other_selection_inputs(tmp_path: Path, args: list[str], environ: dict) -> None: + """Other selection inputs need libraries, which must not be in library_weblog_scenarios""" mapping = '{"python": {"flask-poc": ["DEFAULT"]}}' assert _main(tmp_path, "--library-weblog-scenarios", mapping, *args, environ=environ) == 1 +@scenarios.test_the_test +def test_combined_with_other_selection_inputs(tmp_path: Path) -> None: + """libraries/scenarios/weblogs apply to the other libraries, library_weblog_scenarios to its own libraries""" + mapping = '{"python": {"flask-poc": ["APPSEC_BLOCKING"]}}' + args = ["--libraries", "java", "--scenarios", "DEFAULT", "--weblogs", "spring-boot,flask-poc"] + assert _main(tmp_path, "--library-weblog-scenarios", mapping, *args) == 0 + + pipeline = yaml.safe_load((tmp_path / "generated-pipeline-chunk-0.yml").read_text()) + jobs = {name for name in pipeline if name.startswith(("system_tests_build_", "system_tests_run_"))} + assert jobs == { + "system_tests_build_java_spring-boot", + "system_tests_run_java_DEFAULT_spring-boot", + "system_tests_build_python_flask-poc", + "system_tests_run_python_APPSEC_BLOCKING_flask-poc", + } + + @scenarios.test_the_test @pytest.mark.parametrize( ("library_weblog_scenarios", "error"), diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 78a378f8c1e..8130b6c8f74 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -507,7 +507,7 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: "--library-weblog-scenarios", default="", help='JSON object {library: {weblog: [scenario or group, ...]}}, e.g. {"python": {"flask-poc": ["DEFAULT"]}}. ' - "Alternative to --libraries/--scenarios/--scenario-groups/--weblogs", + "Can be combined with --libraries/--scenarios/--scenario-groups/--weblogs, for other libraries", ) return parser.parse_args(argv) @@ -531,16 +531,24 @@ def main(argv: list[str] | None = None, environ: Mapping[str, str] | None = None output_dir = Path(inputs.output_dir) output_dir.mkdir(parents=True, exist_ok=True) - mapping = None + mapping: dict[str, dict[str, list[str]]] = {} if inputs.library_weblog_scenarios: - if inputs.libraries or inputs.scenarios or inputs.scenario_groups or inputs.weblogs: + # libraries/scenarios/scenarios_groups/weblogs may be used alongside library_weblog_scenarios, for other + # libraries than the ones of library_weblog_scenarios + if (inputs.scenarios or inputs.scenario_groups or inputs.weblogs) and not inputs.libraries: print( # noqa: T201 - "❌ library_weblog_scenarios is not compatible with libraries, scenarios, scenarios_groups and weblogs", + "❌ scenarios, scenarios_groups and weblogs need libraries when used with library_weblog_scenarios", file=sys.stderr, ) return 1 mapping = parse_library_weblog_scenarios(inputs.library_weblog_scenarios) - inputs.libraries = list(mapping) + if both := sorted(set(inputs.libraries) & set(mapping)): + print( # noqa: T201 + f"❌ {', '.join(both)}: listed in both libraries and library_weblog_scenarios", + file=sys.stderr, + ) + return 1 + inputs.libraries = [*inputs.libraries, *mapping] for name in (*MERGED_LIST_VARIABLES, *OVERRIDE_VARIABLES): value = getattr(inputs, name) @@ -553,10 +561,10 @@ def main(argv: list[str] | None = None, environ: Mapping[str, str] | None = None if inputs.params_dir is None: params_dir = output_dir for library in inputs.libraries: - if mapping is None: - compute_params(library, inputs, params_dir) - else: + if library in mapping: compute_mapped_params(library, mapping[library], inputs, params_dir) + else: + compute_params(library, inputs, params_dir) else: params_dir = Path(inputs.params_dir) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index a430f0c860f..b6fc3be3062 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -23,8 +23,9 @@ spec: (uppercase) and/or scenario groups (lowercase), e.g. '{"python": {"flask-poc": ["DEFAULT", "appsec"], "django-poc": ["DEFAULT"]}}'. PARAMETRIC is not weblog-bound, it is enabled with the "parametric" key, e.g. '{"python": {"parametric": []}}'. - Alternative to libraries/scenarios/scenarios_groups/weblogs (not compatible with them); - excluded_scenarios only applies to scenarios coming from groups. + Can be combined with libraries/scenarios/scenarios_groups/weblogs, which then apply to the other libraries + (a library cannot be in both). + In library_weblog_scenarios, excluded_scenarios only applies to scenarios coming from groups. GitLab limits inputs interpolation to 1 KB: for larger mappings, set the LIBRARY_WEBLOG_SCENARIOS variable in your system_tests_param job (extending .system_tests_param_base) instead; it takes precedence over this input. default: "" diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index 5b06c11bbeb..d88a1277c27 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -45,7 +45,22 @@ def _get_scenarios_by_definition_file() -> dict[str, set[str]]: OTEL_LIBRARIES = COMPONENT_GROUPS.otel - {"nodejs_otel"} # nodejs_otel intentionally excluded ALL_LIBRARIES = LIBRARIES | OTEL_LIBRARIES GITHUB_EXCLUDED_LIBRARIES = {"c"} -GITLAB_PR_LIBRARIES = {"c"} + +# Libraries run by GitLab end-to-end on PRs. The value is either None, to run the scenarios selected from the diff +# on every weblog of the library, or a {weblog: [scenario or scenario group, ...]} map, to run only these +# weblogs/scenarios when the library is activated. +GITLAB_PR_LIBRARIES: dict[str, dict[str, list[str]] | None] = { + "c": None, + # GitLab runs containers on the default bridge network: checks that the mocked backend network aliases work there + "python": { + "flask-poc": [ + "APM_TRACING_E2E_SINGLE_SPAN", + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS", + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_DIRECT", + "FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS_SERVERLESS", + ] + }, +} GITLAB_MAIN = {"python"} @@ -190,17 +205,60 @@ def get_outputs(self) -> dict[str, Any]: } +def is_gitlab_pr_pipeline(inputs: Inputs) -> bool: + return inputs.ref != "refs/heads/main" and inputs.event_name in ("pull_request", "push") + + def filter_gitlab_libraries(inputs: Inputs, libraries: set[str]) -> set[str]: """Limit the GitLab end-to-end rollout by pipeline context.""" if inputs.ref == "refs/heads/main" and inputs.event_name != "schedule": return libraries & GITLAB_MAIN - if inputs.event_name in ("pull_request", "push"): - return libraries & GITLAB_PR_LIBRARIES + if is_gitlab_pr_pipeline(inputs): + return libraries & set(GITLAB_PR_LIBRARIES) return set() +def _expand_scenario_entries(entries: Iterable[str]) -> set[str]: + """Scenario names of a list of scenarios (uppercase) and scenario groups (lowercase)""" + result: set[str] = set() + for entry in entries: + if entry.islower(): + result |= {scenario.name for scenario in all_scenario_groups[entry].scenarios} + else: + result.add(entry) + return result + + +def get_gitlab_library_weblog_scenarios( + libraries: set[str], scenario_names: set[str], scenario_group_names: set[str] +) -> dict[str, dict[str, list[str]]]: + """Return the library_weblog_scenarios map of the GitLab PR pipeline, for the libraries with a weblog map. + + The weblog maps of GITLAB_PR_LIBRARIES are filtered by the scenarios selected from the diff: only selected + scenarios are kept, and weblogs (and libraries) with no scenario left are removed. + """ + selected = _expand_scenario_entries(scenario_names | scenario_group_names) + + result: dict[str, dict[str, list[str]]] = {} + for library in sorted(libraries): + weblog_map = GITLAB_PR_LIBRARIES[library] + if not weblog_map: + continue + filtered_map: dict[str, list[str]] = {} + for weblog, weblog_entries in sorted(weblog_map.items()): + if weblog == "parametric": + if scenarios.parametric.name in selected: + filtered_map[weblog] = [] + elif weblog_scenarios := sorted(_expand_scenario_entries(weblog_entries) & selected): + filtered_map[weblog] = weblog_scenarios + if filtered_map: + result[library] = filtered_map + + return result + + class ScenarioProcessor: def __init__(self, scenario_groups: set[str] | None = None): self.scenario_groups = scenario_groups if scenario_groups else set() @@ -460,9 +518,19 @@ def process(inputs: Inputs) -> list[str]: library_processor.selected |= scenario_processor.impacted_libraries if inputs.is_gitlab: - libraries = " ".join(sorted(filter_gitlab_libraries(inputs, library_processor.selected))) - if libraries: - outputs["libraries"] = libraries + gitlab_libraries = filter_gitlab_libraries(inputs, library_processor.selected) + if is_gitlab_pr_pipeline(inputs): + library_weblog_scenarios = get_gitlab_library_weblog_scenarios( + gitlab_libraries, scenario_processor.scenarios, scenario_processor.scenario_groups + ) + if library_weblog_scenarios: + # JSON in a JSON string, so the output can still be sourced by a shell + outputs["library_weblog_scenarios"] = json.dumps(library_weblog_scenarios) + # libraries with a weblog map are only run through library_weblog_scenarios + gitlab_libraries = {library for library in gitlab_libraries if not GITLAB_PR_LIBRARIES[library]} + if gitlab_libraries: + outputs["libraries"] = " ".join(sorted(gitlab_libraries)) + # also used by the SSI pipeline (compute_pipeline job) outputs |= scenario_processor.get_outputs() else: outputs |= (