From a7eb1817ad1eeb1adb6c5cefcc789cb46810b4c7 Mon Sep 17 00:00:00 2001 From: Johannes Raggam Date: Mon, 28 Sep 2026 12:29:43 +0200 Subject: [PATCH] Add include-optional directive. - Add ``include-optional`` for INI configuration files that may be absent, skipping missing local files and HTTP 404 responses while preserving mandatory includes. This allows projects to define an optional include for client-specific customizations. - Fix HTTP error logging during INI inclusion and preserve the original exception. - Fix relative INI includes from URLs with a directory path. Co-authored-by: Codex --- CHANGES.md | 6 +++ README.md | 26 ++++++++++++ src/mxdev/including.py | 88 ++++++++++++++++++++++++++--------------- tests/test_including.py | 82 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 32 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9c87120..339229d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,12 @@ +- Add `include-optional` for INI configuration files that may be absent, skipping + missing local files and HTTP 404 responses while preserving mandatory includes. + This allows projects to define an optional include for client-specific + customizations. +- Fix HTTP error logging during INI inclusion and preserve the original exception. +- Fix relative INI includes from URLs with a directory path. ## 5.4.1 (2026-08-04) diff --git a/README.md b/README.md index c03e1e4..f0741fa 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,32 @@ If an included file is an HTTP-URL, it is loaded from there. If the included file is a relative path, it is loaded relative to the parent's directory or URL. +Missing files and HTTP errors stop configuration loading. Use `include-optional` for files that may be absent. + +Default: empty + +##### `include-optional` + +Optional includes allow projects to define a client-specific configuration file +via a file name defined by a convention - e.g. to make customer- or +developer-specific adaptions which should not be checked into a repository. + +Include one or more optional INI files, one per line: + +```ini +[settings] +include-optional = + mx-custom.ini +``` + +Missing local files and HTTP 404 responses are ignored. Other errors, such as HTTP 403/500, +connection failures, or invalid INI content, still stop configuration loading. +Mandatory `include` entries inside an existing optional file remain mandatory. + +Paths and URLs are resolved in the same way as `include`, and optional files may include other files. +For each file, mandatory includes are read first, then optional includes in their listed order, +then the file itself. Later settings override earlier settings, so the main file takes precedence. + Default: empty ##### `directory` diff --git a/src/mxdev/including.py b/src/mxdev/including.py index f88e83e..ab2ef8c 100644 --- a/src/mxdev/including.py +++ b/src/mxdev/including.py @@ -1,8 +1,10 @@ +from .logging import logger from configparser import ConfigParser from configparser import ExtendedInterpolation from pathlib import Path from urllib import parse from urllib import request +from urllib.error import HTTPError import os import tempfile @@ -12,14 +14,14 @@ def resolve_dependencies( file_or_url: str | Path, tmpdir: str, http_parent=None, + optional: bool = False, ) -> list[Path]: """Resolve dependencies of a file or url - The result is a list of Path objects, starting with the - given file_or_url and followed by all file_or_urls referenced from it. + Return included paths before their parent, so parent settings take precedence. - The file_or_url is assumed to be a ini file or url to such, with an option key "include" - under the "[settings]" section. + Follow "include" and "include-optional" under the "[settings]" section. + If optional, skip a missing file or HTTP 404 for this input only. """ if isinstance(file_or_url, str): if http_parent: @@ -29,40 +31,59 @@ def resolve_dependencies( # Windows drive letters are single characters, URL schemes are longer is_url = parsed.scheme and len(parsed.scheme) > 1 if is_url: - with request.urlopen(str(file_or_url)) as fio: - tf = tempfile.NamedTemporaryFile( - suffix=".ini", - dir=str(tmpdir), - delete=False, - ) - tf.write(fio.read()) - tf.flush() - file = Path(tf.name) - parts = list(parsed) - parts[2] = str(Path(parts[2]).parent) - http_parent = parse.urlunparse(parts) + try: + with request.urlopen(str(file_or_url)) as fio: + with tempfile.NamedTemporaryFile( + suffix=".ini", + dir=str(tmpdir), + delete=False, + ) as tf: + tf.write(fio.read()) + file = Path(tf.name) + except HTTPError as e: + if optional and e.code == 404: + logger.info("Skipping missing optional include: %s", file_or_url) + return [] + logger.error("Error %s for URL: %s", e.code, e.url) + raise + http_parent = parse.urljoin(str(file_or_url), ".") else: file = Path(file_or_url) else: file = file_or_url - if not file.exists(): - raise FileNotFoundError(file) cfg = ConfigParser() - cfg.read(file) - if not ("settings" in cfg and "include" in cfg["settings"]): + try: + with file.open() as fio: + cfg.read_file(fio) + except FileNotFoundError: + if not optional: + raise + logger.info("Skipping missing optional include: %s", file_or_url) + return [] + if "settings" not in cfg: return [file] file_list = [] - for include in cfg["settings"]["include"].split("\n"): - include = include.strip() - if not include: - continue - # Check if it's a real URL scheme (not a Windows drive letter) - parsed_include = parse.urlparse(include) - is_include_url = parsed_include.scheme and len(parsed_include.scheme) > 1 - if http_parent or is_include_url: - file_list += resolve_dependencies(include, tmpdir, http_parent) - else: - file_list += resolve_dependencies(file.parent / include, tmpdir) + for directive in ("include", "include-optional"): + for include in cfg["settings"].get(directive, "").splitlines(): + include = include.strip() + if not include: + continue + # Check if it's a real URL scheme (not a Windows drive letter) + parsed_include = parse.urlparse(include) + is_include_url = parsed_include.scheme and len(parsed_include.scheme) > 1 + if http_parent or is_include_url: + file_list += resolve_dependencies( + file_or_url=include, + tmpdir=tmpdir, + http_parent=http_parent, + optional=directive == "include-optional", + ) + else: + file_list += resolve_dependencies( + file_or_url=file.parent / include, + tmpdir=tmpdir, + optional=directive == "include-optional", + ) file_list.append(file) return file_list @@ -80,6 +101,9 @@ def read_with_included(file_or_url: str | Path) -> ConfigParser: cfg.optionxform = str # type: ignore cfg["settings"]["directory"] = os.getcwd() with tempfile.TemporaryDirectory() as tmpdir: - resolved = resolve_dependencies(file_or_url, tmpdir) + resolved = resolve_dependencies( + file_or_url=file_or_url, + tmpdir=tmpdir, + ) cfg.read(resolved) return cfg diff --git a/tests/test_including.py b/tests/test_including.py index d4fe482..4fd2e15 100644 --- a/tests/test_including.py +++ b/tests/test_including.py @@ -2,6 +2,88 @@ import pytest +def test_optional_includes_and_precedence(tmp_path): + from mxdev.including import read_with_included + + (tmp_path / "required.ini").write_text("[settings]\nvalue = required\n") + (tmp_path / "nested.ini").write_text("[settings]\nnested = yes\n") + (tmp_path / "custom.ini").write_text( + "[settings]\ninclude = nested.ini\ninclude-optional = absent.ini\nvalue = optional\nmain = optional\n" + ) + main = tmp_path / "mx.ini" + main.write_text( + "[settings]\ninclude = required.ini\ninclude-optional =\n missing.ini\n custom.ini\nmain = main\n" + ) + cfg = read_with_included(main) + assert cfg["settings"]["value"] == "optional" + assert cfg["settings"]["main"] == "main" + assert cfg["settings"]["nested"] == "yes" + + +def test_optional_include_preserves_required_nested_failure(tmp_path): + from mxdev.including import read_with_included + + main = tmp_path / "mx.ini" + main.write_text("[settings]\ninclude-optional = custom.ini\n") + (tmp_path / "custom.ini").write_text("[settings]\ninclude = missing.ini\n") + with pytest.raises(FileNotFoundError): + read_with_included(main) + + +@pytest.mark.parametrize("directive", ["include", "include-optional"]) +@pytest.mark.parametrize("status", [404, 403, 500]) +def test_include_http_errors(tmp_path, httpretty, directive, status): + from mxdev.including import read_with_included + from urllib.error import HTTPError + + url = "http://example.com/custom.ini" + httpretty.register_uri(httpretty.GET, url, status=status) + main = tmp_path / "mx.ini" + main.write_text(f"[settings]\n{directive} = {url}\nvalue = main\n") + if directive == "include-optional" and status == 404: + assert read_with_included(main)["settings"]["value"] == "main" + else: + with pytest.raises(HTTPError) as exc: + read_with_included(main) + assert exc.value.code == status + + +@pytest.mark.parametrize("directive", ["include", "include-optional"]) +def test_http_relative_include(tmp_path, httpretty, directive): + from mxdev.including import read_with_included + + url = "http://example.com/config/mx.ini" + httpretty.register_uri(httpretty.GET, url, body=f"[settings]\n{directive} = custom.ini\n") + httpretty.register_uri(httpretty.GET, "http://example.com/config/custom.ini", body="[settings]\ncustom = yes\n") + assert read_with_included(url)["settings"]["custom"] == "yes" + + +def test_optional_include_connection_error(tmp_path, monkeypatch): + from mxdev.including import read_with_included + from urllib.error import URLError + + main = tmp_path / "mx.ini" + main.write_text("[settings]\ninclude-optional = https://example.com/custom.ini\n") + + def fail(url): + raise URLError("Connection failed") + + monkeypatch.setattr("mxdev.including.request.urlopen", fail) + with pytest.raises(URLError, match="Connection failed"): + read_with_included(main) + + +def test_optional_include_invalid_ini(tmp_path): + from configparser import MissingSectionHeaderError + from mxdev.including import read_with_included + + main = tmp_path / "mx.ini" + main.write_text("[settings]\ninclude-optional = custom.ini\n") + (tmp_path / "custom.ini").write_text("invalid ini\n") + with pytest.raises(MissingSectionHeaderError): + read_with_included(main) + + def test_resolve_dependencies_files(): from mxdev.including import resolve_dependencies