diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 6a0a89ba..030d29fd 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -153,3 +153,22 @@ jobs: run: tox run --skip-pkg-install -e ${{ matrix.tox_env }} env: UV_PYTHON_PREFERENCE: only-managed + + changelog: + name: "πŸ“ News fragment" + runs-on: ubuntu-24.04 + # A PR must add a docs/changelog/..rst fragment; automated dependency and pre-commit bumps are exempt. + if: >- + github.event_name == 'pull_request' && !contains(fromJSON('["dependabot[bot]", "pre-commit-ci[bot]"]'), + github.actor) + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + - name: "πŸ”„ Install the latest version of uv" + uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + with: + enable-cache: false + - name: "πŸ“ Require a news fragment" + run: uvx towncrier check --compare-with "origin/${GITHUB_BASE_REF}" diff --git a/.github/workflows/prepare-release.yaml b/.github/workflows/prepare-release.yaml new file mode 100644 index 00000000..81efc238 --- /dev/null +++ b/.github/workflows/prepare-release.yaml @@ -0,0 +1,76 @@ +name: πŸ› οΈ Prepare release +on: + workflow_dispatch: + inputs: + bump: + description: "Version bump" + required: true + default: auto + type: choice + options: [auto, major, minor, patch] + +permissions: + contents: read + +jobs: + prepare-release: + name: πŸ“ build changelog and tag + runs-on: ubuntu-24.04 + # RELEASE_TOKEN is an environment-scoped secret; selecting the environment resolves it so the checkout can push. + environment: + name: release + permissions: + contents: write + steps: + - name: πŸ“₯ Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 # full history so the version calculation sees the latest tag + token: ${{ secrets.RELEASE_TOKEN }} # zizmor: ignore[artipacked] + - name: πŸ“¦ Setup uv + uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + with: + enable-cache: false + - name: πŸ”’ Compute next version + id: version + env: + BUMP: ${{ inputs.bump }} + run: | + current=$(git tag --list --sort=-v:refname | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | head -1) + current=${current:-0.0.0} + IFS='.' read -r major minor patch <<< "$current" + bump="$BUMP" + if [ "$bump" = "auto" ]; then + if ls docs/changelog/*.breaking.rst >/dev/null 2>&1; then + bump=major + elif ls docs/changelog/*.feature.rst >/dev/null 2>&1; then + bump=minor + else + bump=patch + fi + fi + case "$bump" in + major) next="$((major + 1)).0.0" ;; + minor) next="$major.$((minor + 1)).0" ;; + patch) next="$major.$minor.$((patch + 1))" ;; + esac + echo "Bumping $current -> $next ($bump)" + echo "version=$next" >> "$GITHUB_OUTPUT" + - name: πŸ“ Build changelog, commit, and tag + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + if git ls-remote --tags origin "refs/tags/$VERSION" | grep -q .; then + echo "Tag $VERSION already exists, nothing to do" + exit 1 + fi + git config user.name "${GITHUB_ACTOR}" + git config user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" + uvx towncrier build --yes --version "$VERSION" + git add docs/changelog docs/changelog.rst + git commit -m "Release $VERSION" + git tag "$VERSION" + # Push the branch first, then the tag. The tag is pushed with RELEASE_TOKEN (a PAT), so it triggers the + # release workflow on the tag ref, where the build is verified against the changelog and published to PyPI. + git push origin HEAD:main + git push origin "$VERSION" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a07bd850..543f50e4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,24 +1,13 @@ name: πŸš€ Release on: - workflow_dispatch: - inputs: - release: - description: "πŸš€ Release (semver bump)?" - required: true - default: "no" - type: choice - options: ["no", patch, minor, major] push: branches: ["main"] tags: ["[0-9]+.[0-9]+.[0-9]+"] pull_request: concurrency: - group: >- - ${{ github.workflow }}-${{ github.ref }}${{ - github.event.inputs.release && github.event.inputs.release != 'no' && '-release' || '' - }} - cancel-in-progress: ${{ github.event.inputs.release == 'no' || github.event.inputs.release == null }} + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }} permissions: contents: read @@ -32,37 +21,16 @@ jobs: runs-on: ubuntu-24.04 permissions: contents: read - pull-requests: read - outputs: - version: ${{ steps.v.outputs.version }} - changelog: ${{ steps.v.outputs.changelog }} steps: - name: πŸ“₯ Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - fetch-depth: 0 + fetch-depth: 0 # full history so hatch-vcs derives the version from the tag persist-credentials: false - name: πŸ“¦ Setup uv uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 with: enable-cache: false # this workflow builds the artifact that gets published; no cache to poison - - name: πŸ“ Generate changelog - id: v - if: ${{ !startsWith(github.ref, 'refs/tags/') }} # on a tag push we build to publish, no changelog needed - run: >- - uv run tasks/changelog.py "$RELEASE_TYPE" "$EVENT_NUMBER" "$BASE_SHA" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RELEASE_TYPE: - ${{ github.event.inputs.release == 'no' || github.event.inputs.release == null && 'patch' || - github.event.inputs.release }} - EVENT_NUMBER: ${{ github.event.number }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - - name: 🏷️ Create temporary tag for hatch-vcs - if: github.event.inputs.release != 'no' && github.event.inputs.release != null - run: git tag "$STEPS_V_OUTPUTS_VERSION" - env: - STEPS_V_OUTPUTS_VERSION: ${{ steps.v.outputs.version }} - name: πŸ“¦ Build package run: uv build --python 3.14 --python-preference only-managed --sdist --wheel . --out-dir dist - name: πŸ“€ Store the distribution packages @@ -71,63 +39,32 @@ jobs: name: ${{ env.dists-artifact-name }} path: dist/* - release: - name: 🏷️ tag + publish: + name: πŸš€ publish needs: [build] - if: github.event.inputs.release != 'no' && github.event.inputs.release != null && github.ref == 'refs/heads/main' + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-24.04 - # RELEASE_TOKEN is an environment-scoped secret; without selecting the environment it resolves empty and the - # checkout fails with "Input required and not supplied: token". environment: name: release + url: https://pypi.org/project/filelock/${{ github.ref_name }} permissions: + id-token: write contents: write steps: - name: πŸ“₯ Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - token: ${{ secrets.RELEASE_TOKEN }} # zizmor: ignore[artipacked] - - name: πŸ“€ Commit changelog and tag + persist-credentials: false + - name: πŸ›‘οΈ Require a changelog section for this release env: - CHANGELOG: ${{ needs.build.outputs.changelog }} - VERSION: ${{ needs.build.outputs.version }} + VERSION: ${{ github.ref_name }} + # A tag reaches PyPI only through prepare-release, which builds the changelog first. A tag pushed by hand skips + # that, so refuse to publish a version the changelog does not document rather than ship it silently. run: | - if git ls-remote --tags origin "refs/tags/$VERSION" | grep -q .; then - echo "Tag $VERSION already exists, nothing to do" - exit 0 + if ! grep -qE "^ ${VERSION//./\\.} \(" docs/changelog.rst; then + echo "::error::docs/changelog.rst has no section for $VERSION; release via the Prepare release workflow" + exit 1 fi - git config user.name "${GITHUB_ACTOR}" - git config user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" - header="$VERSION ($(date -u +%Y-%m-%d))" - separator=$(printf '%0.s*' $(seq 1 $((${#header} + 1)))) - { - head -4 docs/changelog.rst - printf '%s\n %s\n%s\n\n' "$separator" "$header" "$separator" - printf '%s\n\n' "$CHANGELOG" - tail -n +5 docs/changelog.rst - } > docs/changelog.tmp - mv docs/changelog.tmp docs/changelog.rst - git add docs/changelog.rst - git commit -m "Release $VERSION" - git tag "$VERSION" - # Push the branch, then the tag. The tag is pushed with RELEASE_TOKEN (a PAT, not GITHUB_TOKEN), so it - # triggers this same workflow on the tag ref, where the publish job below uploads to PyPI. Publishing keys - # off the tag, not this dispatch run, so any tag β€” including one pushed by hand β€” always reaches PyPI. - git push origin main - git push origin "$VERSION" - - publish: - name: πŸš€ publish - needs: [build] - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-24.04 - environment: - name: release - url: https://pypi.org/project/filelock/${{ github.ref_name }} - permissions: - id-token: write - contents: write - steps: - name: ⬇️ Download all the dists uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: diff --git a/docs/changelog.rst b/docs/changelog.rst index 4a2aa45b..49ef679e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,41 @@ Changelog ########### +.. towncrier-draft-entries:: Unreleased + +.. towncrier release notes start + +******************** + 3.29.7 (2026-07-07) +******************** + +- _util: drop the dead st_mtime=0 short-circuit in raise_on_not_writable_file :pr:`589` - by :user:`HrachShah` +- soft_rw: evict a non-regular write marker without reading it :pr:`588` +- asyncio: detect cross-instance reentrant deadlocks before the poll loop :pr:`586` - by :user:`HrachShah` + +******************** + 3.29.6 (2026-07-06) +******************** + +- test: silence fork DeprecationWarning on 3.15 :pr:`585` +- _util: drop the dead st_mtime=0 short-circuit in raise_on_not_writable_file :pr:`582` - by :user:`HrachShah` +- serialize singleton construction in FileLockMeta :pr:`581` - by :user:`dxbjavid` +- surface GitHub Sponsors and thanks.dev + +******************** + 3.29.5 (2026-07-02) +******************** + +- lifetime: reject negative, non-numeric, and bool values at the setter :pr:`573` - by :user:`HrachShah` +- roll back a read acquire's open transaction when its SELECT fails :pr:`575` - by :user:`dxbjavid` +- keep Unix lock files after release :pr:`577` - by :user:`itscloud0` +- use a private break name in break_lock_file :pr:`576` - by :user:`dxbjavid` +- don't complete a writer acquire on a peer's reclaimed marker :pr:`571` - by :user:`dxbjavid` +- don't follow symlinks in raise_on_not_writable_file :pr:`567` - by :user:`dxbjavid` +- only unlink the writer marker on release if it is still ours :pr:`566` - by :user:`dxbjavid` +- don't follow symlinks when refreshing soft read/write lock markers :pr:`565` - by :user:`dxbjavid` +- serialize read/write release rollback against a concurrent acquire :pr:`563` - by :user:`dxbjavid` + ******************** 3.29.4 (2026-06-13) ******************** diff --git a/docs/changelog/593.bugfix.rst b/docs/changelog/593.bugfix.rst new file mode 100644 index 00000000..bd8c234b --- /dev/null +++ b/docs/changelog/593.bugfix.rst @@ -0,0 +1 @@ +Ignore ``lifetime`` on native locks with a warning; only ``SoftFileLock`` honors it. diff --git a/docs/changelog/594.bugfix.rst b/docs/changelog/594.bugfix.rst new file mode 100644 index 00000000..1995e26b --- /dev/null +++ b/docs/changelog/594.bugfix.rst @@ -0,0 +1 @@ +Stop mutating the Unix lock file before ``flock`` is held. diff --git a/docs/changelog/596.bugfix.rst b/docs/changelog/596.bugfix.rst new file mode 100644 index 00000000..9e7b0e0b --- /dev/null +++ b/docs/changelog/596.bugfix.rst @@ -0,0 +1 @@ +Bind the Windows reparse-point check to the locked handle to close a symlink-swap race. diff --git a/docs/changelog/597.bugfix.rst b/docs/changelog/597.bugfix.rst new file mode 100644 index 00000000..971e23cc --- /dev/null +++ b/docs/changelog/597.bugfix.rst @@ -0,0 +1 @@ +Evict a non-regular soft lock file without reading it. diff --git a/docs/changelog/614.bugfix.rst b/docs/changelog/614.bugfix.rst new file mode 100644 index 00000000..6f9bea20 --- /dev/null +++ b/docs/changelog/614.bugfix.rst @@ -0,0 +1 @@ +Write a soft lock's holder record atomically so a short write cannot create overlapping holders. diff --git a/docs/changelog/615.bugfix.rst b/docs/changelog/615.bugfix.rst new file mode 100644 index 00000000..b557e44b --- /dev/null +++ b/docs/changelog/615.bugfix.rst @@ -0,0 +1 @@ +Make native lock release transactional so a failed unlock is retried, not dropped. diff --git a/docs/changelog/617.bugfix.rst b/docs/changelog/617.bugfix.rst new file mode 100644 index 00000000..7aff5559 --- /dev/null +++ b/docs/changelog/617.bugfix.rst @@ -0,0 +1 @@ +Open the Windows lock file via ``NtCreateFile`` so a real access denial is raised, not reported as a timeout. diff --git a/docs/changelog/618.feature.rst b/docs/changelog/618.feature.rst new file mode 100644 index 00000000..f438334c --- /dev/null +++ b/docs/changelog/618.feature.rst @@ -0,0 +1 @@ +Add ``context_error_policy`` to surface body and release failures as a ``BaseExceptionGroup``. diff --git a/docs/changelog/619.feature.rst b/docs/changelog/619.feature.rst new file mode 100644 index 00000000..3cd29d04 --- /dev/null +++ b/docs/changelog/619.feature.rst @@ -0,0 +1 @@ +Add ``close_error_policy`` to control an ``os.close`` failure after the OS unlock. diff --git a/docs/changelog/620.feature.rst b/docs/changelog/620.feature.rst new file mode 100644 index 00000000..9263854d --- /dev/null +++ b/docs/changelog/620.feature.rst @@ -0,0 +1 @@ +Add ``lock_descriptor`` and ``unlock_descriptor`` to lock a caller-owned file descriptor. diff --git a/docs/changelog/621.bugfix.rst b/docs/changelog/621.bugfix.rst new file mode 100644 index 00000000..a78e4695 --- /dev/null +++ b/docs/changelog/621.bugfix.rst @@ -0,0 +1 @@ +Canonicalize singleton keys so equivalent path spellings share an instance, without following a final symlink. diff --git a/docs/changelog/622.feature.rst b/docs/changelog/622.feature.rst new file mode 100644 index 00000000..751a0a0f --- /dev/null +++ b/docs/changelog/622.feature.rst @@ -0,0 +1 @@ +Add ``fallback_to_soft`` to fail closed instead of downgrading to ``SoftFileLock`` on ``ENOSYS``. diff --git a/docs/changelog/623.doc.rst b/docs/changelog/623.doc.rst new file mode 100644 index 00000000..84630708 --- /dev/null +++ b/docs/changelog/623.doc.rst @@ -0,0 +1 @@ +Correct the Unix lock-file cleanup and ``flock`` documentation. diff --git a/docs/changelog/624.feature.rst b/docs/changelog/624.feature.rst new file mode 100644 index 00000000..e3c751d4 --- /dev/null +++ b/docs/changelog/624.feature.rst @@ -0,0 +1 @@ +Add ``preserve_lock_file`` so native locks keep the lock pathname on release. diff --git a/docs/changelog/625.feature.rst b/docs/changelog/625.feature.rst new file mode 100644 index 00000000..fffecf47 --- /dev/null +++ b/docs/changelog/625.feature.rst @@ -0,0 +1 @@ +Add an ``on_acquired`` hook that runs a callback on the locked descriptor once the lock is held. diff --git a/docs/changelog/626.packaging.rst b/docs/changelog/626.packaging.rst new file mode 100644 index 00000000..254e8ee1 --- /dev/null +++ b/docs/changelog/626.packaging.rst @@ -0,0 +1 @@ +Build the changelog from towncrier news fragments, render pending fragments as an ``Unreleased`` docs section, and refuse to publish a tag the changelog does not document. diff --git a/docs/changelog/template.jinja2 b/docs/changelog/template.jinja2 new file mode 100644 index 00000000..5efd5d35 --- /dev/null +++ b/docs/changelog/template.jinja2 @@ -0,0 +1,12 @@ +{% set header = versiondata.version + " (" + versiondata.date + ")" %} +{{ "*" * (header | length + 1) }} + {{ header }} +{{ "*" * (header | length + 1) }} + +{% for section, _ in sections.items() %} +{% for category, val in definitions.items() if category in sections[section] %} +{% for text, values in sections[section][category].items() %} +- {{ text }}{{ (" " + values | join(", ")) if values else "" }} +{% endfor %} +{% endfor %} +{% endfor %} diff --git a/docs/concepts.rst b/docs/concepts.rst index 46353783..a9074cfa 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -82,7 +82,7 @@ acquires raise :class:`Timeout ` as usual. How file locking works ************************ -There are two fundamentally different approaches to file locking on modern systems: +File locking takes two approaches: **OS-level locking** (FileLock on Windows/Unix, UnixFileLock, WindowsFileLock) @@ -94,7 +94,7 @@ file handle can lock it. When a process dies, the OS automatically releases its - - Pros - Cons - - - βœ“ Enforced by the kernel, so it is foolproof. + - - βœ“ Enforced by the kernel. - βœ— Exclusive only, with no reader/writer distinction (use ReadWriteLock for that). - - βœ“ Works even if your process crashes. - βœ— Unreliable on some network filesystems. @@ -110,8 +110,8 @@ A separate "lock file" indicates that a resource is in use. The lock file contai deleting it. This is the same concept as a traditional **PID lock file** (as used by Unix daemons and the deprecated `lockfile `_ -library's ``PIDLockFile``). The PID stored in the file enables two important capabilities: identifying the lock holder -via the :attr:`~filelock.SoftFileLock.pid` property, and detecting stale locks when the holding process has died. +library's ``PIDLockFile``). The PID stored in the file identifies the lock holder via the +:attr:`~filelock.SoftFileLock.pid` property and lets a waiter detect stale locks when the holding process has died. On all platforms, processes can check if the lock holder is still alive and break stale locks automatically. On Windows, the lock file additionally stores the process creation time to guard against PID recycling. @@ -133,16 +133,18 @@ the lock file additionally stores the process creation time to guard against PID *************************** **Windows** - Uses the :class:`WindowsFileLock ` class, backed by ``msvcrt.locking``. This is enforced - by Windows, so all code running on the system respects it, whether it uses filelock or not. + Uses the :class:`WindowsFileLock ` class, backed by ``LockFileEx``/``UnlockFileEx`` over a + one-byte range on a handle opened with ``NtCreateFile``. This is enforced by Windows, so all code running on the + system respects it, whether it uses filelock or not. The lock is exclusive and works reliably on local filesystems. Network filesystem (SMB) support is available but considered less reliable. Lock file cleanup: Windows attempts to delete the lock file after release, but deletion is not guaranteed in multi-threaded scenarios. Windows cannot delete files with open handles, so if another thread acquires the lock - before the previous holder finishes cleanup, the lock file persists. This is by design and does not affect lock - correctness. + before the previous holder finishes cleanup, the lock file persists. It does not affect lock correctness. To keep a + stable pathname across releases, construct the lock with ``preserve_lock_file=True`` (see :ref:`how-to:Keep the lock + file on release`). **Unix and macOS** Uses the :class:`UnixFileLock ` class, backed by the kernel's ``fcntl.flock`` interface. The @@ -257,7 +259,7 @@ Lock types compared - No - No (content is not a public API) - - Lifetime expiration - - Yes + - No (ignored with a warning) - Yes - No - Yes (``heartbeat_interval`` / ``stale_threshold``) @@ -335,8 +337,8 @@ On older platforms without ``O_NOFOLLOW``, prefer :class:`UnixFileLock `. For **reader/writer locking** (shared readers + exclusive writers) on NFS, use :class:`SoftReadWriteLock `, @@ -422,7 +424,7 @@ check plus a dirfd-relative open to close symlink races (which ``mkdir`` alone c Writer acquisition is two-phase and writer-preferring: phase one atomically claims ``.write`` (which blocks any new reader as soon as it exists), phase two polls the ``readers/`` directory until every reader has exited. -Writer starvation is impossible, which matters under read-heavy workloads such as the 99/1 reader-to-writer mix +Writer starvation is impossible even under read-heavy workloads such as the 99/1 reader-to-writer mix typical of slurm job queues. Cross-host stale detection @@ -443,7 +445,7 @@ uses a **TTL with a heartbeat** rather than ``SoftFileLock``'s PID-alive check: never gets accidentally refreshed. The trade-off: ``stale_threshold`` must be larger than any realistic pause a holder might hit (GC, syscall delay, -NFS hiccup). Pick it generously. We assume clock synchronization across compute nodes; every HPC cluster runs +NFS hiccup). Pick it generously. We assume clock synchronization across compute nodes; HPC clusters run NTP or chrony, so this is not an additional constraint in the target environment. Fork semantics diff --git a/docs/conf.py b/docs/conf.py index a85fa729..304e41a8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,6 +4,7 @@ from __future__ import annotations from datetime import datetime, timezone +from pathlib import Path from typing import TYPE_CHECKING from docutils.nodes import Element, Text, reference @@ -32,10 +33,16 @@ "sphinx_design", "sphinx_sitemap", "sphinxcontrib.mermaid", + "sphinxcontrib.towncrier.ext", # render unreleased news fragments as a draft "Unreleased" section "sphinxext.opengraph", ] mermaid_output_format = "raw" +# sphinxcontrib-towncrier: the .. towncrier-draft-entries:: directive in changelog.rst shows the unreleased fragments. +towncrier_draft_autoversion_mode = "draft" +towncrier_draft_include_empty = True # keep the docs build green right after a release, when no fragments remain +towncrier_draft_working_directory = Path(__file__).parent.parent + # sphinx-copybutton: add copy button to code blocks copybutton_prompt_is_regexp = True copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: " @@ -57,13 +64,14 @@ html_title, html_last_updated_fmt = name, now.isoformat() html_baseurl = "https://py-filelock.readthedocs.io/" pygments_style, pygments_dark_style = "sphinx", "monokai" -autoclass_content, autodoc_member_order, autodoc_typehints = "class", "bysource", "none" +autoclass_content, autodoc_member_order, autodoc_typehints = "both", "bysource", "none" autodoc_default_options = {"member-order": "bysource", "undoc-members": True, "show-inheritance": True} autosectionlabel_prefix_document = True intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} nitpicky = True -nitpick_ignore = [] +# Towncrier news fragments are rST snippets folded into changelog.rst at release; they are not standalone pages. +exclude_patterns = ["changelog/*"] extlinks = { "issue": ("https://github.com/tox-dev/py-filelock/issues/%s", "issue #%s"), "pr": ("https://github.com/tox-dev/py-filelock/issues/%s", "PR #%s"), diff --git a/docs/how-to.rst b/docs/how-to.rst index 287bc8a7..06f57e1b 100644 --- a/docs/how-to.rst +++ b/docs/how-to.rst @@ -212,7 +212,7 @@ Acquiring through one reference counts toward the same lock depth: .. code-block:: python lock_a.acquire() - lock_b.acquire() # Reentrantβ€”lock counter is now 2 + lock_b.acquire() # reentrant, lock counter is now 2 lock_b.release() # Lock counter is 1 lock_a.release() # Lock is fully released @@ -349,8 +349,8 @@ its ``LeaseKeepAlive``. Lower ``poll_interval`` reduces acquire latency under co NFS ``stat`` calls per waiting client. Writer acquisition is two-phase and writer-preferring: phase one claims the writer marker (which blocks any -new reader), phase two waits for existing readers to drain. This rules out writer starvation even under a -read-heavy workload like the 99/1 reader-to-writer mix typical of slurm job queues. +new reader), phase two waits for existing readers to drain. This rules out writer starvation under read-heavy +workloads. See :doc:`concepts` for the full model. **Fork caveat.** A process that forks while holding a ``SoftReadWriteLock`` loses the lock in the child. filelock marks the inherited instance fork-invalidated; ``release()`` on it becomes a no-op, and the child must call @@ -505,21 +505,27 @@ library: Set lock lifetime ******************* -You can create locks that expire after a set time: +Only :class:`SoftFileLock ` honors ``lifetime``. A waiting process breaks a lock file whose +modification time is older than ``lifetime`` seconds, even if the holder is still alive: .. code-block:: python - from filelock import FileLock + from filelock import SoftFileLock # Lock expires after 3600 seconds (1 hour) - lock = FileLock("work.lock", lifetime=3600) + lock = SoftFileLock("work.lock", lifetime=3600) with lock: - # Lock is held, but will auto-expire after 1 hour + # Lock is held; a waiter may break it once the lock file is older than 1 hour pass -This is useful for distributed systems where a process might crash and leave a lock behind. After the lifetime expires, -other processes can acquire it. +This helps distributed systems where a process might crash and leave a lock behind. After the lifetime expires, other +processes can acquire it. + +Native locks (:class:`FileLock `, :class:`UnixFileLock `, +:class:`WindowsFileLock `) ignore a non-``None`` ``lifetime`` and emit a warning. A kernel +lock lives on the inode, so unlinking the pathname by age cannot revoke it, and a contender would lock a fresh inode +while the holder is still live. Use ``SoftFileLock`` when you need age-based expiry. ************************** Cancel lock acquisition @@ -584,7 +590,7 @@ regardless of the counter, pass ``force=True``: print(lock.lock_counter) # 2 lock.release(force=True) - print(lock.is_locked) # False β€” fully released + print(lock.is_locked) # False, fully released This is useful in error recovery or cleanup handlers where you need to ensure the lock is fully released: @@ -633,3 +639,146 @@ acquire with ``blocking=False``: print("Lock was free") except Timeout: print("Lock is held by another process") + +*************************************** + Reconcile body and release failures +*************************************** + +When a ``with`` block fails and releasing the lock on exit also fails, Python's default keeps the body error in the +release error's ``__context__``. That buries one error inside the other. Set ``context_error_policy="group"`` to raise +both as siblings of a :class:`BaseExceptionGroup`, body first, release second: + +.. code-block:: python + + from filelock import FileLock + + lock = FileLock("work.lock", context_error_policy="group") + + with lock: + raise RuntimeError("body failed") + # if release() then also fails, both surface as a BaseExceptionGroup + +``"group"`` needs Python 3.11+ or the ``exceptiongroup`` backport; filelock checks this when you construct the lock. +When both errors subclass :class:`Exception`, the group is a plain :class:`ExceptionGroup`, so ``except*`` and +``except Exception`` still catch it. The default ``"chain"`` keeps Python's behavior. + +***************************************** + Handle a close failure after unlock +***************************************** + +Native locks close the lock descriptor after the OS unlock has committed. On FUSE and Docker overlay filesystems +``os.close`` can fail with ``EIO`` even though the lock already released. ``close_error_policy`` decides the outcome: + +- ``"default"`` keeps each platform's historical behavior: Unix drops the error, Windows propagates it. +- ``"raise"`` always propagates the ``OSError``. +- ``"suppress"`` always ignores it. + +.. code-block:: python + + from filelock import FileLock + + lock = FileLock("work.lock", close_error_policy="suppress") + +The lock is released in every case; the policy only governs the post-unlock ``os.close``. It does not affect unlock +failures or lock-file deletion. + +*********************************************** + Fail closed instead of downgrading to soft +*********************************************** + +On Unix, when the filesystem's ``flock`` returns ``ENOSYS`` (some network mounts), :class:`FileLock +` switches to :class:`SoftFileLock ` semantics by default, trading +kernel-enforced locking for cooperative existence locking. If your code needs kernel enforcement, pass +``fallback_to_soft=False`` so the ``ENOSYS`` propagates instead of a silent downgrade to soft locking: + +.. code-block:: python + + from filelock import FileLock + + lock = FileLock("work.lock", fallback_to_soft=False) + +It has no effect on Windows or :class:`SoftFileLock `. + +********************************* + Keep the lock file on release +********************************* + +Native backends handle the lock pathname differently: Windows unlinks it after release, Unix leaves it in place. Pass +``preserve_lock_file=True`` for a stable file identity across releases, which matters for ACLs, auditing, or holder +metadata written through :ref:`on_acquired `: + +.. code-block:: python + + from filelock import FileLock + + lock = FileLock("work.lock", preserve_lock_file=True) + +Windows then skips its post-release unlink, and Unix refuses the ``ENOSYS`` soft fallback (which releases by +unlinking). :class:`SoftFileLock ` rejects ``True`` because unlinking its marker is how it +releases. The promise covers filelock's own release path; it cannot stop another process or the filesystem from +removing the file. + +***************************************** + Run a callback once the lock is held +***************************************** + +``on_acquired`` runs once per physical acquisition, after filelock holds the native lock and finished backend +initialization, before :meth:`acquire() ` returns. filelock passes the borrowed lock +descriptor. The callback may read, +write, seek, truncate, or set metadata through ``os`` on it, but must not close, unlock, or take ownership of the +descriptor. A recursive acquire does not call it again. If the callback raises, filelock releases the lock and +re-raises. + +A common use is stamping holder metadata into the lock file: + +.. code-block:: python + + import json + import os + import socket + + from filelock import FileLock + + + def write_holder(fd: int) -> None: + os.ftruncate(fd, 0) + os.lseek(fd, 0, os.SEEK_SET) + os.write(fd, json.dumps({"pid": os.getpid(), "host": socket.gethostname()}).encode()) + + + lock = FileLock("work.lock", on_acquired=write_holder, preserve_lock_file=True) + + with lock: + ... # while held, other processes can read the holder metadata from work.lock + +filelock does not ``fsync`` the descriptor's writes. Native locks only; :class:`SoftFileLock +` rejects the hook because it keeps its own protocol state in the marker file. + +******************************************* + Lock a descriptor you already own +******************************************* + +:func:`lock_descriptor ` and :func:`unlock_descriptor ` take and +release the same one-byte native lock :class:`FileLock ` uses, but on a file descriptor you opened +and own. They add no path handling: no open, truncate, close, unlink, chmod, canonicalize, or fallback. A descriptor +lock and a ``FileLock`` path lock on the same file contend with each other. + +.. code-block:: python + + import os + + from filelock import lock_descriptor, unlock_descriptor + + fd = os.open("work.lock", os.O_RDWR | os.O_CREAT) + try: + lock_descriptor(fd) # blocks until the lock is held + try: + ... # critical section + finally: + unlock_descriptor(fd) + finally: + os.close(fd) + +Pass ``blocking=False`` for a single attempt that returns ``False`` on contention. There is no async wrapper: run it in +an executor, or drive ``blocking=False`` from your own polling loop. On Windows *fd* must be a synchronous descriptor. +For timeout, reentrancy, singleton, lifetime, or stale-break behavior, use :class:`FileLock `. diff --git a/docs/index.rst b/docs/index.rst index 990ad6e0..137c277c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -65,10 +65,10 @@ Choose the right lock for your use case: .. grid-item-card:: **FileLock** - Platform-aware alias. Uses OS-level locking (``fcntl``/``msvcrt``) with automatic fallback to soft locks. + Platform-aware alias. Uses OS-level locking (``fcntl``/``LockFileEx``) with automatic fallback to soft locks. - βœ“ Recommended default - - βœ“ Lifetime expiration, cancellable acquire + - βœ“ Cancellable acquire - βœ“ Self-deadlock detection .. grid-item-card:: @@ -118,7 +118,7 @@ Choose the right lock for your use case: .. grid-item-card:: **Windows** - Uses ``msvcrt.locking``. Enforced by the kernel. + Uses ``LockFileEx`` on a byte range. Enforced by the kernel. - βœ“ Native support - βœ“ Most reliable @@ -129,7 +129,7 @@ Choose the right lock for your use case: Uses ``fcntl.flock``. Common on Unix but not POSIX; kernel-enforced. - βœ“ Native support - - βœ“ Stale detection + - βœ“ Released automatically on crash .. grid-item-card:: **Other Platforms** diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 67344209..c4fd9e35 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -57,7 +57,7 @@ File locks are most useful when protecting data that multiple processes access: with data_file.open("a") as f: f.write("Hello from Process B\n") -The key pattern: **before making changes, check what's already done.** Process A checks whether the file exists before +Before making changes, check what is already done. Process A checks whether the file exists before writing. Process B appends, so it skips the check. Both use the lock so that only one process modifies the file at a time. @@ -104,11 +104,11 @@ inspect this counter and the lock state at any time: print(lock.lock_counter) # 2 lock.release() - print(lock.lock_counter) # 1 β€” still locked + print(lock.lock_counter) # 1, still locked print(lock.is_locked) # True lock.release() - print(lock.lock_counter) # 0 β€” fully released + print(lock.lock_counter) # 0, fully released print(lock.is_locked) # False You can call functions that acquire a lock even while you already hold it. diff --git a/pyproject.toml b/pyproject.toml index 67ba8d5f..d5fb84d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,9 @@ docs = [ "sphinx-design>=0.7", "sphinx-sitemap>=2.9", "sphinxcontrib-mermaid>=2.0.1", + "sphinxcontrib-towncrier>=0.5.0a0", "sphinxext-opengraph>=0.13", + "towncrier>=24.8", ] coverage = [ "covdefaults>=2.3", @@ -96,9 +98,7 @@ pkg-meta = [ "uv>=0.11.5", ] release = [ - "gitpython>=3.1.46", - "pygithub>=2.9", - "pyyaml>=6.0.3", + "towncrier>=24.8", ] [tool.hatch] @@ -211,3 +211,18 @@ paths.source = [ report.fail_under = 76 html.show_contexts = true html.skip_covered = false + +[tool.towncrier] +name = "filelock" +directory = "docs/changelog" +filename = "docs/changelog.rst" +template = "docs/changelog/template.jinja2" +title_format = false +issue_format = ":pr:`{issue}`" +type = [ + { directory = "breaking", name = "Backward incompatible changes", showcontent = true }, + { directory = "feature", name = "Features", showcontent = true }, + { directory = "bugfix", name = "Bug fixes", showcontent = true }, + { directory = "doc", name = "Improved documentation", showcontent = true }, + { directory = "packaging", name = "Packaging updates", showcontent = true }, +] diff --git a/src/filelock/_api.py b/src/filelock/_api.py index f4f109b9..5102f985 100644 --- a/src/filelock/_api.py +++ b/src/filelock/_api.py @@ -364,10 +364,10 @@ def __init__( # noqa: PLR0913 it cannot stop another process or the filesystem from removing the pathname. :param on_acquired: for native locks (:class:`FileLock`), a callable invoked with the borrowed lock descriptor once per physical acquisition, after filelock holds the native lock and finished backend initialization but - before :meth:`acquire` returns. Recursive acquisitions do not call it again. The callback may read, write, - seek, truncate, or set metadata through ``os`` on the descriptor, but must not close, unlock, or take - ownership of it, and filelock does not fsync its writes. If it raises, filelock releases the lock and - re-raises. :class:`SoftFileLock` rejects the hook. + before :meth:`~BaseFileLock.acquire` returns. Recursive acquisitions do not call it again. The callback may + read, write, seek, truncate, or set metadata through ``os`` on the descriptor, but must not close, unlock, + or take ownership of it, and filelock does not fsync its writes. If it raises, filelock releases the lock + and re-raises. :class:`SoftFileLock` rejects the hook. """ self._is_thread_local = thread_local @@ -408,7 +408,7 @@ def context_error_policy(self) -> ContextErrorPolicy: """ How a context manager reconciles a body failure with a release failure on exit. - .. versionadded:: 3.27.0 + .. versionadded:: 3.30.0 """ return self._context_error_policy @@ -418,7 +418,7 @@ def close_error_policy(self) -> CloseErrorPolicy: """ What a native lock does with an ``os.close`` failure after the OS unlock committed. - .. versionadded:: 3.27.0 + .. versionadded:: 3.30.0 """ return self._close_error_policy @@ -443,7 +443,7 @@ def fallback_to_soft(self) -> bool: Only :class:`UnixFileLock` acts on it: when ``False`` an ``ENOSYS`` from ``flock`` propagates instead of switching to existence-lock semantics. - .. versionadded:: 3.27.0 + .. versionadded:: 3.30.0 """ return self._fallback_to_soft @@ -456,7 +456,7 @@ def preserve_lock_file(self) -> bool: When ``True``, Windows skips its post-release unlink and Unix refuses the ``ENOSYS`` soft fallback. :class:`SoftFileLock` rejects ``True`` because unlinking its marker is how it releases. - .. versionadded:: 3.27.0 + .. versionadded:: 3.30.0 """ return self._preserve_lock_file @@ -467,9 +467,9 @@ def on_acquired(self) -> Callable[[int], None] | None: The callback run with the borrowed lock descriptor once per physical acquisition, or ``None``. Native locks only. It runs after the native lock is held and backend initialization finished, before - :meth:`acquire` returns; a raise rolls back the acquisition. :class:`SoftFileLock` rejects it. + :meth:`~BaseFileLock.acquire` returns; a raise rolls back the acquisition. :class:`SoftFileLock` rejects it. - .. versionadded:: 3.27.0 + .. versionadded:: 3.30.0 """ return self._on_acquired @@ -937,4 +937,6 @@ class ThreadLocalFileContext(FileLockContext, local): "_UNSET_FILE_MODE", "AcquireReturnProxy", "BaseFileLock", + "CloseErrorPolicy", + "ContextErrorPolicy", ] diff --git a/src/filelock/_descriptor.py b/src/filelock/_descriptor.py index 5c4ac38c..f8711700 100644 --- a/src/filelock/_descriptor.py +++ b/src/filelock/_descriptor.py @@ -32,6 +32,8 @@ def lock_descriptor(fd: int, *, blocking: bool = True, poll_interval: float = 0. :raises OSError: for a permanent native failure, such as an invalid descriptor. The descriptor is left open. + .. versionadded:: 3.30.0 + """ if not blocking: return _lock_fd_nonblocking(fd) @@ -48,6 +50,8 @@ def unlock_descriptor(fd: int) -> None: :raises OSError: if the native unlock fails; the caller may retry on the same descriptor. + .. versionadded:: 3.30.0 + """ _unlock_fd(fd) diff --git a/src/filelock/_windows.py b/src/filelock/_windows.py index c3eafc56..b26d8ba3 100644 --- a/src/filelock/_windows.py +++ b/src/filelock/_windows.py @@ -223,7 +223,7 @@ def _open_non_reparse_fd(path: str, mode: int) -> int | None: caller's timeout (#604). The reparse option only guards the final path component; Windows still follows reparse points in intermediate - directories. This assumes the lock file sits in a lock directory untrusted users cannot modify β€” a path with + directories. This assumes the lock file sits in a lock directory untrusted users cannot modify. A path with attacker-controlled parent directories would need component-by-component handle validation. :param path: the lock file path. @@ -282,7 +282,6 @@ def _nt_open(path: str, *, read_only: bool) -> tuple[int, int]: attributes.Attributes = _OBJ_CASE_INSENSITIVE handle = wintypes.HANDLE() io_status = _IO_STATUS_BLOCK() - file_attributes = _FILE_ATTRIBUTE_READONLY if read_only else _FILE_ATTRIBUTE_NORMAL status = ( _ntdll.NtCreateFile( ctypes.byref(handle), @@ -290,7 +289,7 @@ def _nt_open(path: str, *, read_only: bool) -> tuple[int, int]: ctypes.byref(attributes), ctypes.byref(io_status), None, - file_attributes, + _FILE_ATTRIBUTE_READONLY if read_only else _FILE_ATTRIBUTE_NORMAL, _FILE_SHARE_READ_WRITE, _FILE_OPEN_IF, _CREATE_OPTIONS, diff --git a/src/filelock/asyncio.py b/src/filelock/asyncio.py index 349abc1e..481d2367 100644 --- a/src/filelock/asyncio.py +++ b/src/filelock/asyncio.py @@ -159,9 +159,10 @@ def __init__( # noqa: PLR0913 identity (Windows skips its unlink, Unix refuses the ``ENOSYS`` soft fallback). :class:`AsyncSoftFileLock` rejects ``True``. :param on_acquired: for native locks (:class:`AsyncFileLock`), a callable invoked with the borrowed lock - descriptor once per physical acquisition, after the lock is held but before :meth:`acquire` returns. With - ``run_in_executor=True`` (the default) it runs in the backend executor. It must not close or unlock the - descriptor; a raise rolls the acquisition back. :class:`AsyncSoftFileLock` rejects it. + descriptor once per physical acquisition, after the lock is held but before + :meth:`~BaseAsyncFileLock.acquire` returns. With ``run_in_executor=True`` (the default) it runs in the + backend executor. It must not close or unlock the descriptor; a raise rolls the acquisition back. + :class:`AsyncSoftFileLock` rejects it. :param loop: The event loop to use. If not specified, the running event loop will be used. :param run_in_executor: If this is set to ``True`` then the lock will be acquired in an executor. :param executor: The executor to use. If not specified, the default executor will be used. @@ -363,7 +364,7 @@ async def _run_internal_method(self, method: Callable[[], Any]) -> None: def __enter__(self) -> NoReturn: """Sync context manager entry is not supported because lock acquisition is a coroutine.""" - msg = "Use `async with` β€” acquire/release are coroutines and cannot be awaited in a sync context manager." + msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager." raise NotImplementedError(msg) def __exit__( @@ -373,7 +374,7 @@ def __exit__( traceback: object, ) -> None: """Sync context manager exit is not supported because lock release is a coroutine.""" - msg = "Use `async with` β€” acquire/release are coroutines and cannot be awaited in a sync context manager." + msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager." raise NotImplementedError(msg) async def __aenter__(self) -> Self: @@ -414,7 +415,7 @@ async def _release_in_context( # ty: ignore[invalid-method-override] _raise_body_and_release(body_error, release_error) def __del__(self) -> None: - """Release on deletion β€” safe to call during GC even when no event loop is running.""" + """Release on deletion; safe to call during GC even when no event loop is running.""" with contextlib.suppress(Exception): try: loop = asyncio.get_running_loop() diff --git a/tasks/changelog.py b/tasks/changelog.py deleted file mode 100644 index a73edbce..00000000 --- a/tasks/changelog.py +++ /dev/null @@ -1,130 +0,0 @@ -# /// script -# requires-python = ">=3.12" -# dependencies = [ -# "gitpython>=3.1.46", -# "pygithub>=2.8.1", -# "pyyaml>=6.0.2", -# ] -# /// -"""Generate the changelog on release.""" - -from __future__ import annotations - -import os -import re -from argparse import ArgumentParser, Namespace -from pathlib import Path -from typing import TYPE_CHECKING - -import yaml -from git import Repo -from github import Github -from github.Auth import Token - -if TYPE_CHECKING: - from collections.abc import Iterator - - from github.Repository import Repository as GitHubRepository - -ROOT = Path(__file__).parents[1] -REPO_SLUG = "tox-dev/py-filelock" - - -class Options(Namespace): - bump: str - pr: int | None - base: str - - -def run() -> None: - options = parse_cli() - git_repo = Repo(ROOT) - github = Github(auth=Token(os.environ["GITHUB_TOKEN"])) - gh_repo = github.get_repo(REPO_SLUG) - excluded_authors = _load_excluded_authors() - - last_version = latest_tag_version(git_repo) - version = compute_next_version(last_version, options.bump) - - logs = [] - for title, pr_number, by in entries(gh_repo, git_repo, options.pr, options.base, excluded_authors): - suffix = f" :pr:`{pr_number}`" if pr_number else "" - by_suffix = f" - by :user:`{by}`" if by != "gaborbernat" else "" - logs.append(f"- {title}{suffix}{by_suffix}") - - changelog_text = "\n".join(logs) if logs else "- No notable changes" - - if output := os.environ.get("GITHUB_OUTPUT"): - with Path(output).open("at+", encoding="utf-8") as file_handler: - file_handler.write(f"version={version}\n") - file_handler.write(f"changelog< set[str]: - release_config = ROOT / ".github" / "release.yml" - if release_config.exists(): - with release_config.open(encoding="utf-8") as file_handler: - config = yaml.safe_load(file_handler) - return set(config.get("changelog", {}).get("exclude", {}).get("authors", [])) - return set() - - -def parse_cli() -> Options: - parser = ArgumentParser() - parser.add_argument("bump", choices=["patch", "minor", "major"]) - parser.add_argument("pr", type=lambda value: int(value) if value else None, nargs="?", default=None) - parser.add_argument("base", type=str, nargs="?", default="") - options = Options() - parser.parse_args(namespace=options) - return options - - -def latest_tag_version(git_repo: Repo) -> str: - tags = sorted( - (tag for tag in git_repo.tags if re.match(r"^\d+\.\d+\.\d+$", tag.name)), - key=lambda tag: tag.commit.committed_datetime, - ) - if not tags: - return "0.0.0" - return tags[-1].name - - -def compute_next_version(current: str, bump: str) -> str: - major, minor, patch = (int(part) for part in current.split(".")) - if bump == "major": - return f"{major + 1}.0.0" - if bump == "minor": - return f"{major}.{minor + 1}.0" - return f"{major}.{minor}.{patch + 1}" - - -def entries( - gh_repo: GitHubRepository, git_repo: Repo, pr_number: int | None, base: str, excluded_authors: set[str] -) -> Iterator[tuple[str, str, str]]: - if pr_number: - pull = gh_repo.get_pull(pr_number) - yield pull.title, str(pr_number), pull.user.login - - tags = {tag.commit.hexsha for tag in git_repo.tags if re.match(r"^\d+\.\d+\.\d+$", tag.name)} - pr_re = re.compile(r"(?P.*)[(]#(?P<pr>\d+)[)]") - release_re = re.compile(r"^Release \d+\.\d+\.\d+") - found_base = not base - for change in git_repo.iter_commits(): - if change.hexsha in tags: - break - commit_title = str(change.message).split("\n")[0].strip() - if release_re.match(commit_title): - break - found_base = found_base or change.hexsha == base - if not found_base or change.author.name in excluded_authors: - continue - by = gh_repo.get_commit(change.hexsha).author.login - if match := pr_re.match(commit_title): - group = match.groupdict() - yield group["title"].strip(), group["pr"], by - else: - yield commit_title, "", by - - -if __name__ == "__main__": - run() diff --git a/tox.toml b/tox.toml index 4feba420..68bce303 100644 --- a/tox.toml +++ b/tox.toml @@ -145,3 +145,9 @@ description = "dev environment with all deps at {envdir}" package = "editable" dependency_groups = [ "dev" ] commands = [ [ "uv", "pip", "list", "--format=columns" ], [ "python", "-c", 'print(r"{env_python}")' ] ] + +[env.release] +description = "build the changelog from the news fragments for the given version, e.g. tox r -e release -- 3.30.0" +skip_install = true +dependency_groups = [ "release" ] +commands = [ [ "towncrier", "build", "--yes", "--version", "{posargs}" ] ]