Skip to content

Environment markers with different sources for a dependency #8303

New issue
New issue
Closed
#8304
Closed
Environment markers with different sources for a dependency#8303
#8304
Labels
area/installerRelated to the dependency installerkind/bugSomething isn't working as expected

Description

@sassanh
sassanh
opened on Aug 12, 2023
  • Poetry version: 1.5.1
  • Python version: 3.11.4
  • OS version and name: macOS Sonoma 14.0
  • pyproject.toml: -
  • I am on the latest stable Poetry version, installed using a recommended method.
  • I have searched the issues of this repo and believe that this is not a duplicate.
  • I have consulted the FAQ and blog for any relevant entries or release notes.
  • If an exception occurs when executing a command, I executed it again in debug mode (-vvv option) and have included the output below.

Issue

I have read #7300, #5205 and #6710. I discussed it in Discord but didn't get any answer. The issue is if I need to install a package from different sources based on the platform, poetry doesn't behave as expected.
So imagine I have this dependency:

[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"

[tool.poetry.dependencies]
kivy = [
  { version = "^2.2.1", markers = "platform_machine == 'aarch64'", source = "piwheels" },
  { version = "^2.2.1", markers = "platform_machine != 'aarch64'", source = "PyPI" }
]

I expect it to install from piwheels on RPi and install from PyPI on my macOS, but it doesn't. It creates this in lock file:

[[package]]
name = "kivy"
version = "2.2.1"
description = "A software library for rapid development of hardware-accelerated multitouch applications."
optional = false
python-versions = ">=3.7"
files = [
    {file = "Kivy-2.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5aa8a0ca0708dd6edb0127c91712db7b8aad3aea3a79d498df145b8a7ee0aee6"},
    {file = "Kivy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af45f433208c16a4d0e755fafac5f2238560880a475a8a041ca9d99695a6b166"},
    {file = "Kivy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:564705e8fbd788dce22dae8dfe0488be705e14381fa59d6bdfa3df5255446429"},
    {file = "Kivy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:19cd0b0b40046d6233f0a30c8003e1ffe5a135b96ae50a5442bd62ef9d9c6744"},
    {file = "Kivy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:4915e931cb26efa016956b16a0d04bcc8eb89abcb2734ed718786c30241795bb"},
    {file = "Kivy-2.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e3df4f6230f4b125c78b1dd923f714e93341afda5b55ae9795df2d6b2c5dbc4d"},
    {file = "Kivy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0eb5de5ffea53a341790ff7ba4d3cea60d37523f0766a4783692e2aa1d50670e"},
    {file = "Kivy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a5e292f232a67bc67dfa91a205aed472dc4df7329ce059c74a88031c652efa"},
    {file = "Kivy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:d78a8d92565c6309bb1edc9a0c99883630adc0503dba25f3eb8b667ceb736395"},
    {file = "Kivy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:e317a710e0be3083901d29457fa8a203c14a71b7cef8008e3cdc7d1eb8e4d7c5"},
    {file = "Kivy-2.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7b8c8725cd1b3575b95f8f8b5d6f804e096af8958156450bb876821fbb2ce37"},
    {file = "Kivy-2.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b89bc5add63bf83fd1b026ec4d492ea9c90067b30652c14afe97cd4c30a2b5a"},
    {file = "Kivy-2.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f19ed8efaea7c880ab6342269c41ac47989812cbd036a524abc49ba9923487f"},
    {file = "Kivy-2.2.1-cp37-cp37m-win32.whl", hash = "sha256:c932fa9a8ade002ec4468c416bedda4188cda53721be84e4086b278c3b00c654"},
    {file = "Kivy-2.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7eeda699fe764c5efa2cc43e0ef28275be2a56ccd8af17933b6e2457d66d5b0d"},
    {file = "Kivy-2.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0456359afe01a94d39bb29ce8a8dd9d9dcd86cec6532f17bffb0725647a8d63a"},
    {file = "Kivy-2.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37874a7a2a9983ae494910578e2e6a4aba1568b35f915b014d85f779b5add8b9"},
    {file = "Kivy-2.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7bfbc97f66d50d71bcbc5c254e74386d94d7a1fb051448fbc1ba6742c1d1d21"},
    {file = "Kivy-2.2.1-cp38-cp38-win32.whl", hash = "sha256:0454ad509b83f5ee04dd7bb46682f077adcde99d86c206127f2dd57aedd28f5b"},
    {file = "Kivy-2.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:a6b47a8b5b47d091ac4604635edef7522ddfb9cf47b3f258cccd8ffbf094b429"},
    {file = "Kivy-2.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0e8370d529548d360803424f803231ed8ad19c7f4c351aa893fe9d39bcc940fc"},
    {file = "Kivy-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1e9cf59dc3875c943c89d3668a73235c96f4c03680c4691416ccc5d1c7f50f0"},
    {file = "Kivy-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97442522fb58ebf4038f3c57a795046e00a23f7c4dc0775b282af9aff845732a"},
    {file = "Kivy-2.2.1-cp39-cp39-win32.whl", hash = "sha256:f3dcfdce8403dc3df7a8b1a7d424c0939d41bc48a0b627403b685931a609a7ec"},
    {file = "Kivy-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:35eb3025a77894380196d5d64852da6c539e72aa2ff4970011385eb7a70c46e0"},
    {file = "Kivy-2.2.1.tar.gz", hash = "sha256:047a434e8efd32d425321e065fcccf725f165adb0c81ed69dc33c360d4796f7e"},
]

[package.dependencies]
docutils = "*"
"kivy-deps.angle" = {version = ">=0.3.3,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.glew" = {version = ">=0.3.1,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.sdl2" = {version = ">=0.6.0,<0.7.0", markers = "sys_platform == \"win32\""}
Kivy-Garden = ">=0.1.4"
pygments = "*"
pypiwin32 = {version = "*", markers = "sys_platform == \"win32\""}

[package.extras]
angle = ["kivy-deps.angle (>=0.3.3,<0.4.0)"]
base = ["docutils", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32", "requests"]
dev = ["flake8", "funcparserlib (==1.0.0a0)", "kivy-deps.glew-dev (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer-dev (>=0.3.3,<0.4.0)", "kivy-deps.sdl2-dev (>=0.6.0,<0.7.0)", "pre-commit", "pyinstaller", "pytest (>=3.6)", "pytest-asyncio (!=0.11.0)", "pytest-benchmark", "pytest-cov", "pytest-timeout", "responses", "sphinx (<=6.2.1)", "sphinxcontrib-actdiag", "sphinxcontrib-blockdiag", "sphinxcontrib-jquery", "sphinxcontrib-nwdiag", "sphinxcontrib-seqdiag"]
full = ["docutils", "ffpyplayer", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32"]
glew = ["kivy-deps.glew (>=0.3.1,<0.4.0)"]
gstreamer = ["kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
media = ["ffpyplayer", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
sdl2 = ["kivy-deps.sdl2 (>=0.6.0,<0.7.0)"]
tuio = ["oscpy"]

[[package]]
name = "kivy"
version = "2.2.1"
description = "A software library for rapid development of hardware-accelerated multitouch applications."
optional = false
python-versions = ">=3.7"
files = [
    {file = "Kivy-2.2.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:f0312dc2b154f2730acaaac1fb9b60081dc55026b2c861fd1d9e34834b5d71ff"},
    {file = "Kivy-2.2.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:f0312dc2b154f2730acaaac1fb9b60081dc55026b2c861fd1d9e34834b5d71ff"},
    {file = "Kivy-2.2.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:61e5dd7ea9d03a6ed8dcd735328049ca1ccb5d7ed5db8db6215b8106115e832a"},
    {file = "Kivy-2.2.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:61e5dd7ea9d03a6ed8dcd735328049ca1ccb5d7ed5db8db6215b8106115e832a"},
]

[package.dependencies]
docutils = "*"
"kivy-deps.angle" = {version = ">=0.3.3,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.glew" = {version = ">=0.3.1,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.sdl2" = {version = ">=0.6.0,<0.7.0", markers = "sys_platform == \"win32\""}
Kivy-Garden = ">=0.1.4"
pygments = "*"
pypiwin32 = {version = "*", markers = "sys_platform == \"win32\""}

[package.extras]
angle = ["kivy-deps.angle (>=0.3.3,<0.4.0)"]
base = ["docutils", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32", "requests"]
dev = ["flake8", "funcparserlib (==1.0.0a0)", "kivy-deps.glew-dev (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer-dev (>=0.3.3,<0.4.0)", "kivy-deps.sdl2-dev (>=0.6.0,<0.7.0)", "pre-commit", "pyinstaller", "pytest (>=3.6)", "pytest-asyncio (!=0.11.0)", "pytest-benchmark", "pytest-cov", "pytest-timeout", "responses", "sphinx (<=6.2.1)", "sphinxcontrib-actdiag", "sphinxcontrib-blockdiag", "sphinxcontrib-jquery", "sphinxcontrib-nwdiag", "sphinxcontrib-seqdiag"]
full = ["docutils", "ffpyplayer", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32"]
glew = ["kivy-deps.glew (>=0.3.1,<0.4.0)"]
gstreamer = ["kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
media = ["ffpyplayer", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
sdl2 = ["kivy-deps.sdl2 (>=0.6.0,<0.7.0)"]
tuio = ["oscpy"]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

Note that markers are completely ignored and dropped in the lock file and the package is mentioned two times.

Now this lock file works on RPi but doesn't work on macOS, if I change the order of sources in my pyproject.toml it will affect the order of packages in the lock file as well and then it will work on macOS but not in RPi.

Activity

  1. sassanh
    added
    kind/bugSomething isn't working as expected
    status/triageThis issue needs to be triaged
    on Aug 12, 2023

  2. sassanh commented on Aug 12, 2023

    @sassanh
    sassanh
    on Aug 12, 2023
    Author

    Another example, if I have this dependency:

    [tool.poetry.dependencies]
rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

    to install this package only in RPi and not in other platforms, I see this in the lock file:

    [[package]]
name = "rpi-gpio"
version = "0.7.1"
description = "A module to control Raspberry Pi GPIO channels"
optional = false
python-versions = "*"
files = [
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv6l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv7l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

    Ignoring the markers, so basically this:

    [tool.poetry.dependencies]
rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

    behaves the same as

    [tool.poetry.dependencies]
rpi-gpio = "^0.7.1"

  3. dimbleby commented on Aug 13, 2023

    @dimbleby
    dimbleby
    on Aug 13, 2023
    Contributor

    looks as though poetry forgets about what repository a package comes from when it re-solves from its lockfile during installation.

    A patch something like this should do it:

    --- a/src/poetry/installation/installer.py
+++ b/src/poetry/installation/installer.py
@@ -286,16 +286,20 @@ class Installer:
             )

         # We resolve again by only using the lock file
-        pool = RepositoryPool(ignore_repository_names=True, config=self._config)
+        pool = RepositoryPool(config=self._config)

-        # Making a new repo containing the packages
-        # newly resolved and the ones from the current lock file
-        repo = Repository("poetry-repo")
+        # Make a new collection of repositories containing the packages newly resolved
+        # and the ones from the current lock file.
         for package in lockfile_repo.packages + locked_repository.packages:
-            if not package.is_direct_origin() and not repo.has_package(package):
-                repo.add_package(package)
+            if not package.is_direct_origin():
+                repo_name = package.source_reference or "PyPI"
+                try:
+                    repo = pool.repository(repo_name)
+                except IndexError:
+                    repo = Repository(repo_name)
+                    pool.add_repository(repo)

-        pool.add_repository(repo)
+                repo.add_package(package)

    The dull bit will be figuring out a unit test which I've no intention of doing! If this is important to you - or anyone else - then contribution surely welcome.

  4. sassanh commented on Aug 13, 2023

    @sassanh
    sassanh
    on Aug 13, 2023
    Author

    Thanks! This fixes dropping sources in the lock file, but it still ignores markers in the lock file, so this is the relevant part of the generated lock file:

    [[package]]
name = "kivy"
version = "2.2.1"
description = "A software library for rapid development of hardware-accelerated multitouch applications."
optional = false
python-versions = ">=3.7"
files = [
    {file = "Kivy-2.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5aa8a0ca0708dd6edb0127c91712db7b8aad3aea3a79d498df145b8a7ee0aee6"},
    {file = "Kivy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af45f433208c16a4d0e755fafac5f2238560880a475a8a041ca9d99695a6b166"},
    {file = "Kivy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:564705e8fbd788dce22dae8dfe0488be705e14381fa59d6bdfa3df5255446429"},
    {file = "Kivy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:19cd0b0b40046d6233f0a30c8003e1ffe5a135b96ae50a5442bd62ef9d9c6744"},
    {file = "Kivy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:4915e931cb26efa016956b16a0d04bcc8eb89abcb2734ed718786c30241795bb"},
    {file = "Kivy-2.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e3df4f6230f4b125c78b1dd923f714e93341afda5b55ae9795df2d6b2c5dbc4d"},
    {file = "Kivy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0eb5de5ffea53a341790ff7ba4d3cea60d37523f0766a4783692e2aa1d50670e"},
    {file = "Kivy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a5e292f232a67bc67dfa91a205aed472dc4df7329ce059c74a88031c652efa"},
    {file = "Kivy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:d78a8d92565c6309bb1edc9a0c99883630adc0503dba25f3eb8b667ceb736395"},
    {file = "Kivy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:e317a710e0be3083901d29457fa8a203c14a71b7cef8008e3cdc7d1eb8e4d7c5"},
    {file = "Kivy-2.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7b8c8725cd1b3575b95f8f8b5d6f804e096af8958156450bb876821fbb2ce37"},
    {file = "Kivy-2.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b89bc5add63bf83fd1b026ec4d492ea9c90067b30652c14afe97cd4c30a2b5a"},
    {file = "Kivy-2.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f19ed8efaea7c880ab6342269c41ac47989812cbd036a524abc49ba9923487f"},
    {file = "Kivy-2.2.1-cp37-cp37m-win32.whl", hash = "sha256:c932fa9a8ade002ec4468c416bedda4188cda53721be84e4086b278c3b00c654"},
    {file = "Kivy-2.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7eeda699fe764c5efa2cc43e0ef28275be2a56ccd8af17933b6e2457d66d5b0d"},
    {file = "Kivy-2.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0456359afe01a94d39bb29ce8a8dd9d9dcd86cec6532f17bffb0725647a8d63a"},
    {file = "Kivy-2.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37874a7a2a9983ae494910578e2e6a4aba1568b35f915b014d85f779b5add8b9"},
    {file = "Kivy-2.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7bfbc97f66d50d71bcbc5c254e74386d94d7a1fb051448fbc1ba6742c1d1d21"},
    {file = "Kivy-2.2.1-cp38-cp38-win32.whl", hash = "sha256:0454ad509b83f5ee04dd7bb46682f077adcde99d86c206127f2dd57aedd28f5b"},
    {file = "Kivy-2.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:a6b47a8b5b47d091ac4604635edef7522ddfb9cf47b3f258cccd8ffbf094b429"},
    {file = "Kivy-2.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0e8370d529548d360803424f803231ed8ad19c7f4c351aa893fe9d39bcc940fc"},
    {file = "Kivy-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1e9cf59dc3875c943c89d3668a73235c96f4c03680c4691416ccc5d1c7f50f0"},
    {file = "Kivy-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97442522fb58ebf4038f3c57a795046e00a23f7c4dc0775b282af9aff845732a"},
    {file = "Kivy-2.2.1-cp39-cp39-win32.whl", hash = "sha256:f3dcfdce8403dc3df7a8b1a7d424c0939d41bc48a0b627403b685931a609a7ec"},
    {file = "Kivy-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:35eb3025a77894380196d5d64852da6c539e72aa2ff4970011385eb7a70c46e0"},
    {file = "Kivy-2.2.1.tar.gz", hash = "sha256:047a434e8efd32d425321e065fcccf725f165adb0c81ed69dc33c360d4796f7e"},
]

[package.dependencies]
docutils = "*"
"kivy-deps.angle" = {version = ">=0.3.3,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.glew" = {version = ">=0.3.1,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.sdl2" = {version = ">=0.6.0,<0.7.0", markers = "sys_platform == \"win32\""}
Kivy-Garden = ">=0.1.4"
pygments = "*"
pypiwin32 = {version = "*", markers = "sys_platform == \"win32\""}

[package.extras]
angle = ["kivy-deps.angle (>=0.3.3,<0.4.0)"]
base = ["docutils", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32", "requests"]
dev = ["flake8", "funcparserlib (==1.0.0a0)", "kivy-deps.glew-dev (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer-dev (>=0.3.3,<0.4.0)", "kivy-deps.sdl2-dev (>=0.6.0,<0.7.0)", "pre-commit", "pyinstaller", "pytest (>=3.6)", "pytest-asyncio (!=0.11.0)", "pytest-benchmark", "pytest-cov", "pytest-timeout", "responses", "sphinx (<=6.2.1)", "sphinxcontrib-actdiag", "sphinxcontrib-blockdiag", "sphinxcontrib-jquery", "sphinxcontrib-nwdiag", "sphinxcontrib-seqdiag"]
full = ["docutils", "ffpyplayer", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32"]
glew = ["kivy-deps.glew (>=0.3.1,<0.4.0)"]
gstreamer = ["kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
media = ["ffpyplayer", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
sdl2 = ["kivy-deps.sdl2 (>=0.6.0,<0.7.0)"]
tuio = ["oscpy"]

[[package]]
name = "kivy"
version = "2.2.1"
description = "A software library for rapid development of hardware-accelerated multitouch applications."
optional = false
python-versions = ">=3.7"
files = [
    {file = "Kivy-2.2.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:f0312dc2b154f2730acaaac1fb9b60081dc55026b2c861fd1d9e34834b5d71ff"},
    {file = "Kivy-2.2.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:f0312dc2b154f2730acaaac1fb9b60081dc55026b2c861fd1d9e34834b5d71ff"},
    {file = "Kivy-2.2.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:61e5dd7ea9d03a6ed8dcd735328049ca1ccb5d7ed5db8db6215b8106115e832a"},
    {file = "Kivy-2.2.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:61e5dd7ea9d03a6ed8dcd735328049ca1ccb5d7ed5db8db6215b8106115e832a"},
]

[package.dependencies]
docutils = "*"
"kivy-deps.angle" = {version = ">=0.3.3,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.glew" = {version = ">=0.3.1,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.sdl2" = {version = ">=0.6.0,<0.7.0", markers = "sys_platform == \"win32\""}
Kivy-Garden = ">=0.1.4"
pygments = "*"
pypiwin32 = {version = "*", markers = "sys_platform == \"win32\""}

[package.extras]
angle = ["kivy-deps.angle (>=0.3.3,<0.4.0)"]
base = ["docutils", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32", "requests"]
dev = ["flake8", "funcparserlib (==1.0.0a0)", "kivy-deps.glew-dev (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer-dev (>=0.3.3,<0.4.0)", "kivy-deps.sdl2-dev (>=0.6.0,<0.7.0)", "pre-commit", "pyinstaller", "pytest (>=3.6)", "pytest-asyncio (!=0.11.0)", "pytest-benchmark", "pytest-cov", "pytest-timeout", "responses", "sphinx (<=6.2.1)", "sphinxcontrib-actdiag", "sphinxcontrib-blockdiag", "sphinxcontrib-jquery", "sphinxcontrib-nwdiag", "sphinxcontrib-seqdiag"]
full = ["docutils", "ffpyplayer", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32"]
glew = ["kivy-deps.glew (>=0.3.1,<0.4.0)"]
gstreamer = ["kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
media = ["ffpyplayer", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
sdl2 = ["kivy-deps.sdl2 (>=0.6.0,<0.7.0)"]
tuio = ["oscpy"]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

    Since markers are dropped, it tries to install from wrong source (PyPI) on RPi.

    And for rpi-gpio with this dependency entry:

    rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

    this is the generated lock file:

    [[package]]
name = "rpi-gpio"
version = "0.7.1"
description = "A module to control Raspberry Pi GPIO channels"
optional = false
python-versions = "*"
files = [
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv6l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv7l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

    Again since markers are dropped it tries to install this package on macOS.

  5. dimbleby commented on Aug 13, 2023

    @dimbleby
    dimbleby
    on Aug 13, 2023
    Contributor

    the lockfile is fine, it doesn't need additional markers. I don't have a mac but I confirm that on my non-aarch64 linux VM the above requirement does not cause rpi-gpio to be installed

  6. radoering
    added
    area/installerRelated to the dependency installer
    and removed
    status/triageThis issue needs to be triaged
    on Aug 13, 2023

  7. sassanh commented on Aug 14, 2023

    @sassanh
    sassanh
    on Aug 14, 2023
    Author

    For me it clearly tries to install rpi-gpio on macOS and shows me this error:

    Installing dependencies from lock file

Package operations: 3 installs, 0 updates, 0 removals

  • Installing rpi-gpio (0.7.1): Failed

  RuntimeError

  Unable to find installation candidates for rpi-gpio (0.7.1)

  at ~/Library/Application Support/pypoetry/venv/lib/python3.11/site-packages/poetry/installation/chooser.py:76 in choose_for
       72│ 
       73│             links.append(link)
       74│ 
       75│         if not links:
    →  76│             raise RuntimeError(f"Unable to find installation candidates for {package}")
       77│ 
       78│         # Get the best link
       79│         chosen = max(links, key=lambda link: self._sort_key(package, link))
       80│ 

  • Installing rpi-ws281x (5.0.0): Failed

  RuntimeError

  Unable to find installation candidates for rpi-ws281x (5.0.0)

  at ~/Library/Application Support/pypoetry/venv/lib/python3.11/site-packages/poetry/installation/chooser.py:76 in choose_for
       72│ 
       73│             links.append(link)
       74│ 
       75│         if not links:
    →  76│             raise RuntimeError(f"Unable to find installation candidates for {package}")
       77│ 
       78│         # Get the best link
       79│         chosen = max(links, key=lambda link: self._sort_key(package, link))
       80│ 

  • Installing sysv-ipc (1.1.0): Failed

  RuntimeError

  Unable to find installation candidates for sysv-ipc (1.1.0)

  at ~/Library/Application Support/pypoetry/venv/lib/python3.11/site-packages/poetry/installation/chooser.py:76 in choose_for
       72│ 
       73│             links.append(link)
       74│ 
       75│         if not links:
    →  76│             raise RuntimeError(f"Unable to find installation candidates for {package}")
       77│ 
       78│         # Get the best link
       79│         chosen = max(links, key=lambda link: self._sort_key(package, link))
       80│

    I even tried the 3 commits in the linked pull request.

    Other than that I wonder how you consider this lockfile fine when it doesn't have the markers? How does the installer know it shouldn't install rpi-gpio without the markers?

  8. sassanh commented on Aug 14, 2023

    @sassanh
    sassanh
    on Aug 14, 2023
    Author

    I noticed when I put sources in this order:

    [[tool.poetry.source]]
name = "PyPI"
priority = "primary"

[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"

    it generates a lock file which doesn't try to install rpi-gpio on macOS but if I change the order it generates a lock file which tries to install rpi-gpio and fails. What is confusing for me is that the relevant part for rpi-gpio is exactly the same in both lock files which is:

    [[package]]
name = "rpi-gpio"
version = "0.7.1"
description = "A module to control Raspberry Pi GPIO channels"
optional = false
python-versions = "*"
files = [
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv6l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv7l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

  9. dimbleby commented on Aug 14, 2023

    @dimbleby
    dimbleby
    on Aug 14, 2023
    Contributor

    your mental model is wrong, the lockfile should be thought of more like a repository of available packages than a list of instructions. At install time poetry re-solves from that repository.

    top-level requirements - including their markers - are listed in pyproject.toml

    this certainly works just fine - poetry itself uses such a construction -

    poetry/pyproject.toml

    Line 61 in 2b16b2f

    xattr = { version = "^0.10.0", markers = "sys_platform == 'darwin'" }

    so I suppose you are either doing something more complicated than you have so far told us about; or the markers you have added just don't exclude the platform you are installing on.

  10. sassanh commented on Aug 14, 2023

    @sassanh
    sassanh
    on Aug 14, 2023
    Author

    Thanks for the quick responses!

    This is my complete pyproject.toml file:

    [tool.poetry]
name = "headless-kivy-pi"
version = "0.2.0"
description = ""
authors = ["Sassan Haradji <sassanh@gmail.com>"]
license = "MIT"
readme = "README.md"
packages = [{ include = "headless_kivy_pi" }]


[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"


[tool.poetry.dependencies]
python = ">=3.9,<3.12"
numpy = "^1.24.2"
kivy = [
  { version = "^2.2.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' },
  { version = "^2.2.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' },
]
typing-extensions = "^4.7.1"
adafruit-circuitpython-rgb-display = "^3.11.0"

rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }
adafruit-circuitpython-aw9523 = { version = "^1.1.7", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

fake-rpi = { version = "^0.7.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' }
watchdog = { version = "^3.0.0", markers = "platform_machine != 'aarch64'", source = 'PyPI' }
screeninfo = { version = "^0.8.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' }


[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

    If I lock this file using poetry lock and then run poetry install it tries to install 3 packages meant to be installed for RPi including rpi-gpio and will fail with the above mentioned error.
    If I change the order of sources like this:

    [tool.poetry]
name = "headless-kivy-pi"
version = "0.2.0"
description = ""
authors = ["Sassan Haradji <sassanh@gmail.com>"]
license = "MIT"
readme = "README.md"
packages = [{ include = "headless_kivy_pi" }]


[[tool.poetry.source]]
name = "PyPI"
priority = "primary"

[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"


[tool.poetry.dependencies]
python = ">=3.9,<3.12"
numpy = "^1.24.2"
kivy = [
  { version = "^2.2.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' },
  { version = "^2.2.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' },
]
typing-extensions = "^4.7.1"
adafruit-circuitpython-rgb-display = "^3.11.0"

rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }
adafruit-circuitpython-aw9523 = { version = "^1.1.7", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

fake-rpi = { version = "^0.7.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' }
watchdog = { version = "^3.0.0", markers = "platform_machine != 'aarch64'", source = 'PyPI' }
screeninfo = { version = "^0.8.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' }


[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

    And then lock it with poetry lock and run poetry install, it finishes its job successfully.

  11. dimbleby commented on Aug 14, 2023

    @dimbleby
    dimbleby
    on Aug 14, 2023
    Contributor

    I expect rpi-gpio is an indirect dependency of one of your other requirements, but on all platforms.

    Then, absent instructions to the contrary, poetry is happy to satisfy that from the piwheels repository .

  12. sassanh commented on Aug 14, 2023

    @sassanh
    sassanh
    on Aug 14, 2023
    Author

    Yes, that was it, seems like adafruit-circuitpython-rgb-display has rpi-gpio as its dependency.

    I really appreciate your help, so it should be safe to close this issue once the PR is merged.

  13. sassanh commented on Aug 15, 2023

    @sassanh
    sassanh
    on Aug 15, 2023
    Author

    OK, now my package locks and installs the dependencies without any problems, you can see it here: https://github.com/sassanh/headless-kivy-pi
    But when I try to install it by { path = "../headless-kivy-pi", develop = true } or { path = "../headless-kivy-pi/dist/....whl" } or { git = "https://github.com/sassanh/headless-kivy-pi.git" } it fails to install after locking when I have sources in this order

    [[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"

    and it installs successfully if I change the order of sources. It is on the macOS, things are opposite on the RPi (successful installation when phwheels is on top and failed installation when pypi is on top).

    If I install the git repository via pip without using poetry at all, it just works without any errors, so I guess poetry deals dependencies with pyproject.toml differently and is not just pip-installing them.

    The error message is something like this:

      • Installing numpy (1.25.2): Failed

  RuntimeError

  Unable to find installation candidates for numpy (1.25.2)

  at ~/Library/Application Support/pypoetry/venv/lib/python3.11/site-packages/poetry/installation/chooser.py:79 in choose_for
       75│ 
       76│             links.append(link)
       77│ 
       78│         if not links:
    →  79│             raise RuntimeError(
       80│                 f"Unable to find installation candidates for {package}")
       81│ 
       82│         # Get the best link
       83│         chosen = max(links, key=lambda link: self._sort_key(package, link))

    when I print links when piwheels is on top I get this:

    [<Link https://www.piwheels.org/simple/numpy/numpy-1.25.2-cp39-cp39-linux_armv7l.whl#sha256=4b0bf79e4c9ea416d3e7ddfd463263b7489c80e8fac9c94f3c9f11
ee443210b7 (requires-python:>=3.9)>, <Link https://www.piwheels.org/simple/numpy/numpy-1.25.2-cp39-cp39-linux_armv6l.whl#sha256=4b0bf79e4c9ea416d3
e7ddfd463263b7489c80e8fac9c94f3c9f11ee443210b7 (requires-python:>=3.9)>]```

so while the original issue is fixed for the dependencies, it is still an issue for the dependencies of the dependencies.

  14. dimbleby commented on Aug 15, 2023

    @dimbleby
    dimbleby
    on Aug 15, 2023
    Contributor

    sources are a poetry-special, there is no way to pass them through standard metadata, it's expected that project can know nothing about sources defined in dependency.

    if you search you will find other issues in this repository discussing this

  15. sassanh commented on Aug 15, 2023

    @sassanh
    sassanh
    on Aug 15, 2023
    Author

    there is no way to pass them through standard metadata

    It's good that sources of project A being a dependency of project B, doesn't have any effect on project B, what is happening here is that the order of sources of the project B, affects installation of project A (the reverse direction) if I can install A via pip successfully, I should be able to install it via poetry as well, and changing the order of sources in B shouldn't cause A to fail to install. Am I missing something here?

  16. sassanh commented on Aug 15, 2023

    @sassanh
    sassanh
    on Aug 15, 2023
    Author

    So to be clear what is happening here is that I have project A, I can run poetry install in it and it runs successfully, I can build a wheel out of it and install the wheel using pip install and it installs successfully. But installing it via poetry sometimes fails and it depends on the order of sources in the pyproject.toml (not the one inside A).

    To me it looks like a leak of information (and in this case a destructive leak of information) from the current project to its dependencies.

  17. radoering commented on Aug 16, 2023

    @radoering
    radoering
    on Aug 16, 2023
    Member

    That's how it is. If project B depends on project A, project B will not know about any poetry-specific information in project A's pyproject.toml, e.g. explicit sources. You basically have to repeat explicit source constraints for dependencies where it's relevant in project B's pyproject.toml. I'd propose to create an additional (optional) dependency group for transitive dependencies in such cases. For example:

    project A:

    [tool.poetry.dependencies]
kivy = [
  { version = "^2.2.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' },
  { version = "^2.2.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' },
]

    project B:

    [tool.poetry.dependencies]
project-a = ...

[tool.poetry.group.transitive]
optional = true

[tool.poetry.group.transitive.dependencies]
kivy = [
  { version = "*", markers = "platform_machine == 'aarch64'", source = 'piwheels' },
  { version = "*", markers = "platform_machine != 'aarch64'", source = 'PyPI' },
]

    I am aware that this may not be a satisfactory solution in this case but that's what Poetry's design allows.

  18. radoering
    closed this as completedin #8304on Aug 17, 2023

  19. github-actions commented on Feb 29, 2024

    @github-actions
    github-actionsBot
    on Feb 29, 2024 – with GitHub Actions

    This issue has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs.

  20. github-actions
    locked as resolved and limited conversation to collaborators on Feb 29, 2024
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    area/installerRelated to the dependency installerkind/bugSomething isn't working as expected

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions